Create & Init Project...

app/service/main/bns/agent/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "agent.go",
+        "dns.go",
+        "http.go",
+        "ns_endpoint.go",
+    ],
+    importpath = "go-common/app/service/main/bns/agent",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/main/bns/agent/backend:go_default_library",
+        "//app/service/main/bns/conf:go_default_library",
+        "//app/service/main/bns/lib/resolvconf:go_default_library",
+        "//app/service/main/bns/lib/shuffle:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/miekg/dns:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/main/bns/agent/backend:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/bns/agent/agent.go

@@ -0,0 +1,284 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/service/main/bns/agent/backend"
+	"go-common/app/service/main/bns/conf"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/stat/prom"
+)
+
+// Agent easyns agent
+type Agent struct {
+	// plugable easyns server backend
+	backend backend.Backend
+
+	// agent id, now it is os hostname
+	agentID string
+
+	// agent cfg
+	cfg *conf.Config
+
+	// agent local cache, distributed by region, zone, env
+	//caches cache.LocalCaches
+
+	// httpAddrs are the addresses per protocol the HTTP server binds to
+	httpAddrs []conf.ProtoAddr
+
+	// httpServers provides the HTTP API on various endpoints
+	httpServers []*HTTPServer
+
+	// dnsAddr is the address the DNS server binds to
+	dnsAddrs []conf.ProtoAddr
+
+	// dnsServer provides the DNS API
+	dnsServers []*DNSServer
+
+	// wgServers is the wait group for all HTTP servers
+	wgServers sync.WaitGroup
+}
+
+// New agent
+func New(c *conf.Config) (*Agent, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("get hostname as agent id failed: %s", err)
+	}
+
+	httpAddrs, err := c.HTTPAddrs()
+	if err != nil {
+		return nil, fmt.Errorf("invalid HTTP bind address: %s", err)
+	}
+
+	dnsAddrs, err := c.DNSAddrs()
+	if err != nil {
+		return nil, fmt.Errorf("invalid DNS bind address: %s", err)
+	}
+	//	var caches cache.LocalCaches
+	a := &Agent{
+		cfg:       c,
+		agentID:   hostname,
+		httpAddrs: httpAddrs,
+		dnsAddrs:  dnsAddrs,
+	}
+
+	return a, nil
+}
+
+// Start agent
+func (a *Agent) Start() error {
+	// start DNS servers
+	if err := a.listenAndServeDNS(); err != nil {
+		return err
+	}
+
+	// listen HTTP
+	httpln, err := a.listenHTTP(a.httpAddrs)
+	if err != nil {
+		return err
+	}
+
+	// initial backend
+	a.backend, err = backend.New(a.cfg.Backend.Backend, a.cfg.Backend.Config)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	// make sure backend is available
+	if err = a.backend.Ping(ctx); err != nil {
+		return err
+	}
+
+	// start serving http servers
+	for _, l := range httpln {
+		srv := NewHTTPServer(l.Addr().String(), a)
+		if err := a.serveHTTP(l, srv); err != nil {
+			return err
+		}
+		a.httpServers = append(a.httpServers, srv)
+	}
+
+	return nil
+}
+
+// Query name
+func (a *Agent) Query(name string) ([]*backend.Instance, error) {
+	target, sel, err := backend.ParseName(name, a.DefaultSel())
+	if err != nil {
+		log.Error("dns: parse name failed! name: %s, err: %s", name, err)
+		return nil, err
+	}
+	// TODO
+	inss, err := a.backend.Query(context.Background(), target, sel, backend.Metadata{})
+	if err != nil {
+		prom.BusinessErrCount.Incr("bns:query")
+	}
+	return inss, err
+}
+
+func (a *Agent) listenAndServeDNS() error {
+	notif := make(chan conf.ProtoAddr, len(a.dnsAddrs))
+	for _, p := range a.dnsAddrs {
+		p := p // capture loop var
+
+		// create server
+		svr, err := NewDNSServer(a, a.cfg.DNS)
+		if err != nil {
+			return err
+		}
+		a.dnsServers = append(a.dnsServers, svr)
+
+		// start server
+		a.wgServers.Add(1)
+		go func() {
+			defer a.wgServers.Done()
+
+			err := svr.ListenAndServe(p.Net, p.Addr, func() { notif <- p })
+			if err != nil && !strings.Contains(err.Error(), "accept") {
+				log.Error("agent: Error starting DNS server %s (%s): %v", p.Addr, p.Net, err)
+			}
+		}()
+	}
+
+	// wait for servers to be up
+	timeout := time.After(time.Second)
+	for range a.dnsAddrs {
+		select {
+		case p := <-notif:
+			log.Info("agent: Started DNS server %s (%s)", p.Addr, p.Net)
+			continue
+		case <-timeout:
+			return fmt.Errorf("agent: timeout starting DNS servers")
+		}
+	}
+	return nil
+}
+
+func (a *Agent) listenHTTP(addrs []conf.ProtoAddr) ([]net.Listener, error) {
+	var ln []net.Listener
+	for _, p := range addrs {
+		var l net.Listener
+		var err error
+
+		switch {
+		case p.Net == "unix":
+			l, err = a.listenSocket(p.Addr)
+		case p.Net == "tcp" && p.Proto == "http":
+			l, err = net.Listen("tcp", p.Addr)
+		default:
+			return nil, fmt.Errorf("%s:%s listener not supported", p.Net, p.Proto)
+		}
+
+		if err != nil {
+			for _, l := range ln {
+				l.Close()
+			}
+			return nil, err
+		}
+
+		if tcpl, ok := l.(*net.TCPListener); ok {
+			l = &tcpKeepAliveListener{tcpl}
+		}
+
+		ln = append(ln, l)
+	}
+	return ln, nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by NewHttpServer so dead TCP connections
+// eventually go away.
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(30 * time.Second)
+	return tc, nil
+}
+
+func (a *Agent) listenSocket(path string) (net.Listener, error) {
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		log.Warn("agent: Replacing socket %q\n", path)
+	}
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("error removing socket file: %s", err)
+	}
+	l, err := net.Listen("unix", path)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: set file permissions.
+	return l, nil
+}
+
+func (a *Agent) serveHTTP(l net.Listener, srv *HTTPServer) error {
+	srv.proto = "http"
+	notif := make(chan string)
+	a.wgServers.Add(1)
+	go func() {
+		defer a.wgServers.Done()
+		notif <- srv.Addr
+		err := srv.Serve(l)
+		if err != nil && err != http.ErrServerClosed {
+			log.Error("agent: Error starting http service: %s\n", err.Error())
+		}
+	}()
+
+	select {
+	case addr := <-notif:
+		log.Info("agent: Started HTTP Server on %s\n", addr)
+		return nil
+	case <-time.After(time.Second):
+		return fmt.Errorf("agent: timeout starting HTTP servers")
+	}
+}
+
+// ShutDown agent
+func (a *Agent) ShutDown(ctx context.Context) error {
+	errmsg := make([]string, 0)
+	for _, hsrv := range a.httpServers {
+		if err := hsrv.Shutdown(ctx); err != nil {
+			errmsg = append(errmsg, err.Error())
+		}
+	}
+	for _, dsrv := range a.dnsServers {
+		if err := dsrv.Shutdown(); err != nil {
+			errmsg = append(errmsg, err.Error())
+		}
+	}
+	err := a.backend.Close(ctx)
+	if err != nil {
+		errmsg = append(errmsg, err.Error())
+	}
+	if len(errmsg) > 0 {
+		return fmt.Errorf("%s", strings.Join(errmsg, "\n"))
+	}
+	return nil
+}
+
+// DefaultSel default selector from config
+func (a *Agent) DefaultSel() backend.Selector {
+	return backend.Selector{
+		Env:    env.DeployEnv,
+		Region: env.Region,
+		Zone:   env.Zone,
+	}
+}

app/service/main/bns/agent/backend/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["backend_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["backend.go"],
+    importpath = "go-common/app/service/main/bns/agent/backend",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/main/bns/agent/backend/discovery:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/bns/agent/backend/backend.go

@@ -0,0 +1,97 @@
+package backend
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+)
+
+var factoryMap map[string]Factory
+
+func init() {
+	factoryMap = make(map[string]Factory)
+}
+
+// Factory backend factory
+type Factory func(map[string]interface{}) (Backend, error)
+
+// Registry registry backdend
+func Registry(name string, factory Factory) {
+	if _, ok := factoryMap[name]; ok {
+		panic(fmt.Sprintf("backend %s already exists", name))
+	}
+	factoryMap[name] = factory
+}
+
+// New backend
+func New(name string, conf map[string]interface{}) (Backend, error) {
+	if factory, ok := factoryMap[name]; ok {
+		return factory(conf)
+	}
+	return nil, fmt.Errorf("backend %s not exists", name)
+}
+
+// Selector selector
+type Selector struct {
+	Env      string
+	Region   string
+	Zone     string
+	Hostname string
+}
+
+func (s Selector) String() string {
+	strs := make([]string, 0, 4)
+	if s.Env != "" {
+		strs = append(strs, s.Env)
+	}
+	if s.Region != "" {
+		strs = append(strs, s.Region)
+	}
+	if s.Zone != "" {
+		strs = append(strs, s.Zone)
+	}
+	if s.Hostname != "" {
+		strs = append(strs, s.Hostname)
+	}
+	return strings.Join(strs, "-")
+}
+
+// Target global unique application identifier
+type Target struct {
+	Name string
+}
+
+func (t Target) String() string {
+	return t.Name
+}
+
+// ParseName parse qname get name and selector
+func ParseName(name string, defaultSel Selector) (target Target, sel Selector, err error) {
+	// TODO: support selector
+	return Target{Name: name}, defaultSel, nil
+}
+
+// Metadata metadata contain env, zone, region e.g.
+type Metadata struct {
+	ClientHost       string
+	LatestTimestamps string
+}
+
+// Instance service instance struct
+type Instance struct {
+	Region      string `json:"region"`
+	Zone        string `json:"zone"`
+	Env         string `json:"env"`
+	Hostname    string `json:"hostname"`
+	DiscoveryID string `json:"discovery_id"`
+	TreeID      int64  `json:"tree_id"`
+	IPAddr      net.IP `json:"ip_addr,omitempty"` // hacked field
+}
+
+// Backend provide service query
+type Backend interface {
+	Ping(ctx context.Context) error
+	Query(ctx context.Context, target Target, sel Selector, md Metadata) ([]*Instance, error)
+	Close(ctx context.Context) error
+}

app/service/main/bns/agent/backend/backend_test.go

@@ -0,0 +1 @@
+package backend

app/service/main/bns/agent/backend/discovery/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["discovery_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/bns/agent/backend:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["discovery.go"],
+    importpath = "go-common/app/service/main/bns/agent/backend/discovery",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/main/bns/agent/backend:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/bns/agent/backend/discovery/discovery.go

@@ -0,0 +1,355 @@
+package discovery
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/library/log"
+	"go-common/library/naming"
+	"go-common/library/stat/prom"
+
+	"go-common/app/service/main/bns/agent/backend"
+)
+
+func init() {
+	backend.Registry("discovery", New)
+}
+
+const (
+	// NodeStatusUP Ready to receive register
+	NodeStatusUP NodeStatus = iota
+	// NodeStatusLost lost with each other
+	NodeStatusLost
+	defaultCacheExpireIn int64 = 10
+)
+
+// NodeStatus Status of instance
+type NodeStatus int
+
+// ServerNode backend servier node status
+type ServerNode struct {
+	Addr   string     `json:"addr"`
+	Zone   string     `json:"zone"`
+	Status NodeStatus `json:"status"`
+}
+
+// InstanceList discovery instance list
+type InstanceList struct {
+	Instances       []naming.Instance `json:"instances"`
+	LatestTimestamp int64             `json:"latest_timestamp"`
+}
+
+// InstanceMetadata discovery instance metadata
+type InstanceMetadata struct {
+	Provider interface{} `json:"provider"`
+}
+
+type discoveryResp struct {
+	Code    int64  `json:"code"`
+	Message string `json:"message"`
+}
+
+type discoveryFetchResp struct {
+	discoveryResp `json:",inline"`
+	Data          *InstanceList `json:"data"`
+}
+
+type discoveryNodeResp struct {
+	discoveryResp `json:",inline"`
+	Data          []ServerNode `json:"data"`
+}
+
+var urlParse = url.Parse
+
+// New discovery backend
+func New(config map[string]interface{}) (backend.Backend, error) {
+	if config == nil {
+		return nil, fmt.Errorf("discovery require url, secret, appkey")
+	}
+	var url, secret, appKey string
+	var ok bool
+
+	discoveryURL := os.Getenv("DISCOVERY_URL")
+	if url, ok = config["url"].(string); !ok && discoveryURL == "" {
+		return nil, fmt.Errorf("discovery require `url`")
+	}
+	// use env DISCOVERY_URL overwrite config
+	if discoveryURL != "" {
+		url = discoveryURL
+	}
+
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		url = "http://" + url
+	}
+
+	refreshInterval := time.Minute
+	if second, ok := config["refresh_interval"].(int); ok {
+		refreshInterval = time.Duration(second) * time.Second
+	}
+	cacheExpireIn := defaultCacheExpireIn
+	if second, ok := config["cacheExpireIn"].(int); ok {
+		cacheExpireIn = int64(second)
+	}
+	u, err := urlParse(url)
+	if err != nil {
+		return nil, err
+	}
+	dis := &discovery{
+		client:         http.DefaultClient,
+		scheme:         u.Scheme,
+		host:           u.Host,
+		discoveryHosts: []string{u.Host},
+		secret:         secret,
+		appKey:         appKey,
+		refreshTick:    time.NewTicker(refreshInterval),
+		cacheMap:       make(map[string]*cacheNode),
+		cacheExpireIn:  cacheExpireIn,
+	}
+	go dis.daemon()
+	return dis, nil
+}
+
+type cacheNode struct {
+	expired int64
+	data    []*backend.Instance
+}
+
+func (c *cacheNode) IsExpired() bool {
+	return time.Now().Unix() > c.expired
+}
+
+func newCacheNode(data []*backend.Instance, expireIn int64) *cacheNode {
+	return &cacheNode{expired: time.Now().Unix() + expireIn, data: data}
+}
+
+type discovery struct {
+	client *http.Client
+
+	secret string
+	appKey string
+
+	host   string
+	scheme string
+
+	discoveryHosts []string
+	rmx            sync.RWMutex
+
+	cachermx      sync.RWMutex
+	cacheMap      map[string]*cacheNode
+	cacheExpireIn int64
+
+	refreshTick *time.Ticker
+}
+
+var _ backend.Backend = &discovery{}
+
+func (d *discovery) Ping(ctx context.Context) error {
+	_, err := d.Nodes(ctx)
+	return err
+}
+
+func (d *discovery) Close(ctx context.Context) error {
+	d.refreshTick.Stop()
+	return nil
+}
+
+func (d *discovery) daemon() {
+	for range d.refreshTick.C {
+		log.V(10).Info("refresh discovery nodes ...")
+		nodes, err := d.Nodes(context.Background())
+		if err != nil {
+			log.Error("refresh discovery nodes error %s", err)
+			continue
+		}
+		hosts := make([]string, 0, len(nodes))
+		for i := range nodes {
+			if nodes[i].Status == NodeStatusUP {
+				hosts = append(hosts, nodes[i].Addr)
+			}
+		}
+		d.rmx.Lock()
+		d.discoveryHosts = hosts
+		d.rmx.Unlock()
+		log.V(10).Info("new discovery nodes list %v", hosts)
+	}
+}
+
+func (d *discovery) Nodes(ctx context.Context) ([]ServerNode, error) {
+	req, err := http.NewRequest(http.MethodGet, "/discovery/nodes", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := d.do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	nodesResp := &discoveryNodeResp{}
+	if err := json.NewDecoder(resp.Body).Decode(nodesResp); err != nil {
+		return nil, err
+	}
+
+	if nodesResp.Code < 0 || nodesResp.Data == nil || len(nodesResp.Data) == 0 {
+		return nil, fmt.Errorf("no data found, err: %s", nodesResp.Message)
+	}
+
+	log.V(10).Info("responsed data: %v", nodesResp.Data)
+	return nodesResp.Data, nil
+}
+
+func (d *discovery) do(req *http.Request) (resp *http.Response, err error) {
+	req.URL.Scheme = d.scheme
+	req.Host = d.host
+	req.URL.Scheme = d.scheme
+	req.Host = d.host
+	d.rmx.RLock()
+	hosts := d.discoveryHosts
+	d.rmx.RUnlock()
+	for _, host := range shuffle(hosts) {
+		req.URL.Host = host
+		resp, err = d.client.Do(req)
+		log.V(5).Info("request discovery.. request: %s", req.URL)
+		if err == nil {
+			return
+		}
+		log.Error("request discovery %s err %s, try next", host, err)
+	}
+	return
+}
+
+// Query appid
+func (d *discovery) Query(ctx context.Context, target backend.Target, sel backend.Selector, md backend.Metadata) ([]*backend.Instance, error) {
+	// TODO: parallel query
+	key := target.Name + sel.String()
+	if data, ok := d.fromCache(key); ok {
+		prom.CacheHit.Incr("bns:discovery_mem_cache_hit")
+		return data, nil
+	}
+	prom.CacheMiss.Incr("bns:discovery_mem_cache_miss")
+	instanceList, err := d.fetch(ctx, target.Name, sel, md)
+	if err != nil {
+		return nil, err
+	}
+	data := copyInstance(instanceList)
+	if len(data) != 0 {
+		d.setCache(key, data)
+	}
+	return data, nil
+}
+
+func (d *discovery) fromCache(key string) ([]*backend.Instance, bool) {
+	d.cachermx.RLock()
+	defer d.cachermx.RUnlock()
+	node, ok := d.cacheMap[key]
+	if !ok || node.IsExpired() {
+		return nil, false
+	}
+	return node.data, true
+}
+
+func (d *discovery) setCache(key string, data []*backend.Instance) {
+	d.cachermx.Lock()
+	defer d.cachermx.Unlock()
+	d.cacheMap[key] = newCacheNode(data, d.cacheExpireIn)
+}
+
+func (d *discovery) fetch(ctx context.Context, discoveryID string, sel backend.Selector, md backend.Metadata) (*InstanceList, error) {
+	params := url.Values{}
+	params.Add("appid", discoveryID)
+	params.Add("env", sel.Env)
+	params.Add("region", sel.Region)
+	params.Add("hostname", md.ClientHost)
+	params.Add("zone", sel.Zone)
+	params.Add("status", "1")
+
+	if md.LatestTimestamps != "" {
+		params.Add("latest_timestamp", md.LatestTimestamps)
+	} else {
+		params.Add("latest_timestamp", "0")
+	}
+
+	payload := params.Encode()
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/discovery/fetch?%s", payload), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := d.do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	fetchResp := &discoveryFetchResp{}
+	if err := json.NewDecoder(resp.Body).Decode(fetchResp); err != nil {
+		return nil, err
+	}
+
+	if fetchResp.Code < 0 || fetchResp.Data == nil || fetchResp.Data.Instances == nil || len(fetchResp.Data.Instances) == 0 {
+		return nil, fmt.Errorf("no data found, err: %s", fetchResp.Message)
+	}
+
+	log.V(10).Info("fetchResponsed data: %v", fetchResp.Data)
+
+	return fetchResp.Data, nil
+}
+
+func shuffle(hosts []string) []string {
+	for i := 0; i < len(hosts); i++ {
+		j := rand.Intn(len(hosts) - i)
+		hosts[i], hosts[i+j] = hosts[i+j], hosts[i]
+	}
+	return hosts
+}
+
+func copyInstance(il *InstanceList) (inss []*backend.Instance) {
+copyloop:
+	for _, in := range il.Instances {
+		out := &backend.Instance{
+			DiscoveryID: in.AppID,
+			Env:         in.Env,
+			Hostname:    in.Hostname,
+			Zone:        in.Zone,
+		}
+		for _, addr := range in.Addrs {
+			ip, err := ipFromURI(addr)
+			if err == nil {
+				out.IPAddr = ip
+				inss = append(inss, out)
+				continue copyloop
+			}
+			log.Error("extract ip from addr %s error: %s", addr, err)
+		}
+		log.Error("can't found any ip for discoveryID: %s", in.AppID)
+	}
+	return
+}
+
+// extract ip from uri
+func ipFromURI(uri string) (net.IP, error) {
+	var hostport string
+	if u, err := url.Parse(uri); err != nil {
+		hostport = uri
+	} else {
+		hostport = u.Host
+	}
+	if strings.ContainsRune(hostport, ':') {
+		host, _, err := net.SplitHostPort(hostport)
+		if err != nil {
+			return net.IPv4zero, err
+		}
+		return net.ParseIP(host), nil
+	}
+	return net.ParseIP(hostport), nil
+}

app/service/main/bns/agent/backend/discovery/discovery_test.go

@@ -0,0 +1,87 @@
+package discovery
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"go-common/app/service/main/bns/agent/backend"
+	"go-common/library/log"
+)
+
+func init() {
+	log.Init(&log.Config{
+		Stdout: true,
+	})
+}
+
+var (
+	// test discovery
+	testURL    = "http://api.bilibili.co"
+	testSecret = "b370880d1aca7d3a289b9b9a7f4d6812"
+	testAppKey = "0c4b8fe3ff35a4b6"
+
+	// test app
+	testAppID          = "middleware.databus"
+	testApplicationEnv = "uat"
+	testZone           = "sh001"
+	testRegion         = "sh"
+)
+
+var dis *discovery
+
+func TestMain(m *testing.M) {
+	config := map[string]interface{}{
+		"url":    testURL,
+		"secret": testSecret,
+		"appKey": testAppKey,
+	}
+	backend, err := New(config)
+	if err != nil {
+		log.Error("new discovery error %s", err)
+		os.Exit(1)
+	}
+	dis = backend.(*discovery)
+	os.Exit(m.Run())
+}
+
+func TestNodes(t *testing.T) {
+	ctx := context.Background()
+	nodes, err := dis.Nodes(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", nodes)
+}
+
+func TestQuery(t *testing.T) {
+	ctx := context.Background()
+	appID, sel, err := backend.ParseName(testAppID, backend.Selector{Env: testApplicationEnv, Region: testRegion, Zone: testZone})
+	if err != nil {
+		t.Fatal(err)
+	}
+	instances, err := dis.Query(ctx, appID, sel, backend.Metadata{
+		ClientHost: "locahost",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%v", instances)
+}
+
+func BenchmarkQuery(b *testing.B) {
+	ctx := context.Background()
+	appID, sel, err := backend.ParseName(testAppID, backend.Selector{Env: testApplicationEnv, Region: testRegion, Zone: testZone})
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < b.N; i++ {
+		data, err := dis.Query(ctx, appID, sel, backend.Metadata{ClientHost: "locahost"})
+		if err != nil {
+			b.Error(err)
+		}
+		if len(data) == 0 {
+			b.Error("not data found")
+		}
+	}
+}

app/service/main/bns/agent/dns.go

@@ -0,0 +1,497 @@
+package agent
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"go-common/app/service/main/bns/conf"
+	"go-common/app/service/main/bns/lib/resolvconf"
+	"go-common/app/service/main/bns/lib/shuffle"
+	"go-common/library/log"
+	"go-common/library/stat/prom"
+)
+
+const (
+	maxRecurseRecords = 5
+)
+
+var grpcPrefixs = []string{"_grpclb._tcp.", "_grpc_config."}
+
+var dnsProm = prom.New().WithTimer("go_bns_server", []string{"time"})
+
+func wrapProm(handler func(dns.ResponseWriter, *dns.Msg)) func(dns.ResponseWriter, *dns.Msg) {
+	return func(w dns.ResponseWriter, m *dns.Msg) {
+		start := time.Now()
+		handler(w, m)
+		dt := int64(time.Since(start) / time.Millisecond)
+		dnsProm.Timing("bns:dns_query", dt)
+	}
+}
+
+// DNSServer is used to wrap an Agent and expose various
+// service discovery endpoints using a DNS interface.
+type DNSServer struct {
+	*dns.Server
+	cfg       *conf.DNSServer
+	agent     *Agent
+	domain    string
+	recursors []string
+
+	// disableCompression is the config.DisableCompression flag that can
+	// be safely changed at runtime. It always contains a bool and is
+	// initialized with the value from config.DisableCompression.
+	disableCompression atomic.Value
+
+	udpClient *dns.Client
+	tcpClient *dns.Client
+}
+
+// NewDNSServer new dns server
+func NewDNSServer(a *Agent, cfg *conf.DNSServer) (*DNSServer, error) {
+	var recursors []string
+	var confRecursors = cfg.Config.Recursors
+	if len(confRecursors) == 0 {
+		resolv, err := resolvconf.ParseResolvConf()
+		if err != nil {
+			log.Warn("read resolv.conf error: %s", err)
+		} else {
+			confRecursors = resolv
+		}
+	}
+	for _, r := range confRecursors {
+		ra, err := recursorAddr(r)
+		if err != nil {
+			return nil, fmt.Errorf("Invalid recursor address: %v", err)
+		}
+		recursors = append(recursors, ra)
+	}
+
+	log.Info("recursors %v", recursors)
+
+	// Make sure domain is FQDN, make it case insensitive for ServeMux
+	domain := dns.Fqdn(strings.ToLower(cfg.Config.Domain))
+
+	srv := &DNSServer{
+		agent:     a,
+		domain:    domain,
+		recursors: recursors,
+		cfg:       cfg,
+
+		udpClient: &dns.Client{Net: "udp", Timeout: time.Duration(cfg.Config.RecursorTimeout)},
+		tcpClient: &dns.Client{Net: "tcp", Timeout: time.Duration(cfg.Config.RecursorTimeout)},
+	}
+	srv.disableCompression.Store(cfg.Config.DisableCompression)
+
+	return srv, nil
+}
+
+// ListenAndServe listen and serve dns
+func (s *DNSServer) ListenAndServe(network, addr string, notif func()) error {
+	mux := dns.NewServeMux()
+	mux.HandleFunc("arpa.", wrapProm(s.handlePtr))
+	mux.HandleFunc(".", wrapProm(s.handleRecurse))
+	mux.HandleFunc(s.domain, wrapProm(s.handleQuery))
+
+	s.Server = &dns.Server{
+		Addr:              addr,
+		Net:               network,
+		Handler:           mux,
+		NotifyStartedFunc: notif,
+	}
+	if network == "udp" {
+		s.UDPSize = 65535
+	}
+	return s.Server.ListenAndServe()
+}
+
+// recursorAddr is used to add a port to the recursor if omitted.
+func recursorAddr(recursor string) (string, error) {
+	// Add the port if none
+START:
+	_, _, err := net.SplitHostPort(recursor)
+	if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+		recursor = fmt.Sprintf("%s:%d", recursor, 53)
+		goto START
+	}
+	if err != nil {
+		return "", err
+	}
+
+	// Get the address
+	addr, err := net.ResolveTCPAddr("tcp", recursor)
+	if err != nil {
+		return "", err
+	}
+
+	// Return string
+	return addr.String(), nil
+}
+
+// handlePtr is used to handle "reverse" DNS queries
+func (s *DNSServer) handlePtr(resp dns.ResponseWriter, req *dns.Msg) {
+	q := req.Question[0]
+	defer func(s time.Time) {
+		log.V(5).Info("dns: request for %v (%v) from client %s (%s)",
+			q, time.Since(s), resp.RemoteAddr().String(),
+			resp.RemoteAddr().Network())
+	}(time.Now())
+
+	// Setup the message response
+	m := new(dns.Msg)
+	m.SetReply(req)
+	m.Compress = !s.disableCompression.Load().(bool)
+	m.Authoritative = true
+	m.RecursionAvailable = (len(s.recursors) > 0)
+
+	// Only add the SOA if requested
+	if req.Question[0].Qtype == dns.TypeSOA {
+		s.addSOA(m)
+	}
+
+	// Get the QName without the domain suffix
+	qName := strings.ToLower(dns.Fqdn(req.Question[0].Name))
+
+	// FIXME: should return multiple nameservers?
+	log.V(5).Info("dns: we said handled ptr with %v", qName)
+
+	// nothing found locally, recurse
+	if len(m.Answer) == 0 {
+		s.handleRecurse(resp, req)
+		return
+	}
+
+	// Enable EDNS if enabled
+	if edns := req.IsEdns0(); edns != nil {
+		m.SetEdns0(edns.UDPSize(), false)
+	}
+
+	// Write out the complete response
+	if err := resp.WriteMsg(m); err != nil {
+		log.Warn("dns: failed to respond: %v", err)
+	}
+}
+
+// handleQuery is used to handle DNS queries in the configured domain
+func (s *DNSServer) handleQuery(resp dns.ResponseWriter, req *dns.Msg) {
+	q := req.Question[0]
+	defer func(s time.Time) {
+		log.V(5).Info("dns: request for %v (%v) from client %s (%s)",
+			q, time.Since(s), resp.RemoteAddr().String(),
+			resp.RemoteAddr().Network())
+	}(time.Now())
+
+	// Switch to TCP if the client is
+	network := "udp"
+	if _, ok := resp.RemoteAddr().(*net.TCPAddr); ok {
+		network = "tcp"
+	}
+
+	// Setup the message response
+	m := new(dns.Msg)
+	m.SetReply(req)
+	m.Compress = !s.disableCompression.Load().(bool)
+	m.Authoritative = true
+	m.RecursionAvailable = (len(s.recursors) > 0)
+
+	switch req.Question[0].Qtype {
+	case dns.TypeSOA:
+		ns, glue := s.nameservers(req.IsEdns0() != nil)
+		m.Answer = append(m.Answer, s.soa())
+		m.Ns = append(m.Ns, ns...)
+		m.Extra = append(m.Extra, glue...)
+		m.SetRcode(req, dns.RcodeSuccess)
+
+	case dns.TypeNS:
+		ns, glue := s.nameservers(req.IsEdns0() != nil)
+		m.Answer = ns
+		m.Extra = glue
+		m.SetRcode(req, dns.RcodeSuccess)
+
+	case dns.TypeAXFR:
+		m.SetRcode(req, dns.RcodeNotImplemented)
+
+	default:
+		s.dispatch(network, req, m)
+	}
+
+	// Handle EDNS
+	if edns := req.IsEdns0(); edns != nil {
+		m.SetEdns0(edns.UDPSize(), false)
+	}
+
+	// Write out the complete response
+	if err := resp.WriteMsg(m); err != nil {
+		log.Warn("dns: failed to respond: %v", err)
+	}
+}
+
+func (s *DNSServer) soa() *dns.SOA {
+	return &dns.SOA{
+		Hdr: dns.RR_Header{
+			Name:   s.domain,
+			Rrtype: dns.TypeSOA,
+			Class:  dns.ClassINET,
+			Ttl:    0,
+		},
+		Ns:     "ns." + s.domain,
+		Serial: uint32(time.Now().Unix()),
+
+		// todo(fs): make these configurable
+		Mbox:    "hostmaster." + s.domain,
+		Refresh: 3600,
+		Retry:   600,
+		Expire:  86400,
+		Minttl:  30,
+	}
+}
+
+// addSOA is used to add an SOA record to a message for the given domain
+func (s *DNSServer) addSOA(msg *dns.Msg) {
+	msg.Ns = append(msg.Ns, s.soa())
+}
+
+// formatNodeRecord takes an Easyns Agent node and returns an A, AAAA, or CNAME record
+func (s *DNSServer) formatNodeRecord(addr, qName string, qType uint16, ttl time.Duration, edns bool) (records []dns.RR) {
+	// Parse the IP
+	ip := net.ParseIP(addr)
+	var ipv4 net.IP
+	if ip != nil {
+		ipv4 = ip.To4()
+	}
+	switch {
+	case ipv4 != nil && (qType == dns.TypeANY || qType == dns.TypeA):
+		return []dns.RR{&dns.A{
+			Hdr: dns.RR_Header{
+				Name:   qName,
+				Rrtype: dns.TypeA,
+				Class:  dns.ClassINET,
+				Ttl:    uint32(ttl / time.Second),
+			},
+			A: ip,
+		}}
+
+	case ip != nil && ipv4 == nil && (qType == dns.TypeANY || qType == dns.TypeAAAA):
+		return []dns.RR{&dns.AAAA{
+			Hdr: dns.RR_Header{
+				Name:   qName,
+				Rrtype: dns.TypeAAAA,
+				Class:  dns.ClassINET,
+				Ttl:    uint32(ttl / time.Second),
+			},
+			AAAA: ip,
+		}}
+
+	case ip == nil && (qType == dns.TypeANY || qType == dns.TypeCNAME ||
+		qType == dns.TypeA || qType == dns.TypeAAAA):
+		// Get the CNAME
+		cnRec := &dns.CNAME{
+			Hdr: dns.RR_Header{
+				Name:   qName,
+				Rrtype: dns.TypeCNAME,
+				Class:  dns.ClassINET,
+				Ttl:    uint32(ttl / time.Second),
+			},
+			Target: dns.Fqdn(addr),
+		}
+		records = append(records, cnRec)
+
+		// Recurse
+		more := s.resolveCNAME(cnRec.Target)
+		extra := 0
+	MORE_REC:
+		for _, rr := range more {
+			switch rr.Header().Rrtype {
+			case dns.TypeCNAME, dns.TypeA, dns.TypeAAAA:
+				records = append(records, rr)
+				extra++
+				if extra == maxRecurseRecords && !edns {
+					break MORE_REC
+				}
+			}
+		}
+	}
+	return records
+}
+
+// nameservers returns the names and ip addresses of up to three random servers
+// in the current cluster which serve as authoritative name servers for zone.
+func (s *DNSServer) nameservers(edns bool) (ns []dns.RR, extra []dns.RR) {
+	// TODO: get list of bns dns nameservers
+	// Then, construct them into NS RR.
+	// We just hardcode here right now...
+	name := "bns"
+	addr := s.agent.cfg.DNS.Addr
+	fqdn := name + "." + s.domain
+	fqdn = dns.Fqdn(strings.ToLower(fqdn))
+
+	// NS record
+	nsrr := &dns.NS{
+		Hdr: dns.RR_Header{
+			Name:   s.domain,
+			Rrtype: dns.TypeNS,
+			Class:  dns.ClassINET,
+			Ttl:    uint32(time.Duration(s.agent.cfg.DNS.Config.TTL) / time.Second),
+		},
+		Ns: fqdn,
+	}
+	ns = append(ns, nsrr)
+	// A or AAAA glue record
+	glue := s.formatNodeRecord(addr, fqdn, dns.TypeANY, time.Duration(s.agent.cfg.DNS.Config.TTL), edns)
+	extra = append(extra, glue...)
+
+	return
+}
+
+func trimDomainSuffix(name string, domain string) (reversed string) {
+	reversed = strings.TrimSuffix(name, "."+domain)
+	return strings.Trim(reversed, ".")
+}
+
+// Answers answers
+type Answers []dns.RR
+
+// Len the number of answer
+func (as Answers) Len() int {
+	return len(as)
+}
+
+// Swap order
+func (as Answers) Swap(i, j int) {
+	as[i], as[j] = as[j], as[i]
+}
+
+// dispatch is used to parse a request and invoke the correct handler
+func (s *DNSServer) dispatch(network string, req, resp *dns.Msg) {
+	var answers Answers
+	// Get the QName
+	qName := strings.ToLower(dns.Fqdn(req.Question[0].Name))
+	name := trimDomainSuffix(qName, s.agent.cfg.DNS.Config.Domain)
+	for _, prefix := range grpcPrefixs {
+		name = strings.TrimPrefix(name, prefix)
+	}
+	inss, err := s.agent.Query(name)
+	if err != nil {
+		log.Error("dns: query %s failed to resolve from bns server, err: %s", name, err)
+		goto INVALID
+	}
+
+	if len(inss) == 0 {
+		log.Error("dns: QName %s has no upstreams found!", qName)
+		goto INVALID
+	}
+
+	for _, ins := range inss {
+		answers = append(answers, &dns.A{
+			Hdr: dns.RR_Header{
+				Name:   qName,
+				Rrtype: dns.TypeA,
+				Class:  dns.ClassINET,
+				Ttl:    uint32(time.Duration(s.agent.cfg.DNS.Config.TTL) / time.Second),
+			},
+			A: ins.IPAddr,
+		})
+		log.V(5).Info("dns: QName resolved ipAddress: %s - %s", qName, ins.IPAddr)
+	}
+	shuffle.Shuffle(answers)
+	resp.Answer = []dns.RR(answers)
+	return
+INVALID:
+	s.addSOA(resp)
+	resp.SetRcode(req, dns.RcodeNameError)
+}
+
+// handleRecurse is used to handle recursive DNS queries
+func (s *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
+	q := req.Question[0]
+	network := "udp"
+	client := s.udpClient
+
+	defer func(s time.Time) {
+		log.V(5).Info("dns: request for %v (%s) (%v) from client %s (%s)",
+			q, network, time.Since(s), resp.RemoteAddr().String(),
+			resp.RemoteAddr().Network())
+	}(time.Now())
+
+	// Switch to TCP if the client is
+	if _, ok := resp.RemoteAddr().(*net.TCPAddr); ok {
+		network = "tcp"
+		client = s.tcpClient
+	}
+
+	for _, recursor := range s.recursors {
+		r, rtt, err := client.Exchange(req, recursor)
+		if err == nil || err == dns.ErrTruncated {
+			// Compress the response; we don't know if the incoming
+			// response was compressed or not, so by not compressing
+			// we might generate an invalid packet on the way out.
+			r.Compress = !s.disableCompression.Load().(bool)
+
+			// Forward the response
+			log.V(5).Info("dns: recurse RTT for %v (%v)", q, rtt)
+			if err = resp.WriteMsg(r); err != nil {
+				log.Warn("dns: failed to respond: %v", err)
+			}
+			return
+		}
+		log.Error("dns: recurse failed: %v", err)
+	}
+
+	// If all resolvers fail, return a SERVFAIL message
+	log.Error("dns: all resolvers failed for %v from client %s (%s)",
+		q, resp.RemoteAddr().String(), resp.RemoteAddr().Network())
+	m := &dns.Msg{}
+	m.SetReply(req)
+	m.Compress = !s.disableCompression.Load().(bool)
+	m.RecursionAvailable = true
+	m.SetRcode(req, dns.RcodeServerFailure)
+	if edns := req.IsEdns0(); edns != nil {
+		m.SetEdns0(edns.UDPSize(), false)
+	}
+	resp.WriteMsg(m)
+}
+
+// resolveCNAME is used to recursively resolve CNAME records
+func (s *DNSServer) resolveCNAME(name string) []dns.RR {
+	// If the CNAME record points to a Easyns Name address, resolve it internally
+	// Convert query to lowercase because DNS is case insensitive; d.domain is
+	// already converted
+	if strings.HasSuffix(strings.ToLower(name), "."+s.domain) {
+		req := &dns.Msg{}
+		resp := &dns.Msg{}
+
+		req.SetQuestion(name, dns.TypeANY)
+		s.dispatch("udp", req, resp)
+
+		return resp.Answer
+	}
+
+	// Do nothing if we don't have a recursor
+	if len(s.recursors) == 0 {
+		return nil
+	}
+
+	// Ask for any A records
+	m := new(dns.Msg)
+	m.SetQuestion(name, dns.TypeA)
+
+	// Make a DNS lookup request
+	c := &dns.Client{Net: "udp", Timeout: time.Duration(s.agent.cfg.DNS.Config.RecursorTimeout)}
+	var r *dns.Msg
+	var rtt time.Duration
+	var err error
+	for _, recursor := range s.recursors {
+		r, rtt, err = c.Exchange(m, recursor)
+		if err == nil {
+			log.V(5).Info("dns: cname recurse RTT for %v (%v)", name, rtt)
+			return r.Answer
+		}
+		log.Error("dns: cname recurse failed for %v: %v", name, err)
+	}
+	log.Error("dns: all resolvers failed for %v", name)
+	return nil
+}

app/service/main/bns/agent/http.go

@@ -0,0 +1,97 @@
+package agent
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"go-common/library/log"
+)
+
+// HTTPServer provides an HTTP api for an agent.
+type HTTPServer struct {
+	*http.Server
+	agent *Agent
+
+	// proto is filled by the agent to "http" or "https".
+	proto string
+}
+
+// NewHTTPServer http server provide simple query api
+func NewHTTPServer(addr string, a *Agent) *HTTPServer {
+	s := &HTTPServer{
+		Server: &http.Server{Addr: addr},
+		agent:  a,
+	}
+	s.Server.Handler = s.handler()
+	return s
+}
+
+// handler is used to attach our handlers to the mux
+func (s *HTTPServer) handler() http.Handler {
+	mux := http.NewServeMux()
+	// TODO: simple manage ui
+
+	// API V1
+	mux.HandleFunc("/v1/naming/", s.wrap(s.NSTranslation)) // naming path
+
+	mux.Handle("/metrics", promhttp.Handler())
+
+	return mux
+}
+
+// wrap is used to wrap functions to make them more convenient
+func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Request) (interface{}, error)) http.HandlerFunc {
+	return func(resp http.ResponseWriter, req *http.Request) {
+		logURL := req.URL.String()
+		handleErr := func(err error) {
+			log.Error("http: Request %s %v from %s, err: %v\n", req.Method, logURL, req.RemoteAddr, err)
+			resp.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(resp, err.Error())
+		}
+
+		// Invoke the handler
+		start := time.Now()
+		defer func() {
+			log.V(5).Info("http: Request %s %v from %s, Timing: %v\n", req.Method, logURL, req.RemoteAddr, time.Since(start))
+		}()
+		obj, err := handler(resp, req)
+		if err != nil {
+			handleErr(err)
+			return
+		}
+		if obj == nil {
+			return
+		}
+
+		buf, err := s.marshalJSON(req, obj)
+		if err != nil {
+			handleErr(err)
+			return
+		}
+		resp.Header().Set("Content-Type", "application/json")
+		resp.Write(buf)
+	}
+}
+
+// marshalJSON marshals the object into JSON, respecting the user's pretty-ness
+// configuration.
+func (s *HTTPServer) marshalJSON(req *http.Request, obj interface{}) ([]byte, error) {
+	if _, ok := req.URL.Query()["pretty"]; ok {
+		buf, err := json.MarshalIndent(obj, "", "    ")
+		if err != nil {
+			return nil, err
+		}
+		buf = append(buf, "\n"...)
+		return buf, nil
+	}
+
+	buf, err := json.Marshal(obj)
+	if err != nil {
+		return nil, err
+	}
+	return buf, err
+}

app/service/main/bns/agent/ns_endpoint.go

@@ -0,0 +1,24 @@
+package agent
+
+import (
+	"net/http"
+	"strings"
+
+	"go-common/app/service/main/bns/agent/backend"
+	"go-common/library/log"
+)
+
+// NSTranslation query name from http api
+func (s *HTTPServer) NSTranslation(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	name := strings.TrimPrefix(req.URL.Path, "/v1/naming/")
+
+	inss, err := s.agent.Query(name)
+	if err != nil {
+		log.Error("call easyns server failed with naming translation, err: %s", err.Error())
+		return nil, err
+	}
+	obj := struct {
+		Instances []*backend.Instance `json:"instances"`
+	}{inss}
+	return obj, nil
+}