Create & Init Project...

app/service/main/dapper-query/pkg/cltclient/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["clt.go"],
+    importpath = "go-common/app/service/main/dapper-query/pkg/cltclient",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/sync/errgroup:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/dapper-query/pkg/cltclient/clt.go

@@ -0,0 +1,113 @@
+// Package cltclient provide fetch and merge data from collector
+package cltclient
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+
+	"go-common/library/sync/errgroup"
+)
+
+const (
+	_jsonMime = "application/json"
+)
+
+// ClientStatusResp response clientstatus request just for debug
+type ClientStatusResp struct {
+	QueueLen int             `json:"queue_len"`
+	Clients  []*ClientStatus `json:"clients"`
+}
+
+// ClientStatus client status
+type ClientStatus struct {
+	Addr     string `json:"addr"`
+	UpTime   int64  `json:"up_time"`
+	ErrCount int64  `json:"err_count"`
+	Rate     int64  `json:"rate"`
+}
+
+// CltStatus collector status
+type CltStatus struct {
+	Node     string          `json:"node"`
+	QueueLen int             `json:"queue_len"`
+	Clients  []*ClientStatus `json:"clients"`
+}
+
+// New collector client
+func New(nodes []string, httpclient *http.Client) (*Client, error) {
+	if len(nodes) == 0 {
+		return nil, fmt.Errorf("no node provided")
+	}
+	if httpclient == nil {
+		httpclient = http.DefaultClient
+	}
+	return &Client{nodes: nodes, httpclient: httpclient}, nil
+}
+
+// Client collector client
+type Client struct {
+	nodes      []string
+	httpclient *http.Client
+}
+
+// Status return all collector status
+func (c *Client) Status(ctx context.Context) ([]*CltStatus, error) {
+	var mx sync.Mutex
+	var g errgroup.Group
+	results := make(map[string]*ClientStatusResp)
+	for _, node := range c.nodes {
+		node := node // https://golang.org/doc/faq#closures_and_goroutines
+		g.Go(func() error {
+			resp, err := c.fetchStatus(ctx, node)
+			if err != nil {
+				return err
+			}
+			mx.Lock()
+			results[node] = resp
+			mx.Unlock()
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	clts := make([]*CltStatus, 0, len(results))
+	for node, resp := range results {
+		clts = append(clts, &CltStatus{
+			Node:     node,
+			QueueLen: resp.QueueLen,
+			Clients:  resp.Clients,
+		})
+	}
+	return clts, nil
+}
+
+func (c *Client) fetchStatus(ctx context.Context, node string) (*ClientStatusResp, error) {
+	var wrapResp struct {
+		Code    int              `json:"code"`
+		Message string           `json:"message"`
+		Data    ClientStatusResp `json:"data"`
+	}
+	reqURL := "http://" + node + "/x/internal/dapper-collector/client-status"
+	req, err := http.NewRequest(http.MethodGet, reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", _jsonMime)
+	req = req.WithContext(ctx)
+	resp, err := c.httpclient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		p := make([]byte, 2048)
+		n, _ := resp.Body.Read(p)
+		return nil, fmt.Errorf("request url: %s status code: %d, body: %s", reqURL, resp.StatusCode, p[:n])
+	}
+	err = json.NewDecoder(resp.Body).Decode(&wrapResp)
+	return &wrapResp.Data, err
+}

app/service/main/dapper-query/pkg/opslog/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["opslog_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["opslog.go"],
+    importpath = "go-common/app/service/main/dapper-query/pkg/opslog",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/dapper-query/pkg/opslog/opslog.go

@@ -0,0 +1,221 @@
+// Package opslog provide ops-log api
+package opslog
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_kbnVersion        = "5.4.3"
+	_indexPrefix       = "billions-"
+	_mqueryContentType = "application/x-ndjson"
+	_ajsSessioID       = "_AJSESSIONID"
+)
+
+// Err errors
+var (
+	ErrOverRange = errors.New("search time over range")
+)
+
+type response struct {
+	Hits struct {
+		Hits []struct {
+			Source map[string]interface{} `json:"_source"`
+		} `json:"hits"`
+	} `json:"hits"`
+}
+
+// Record represent log record
+type Record struct {
+	Time    time.Time              `json:"timestamp"`
+	Fields  map[string]interface{} `json:"fields"`
+	Level   string                 `json:"level"`
+	Message string                 `json:"message"`
+}
+
+// Client query log from ops-log
+type Client interface {
+	Query(ctx context.Context, familys []string, traceID uint64, sessionID string, start, end int64, options ...Option) ([]*Record, error)
+}
+
+type option struct {
+	traceField string
+	size       int
+	level      string
+}
+
+var _defaultOpt = option{
+	traceField: "traceid",
+	size:       100,
+}
+
+// Option for query
+type Option func(opt *option)
+
+// SetTraceField default "traceid"
+func SetTraceField(traceField string) Option {
+	return func(opt *option) {
+		opt.traceField = traceField
+	}
+}
+
+// SetSize default 100
+func SetSize(size int) Option {
+	return func(opt *option) {
+		opt.size = size
+	}
+}
+
+// SetLevel return all if level is empty
+func SetLevel(level string) Option {
+	return func(opt *option) {
+		opt.level = level
+	}
+}
+
+// New ops-log client
+func New(searchAPI string, httpClient *http.Client) Client {
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+	return &client{
+		searchAPI:  searchAPI,
+		httpclient: httpClient,
+	}
+}
+
+type client struct {
+	searchAPI  string
+	httpclient *http.Client
+}
+
+func (c *client) Query(ctx context.Context, familys []string, traceID uint64, sessionID string, start, end int64, options ...Option) ([]*Record, error) {
+	if start <= 0 || end <= 0 {
+		return nil, ErrOverRange
+	}
+	if len(familys) == 0 {
+		return make([]*Record, 0), nil
+	}
+	opt := _defaultOpt
+	for _, fn := range options {
+		fn(&opt)
+	}
+	req, err := c.newReq(familys, traceID, sessionID, start, end, &opt)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.httpclient.Do(req)
+	if err != nil {
+		return nil, errors.Wrapf(err, "send request to %s fail", c.searchAPI)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		buf := make([]byte, 1024)
+		n, _ := resp.Body.Read(buf)
+		return nil, errors.Errorf("ops-log response error: status_code: %d, body: %s", resp.StatusCode, buf[:n])
+	}
+	return decodeRecord(resp.Body)
+}
+
+func (c *client) newReq(familys []string, traceID uint64, sessionID string, start, end int64, opt *option) (*http.Request, error) {
+	prefixTraceID := strconv.FormatUint(traceID, 16)
+	leagcyTraceID := strconv.FormatUint(traceID, 10)
+	startMillis := start * int64((time.Second / time.Millisecond))
+	endMillis := end * int64((time.Second / time.Millisecond))
+	body := &bytes.Buffer{}
+	enc := json.NewEncoder(body)
+	header := map[string]interface{}{"index": formatIndices(familys), "ignore_unavailable": true}
+	if err := enc.Encode(header); err != nil {
+		return nil, err
+	}
+	shoulds := []map[string]interface{}{
+		{"prefix": map[string]interface{}{opt.traceField: prefixTraceID}},
+		{"match": map[string]interface{}{opt.traceField: leagcyTraceID}},
+	}
+	traceQuery := map[string]interface{}{"bool": map[string]interface{}{"should": shoulds}}
+	rangeQuery := map[string]interface{}{
+		"range": map[string]interface{}{
+			"@timestamp": map[string]interface{}{"gte": startMillis, "lte": endMillis, "format": "epoch_millis"},
+		},
+	}
+	musts := []map[string]interface{}{traceQuery, rangeQuery}
+	if opt.level != "" {
+		musts = append(musts, map[string]interface{}{"match": map[string]interface{}{"level": opt.level}})
+	}
+	query := map[string]interface{}{
+		"sort": map[string]interface{}{
+			"@timestamp": map[string]interface{}{
+				"order":         "desc",
+				"unmapped_type": "boolean",
+			},
+		},
+		"query": map[string]interface{}{
+			"bool": map[string]interface{}{"must": musts},
+		},
+		"version": true,
+		"size":    opt.size,
+	}
+	if err := enc.Encode(query); err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest(http.MethodPost, c.searchAPI, body)
+	if err != nil {
+		return nil, err
+	}
+	session := &http.Cookie{Name: _ajsSessioID, Value: sessionID}
+	req.AddCookie(session)
+	req.Header.Set("Content-Type", _mqueryContentType)
+	req.Header.Set("kbn-version", _kbnVersion)
+	return req, nil
+}
+
+func decodeRecord(src io.Reader) ([]*Record, error) {
+	var resp struct {
+		Responses []response `json:"responses"`
+	}
+	if err := json.NewDecoder(src).Decode(&resp); err != nil {
+		return nil, errors.Wrap(err, "decode response error")
+	}
+	if len(resp.Responses) == 0 {
+		return nil, nil
+	}
+	records := make([]*Record, 0, len(resp.Responses[0].Hits.Hits))
+	for _, hit := range resp.Responses[0].Hits.Hits {
+		record := &Record{
+			Fields: make(map[string]interface{}),
+		}
+		for k, v := range hit.Source {
+			switch k {
+			case "@timestamp":
+				s, _ := v.(string)
+				record.Time, _ = time.Parse(time.RFC3339Nano, s)
+			case "log":
+				s, _ := v.(string)
+				record.Message = s
+			case "level":
+				s, _ := v.(string)
+				record.Level = s
+			default:
+				record.Fields[k] = v
+			}
+		}
+		records = append(records, record)
+	}
+	return records, nil
+}
+
+func formatIndices(familys []string) []string {
+	indices := make([]string, len(familys))
+	for i := range familys {
+		indices[i] = _indexPrefix + familys[i] + "*"
+	}
+	return indices
+}

app/service/main/dapper-query/pkg/opslog/opslog_test.go

@@ -0,0 +1,68 @@
+package opslog
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestOpsLog(t *testing.T) {
+	testSessionID := "c860e25e5360fc08888a3aaf8c7a0bec"
+	testResponse := `{"responses":[{"took":4,"timed_out":false,"_shards":{"total":8,"successful":8,"failed":0},"hits":{"total":2,"max_score":null,"hits":[{"_index":"billions-main.web-svr.web-interface-@2018.09.10-uat-1","_type":"logs","_id":"AWXBhTtRe-NhC44S955A","_version":1,"_score":null,"_source":{"@timestamp":"2018-09-10T03:27:34.42933Z","app_id":"main.web-svr.web-interface","args":"","env":"uat","error":"","instance_id":"web-interface-32096-758958f64f-k4gnc","ip":"172.22.35.133:9000","level":"INFO","level_value":1,"path":"/passport.service.identify.v1.Identify/GetCookieInfo","ret":0,"source":"go-common/library/net/rpc/warden.logging:195","stack":"\u003cnil\u003e","traceid":"2406767965117552819","ts":0.001041696,"user":"","zone":"sh001"},"fields":{"@timestamp":[1536550054429]},"highlight":{"traceid":["@kibana-highlighted-field@2406767965117552819@/kibana-highlighted-field@"]},"sort":[1536550054429]},{"_index":"billions-main.web-svr.web-interface-@2018.09.10-uat-1","_type":"logs","_id":"AWXBhTfFS1y0J6vacgAH","_version":1,"_score":null,"_source":{"@timestamp":"2018-09-10T03:27:34.429376Z","app_id":"main.web-svr.web-interface","env":"uat","err":"-101","instance_id":"web-interface-32096-758958f64f-k4gnc","ip":"10.23.50.21","level":"ERROR","level_value":3,"method":"GET","mid":null,"msg":"账号未登录","params":"","path":"/x/web-interface/nav","ret":-101,"source":"go-common/library/net/http/blademaster.Logger.func1:46","stack":"-101","traceid":"2406767965117552819","ts":0.001167299,"user":"no_user","zone":"sh001"},"fields":{"@timestamp":[1536550054429]},"highlight":{"traceid":["@kibana-highlighted-field@2406767965117552819@/kibana-highlighted-field@"]},"sort":[1536550054429]}]},"aggregations":{"2":{"buckets":[{"key_as_string":"2018-09-10T09:00:00.000+08:00","key":1536541200000,"doc_count":2}]}},"status":200}]}`
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		session, err := r.Cookie(_ajsSessioID)
+		if err != nil || session.Value != testSessionID {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, "invalid session id: %s", session.Value)
+			return
+		}
+		bufReader := bufio.NewReader(r.Body)
+		first, err := bufReader.ReadString('\n')
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		if !strings.Contains(first, "billions-main.web-svr.web-interface*") {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, "invalid familys: %s", first)
+			return
+		}
+		w.Write([]byte(testResponse))
+	}))
+	defer svr.Close()
+	client := New(svr.URL, nil)
+	end := time.Now().Unix()
+	start := end - 3600
+	familys := []string{"main.web-svr.web-interface"}
+	records, err := client.Query(context.Background(), familys, 8111326167741382285, testSessionID, start, end)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, record := range records {
+		t.Logf("record: %v", record)
+	}
+}
+
+func TestOpsLogReal(t *testing.T) {
+	sessionID := os.Getenv("TEST_SESSION_ID")
+	if sessionID == "" {
+		t.Skipf("miss sessionID skip test")
+	}
+
+	traceID, _ := strconv.ParseUint("7b91b9a72f87c13", 16, 64)
+	client := New("http://uat-ops-log.bilibili.co/elasticsearch/_msearch", nil)
+	records, err := client.Query(context.Background(), []string{"main.community.tag"}, traceID, sessionID, 1545296000, 1545296286)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, record := range records {
+		t.Logf("record: %v", record)
+	}
+}