Create & Init Project...

library/database/elastic/BUILD

@@ -0,0 +1,53 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "query_test.go",
+        "update_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//library/queue/databus/report:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "query.go",
+        "update.go",
+    ],
+    importpath = "go-common/library/database/elastic",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/ecode:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

library/database/elastic/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+haoguanwei
+renwei
+zhapuyu
+
+# Author
+wangjian
+
+# Reviewer
+zhapuyu
+guanhuaxin

library/database/elastic/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- renwei
+- wangjian
+- zhapuyu
+labels:
+- library
+- library/database/elastic
+reviewers:
+- guanhuaxin
+- wangjian
+- zhapuyu

library/database/elastic/query.go

@@ -0,0 +1,578 @@
+package elastic
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"go-common/library/ecode"
+	httpx "go-common/library/net/http/blademaster"
+	timex "go-common/library/time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	// OrderAsc order ascend
+	OrderAsc = "asc"
+	// OrderDesc order descend
+	OrderDesc = "desc"
+
+	// RangeScopeLoRo left open & right open
+	RangeScopeLoRo rangeScope = "( )"
+	// RangeScopeLoRc left open & right close
+	RangeScopeLoRc rangeScope = "( ]"
+	// RangeScopeLcRo left close & right open
+	RangeScopeLcRo rangeScope = "[ )"
+	// RangeScopeLcRc lect close & right close
+	RangeScopeLcRc rangeScope = "[ ]"
+
+	// NotTypeEq not type eq
+	NotTypeEq notType = "eq"
+	// NotTypeIn not type in
+	NotTypeIn notType = "in"
+	// NotTypeRange not type range
+	NotTypeRange notType = "range"
+
+	// LikeLevelHigh wildcard keyword
+	LikeLevelHigh likeLevel = "high"
+	// LikeLevelMiddle ngram(1,2)
+	LikeLevelMiddle likeLevel = "middle"
+	// LikeLevelLow match split word
+	LikeLevelLow likeLevel = "low"
+
+	// IndexTypeYear index by year
+	IndexTypeYear indexType = "year"
+	// IndexTypeMonth index by month
+	IndexTypeMonth indexType = "month"
+	// IndexTypeWeek index by week
+	IndexTypeWeek indexType = "week"
+	// IndexTypeDay index by day
+	IndexTypeDay indexType = "day"
+
+	// EnhancedModeGroupBy group by mode
+	EnhancedModeGroupBy enhancedType = "group_by"
+	// EnhancedModeDistinct distinct mode
+	EnhancedModeDistinct enhancedType = "distinct"
+	// EnhancedModeSum sum mode
+	EnhancedModeSum enhancedType = "sum"
+)
+
+type (
+	notType      string
+	rangeScope   string
+	likeLevel    string
+	indexType    string
+	enhancedType string
+)
+
+var (
+	_defaultHost = "http://manager.bilibili.co"
+	_pathQuery   = "/x/admin/search/query"
+	_pathUpsert  = "/x/admin/search/upsert"
+
+	_defaultHTTPClient = &httpx.ClientConfig{
+		App: &httpx.App{
+			Key:    "3c4e41f926e51656",
+			Secret: "26a2095b60c24154521d24ae62b885bb",
+		},
+		Dial:    timex.Duration(time.Second),
+		Timeout: timex.Duration(time.Second),
+	}
+
+	// for index by week
+	weeks = map[int]string{0: "0107", 1: "0815", 2: "1623", 3: "2431"}
+)
+
+// Config Elastic config
+type Config struct {
+	Host       string
+	HTTPClient *httpx.ClientConfig
+}
+
+// response query elastic response
+type response struct {
+	Code    int             `json:"code"`
+	Message string          `json:"message"`
+	Data    json.RawMessage `json:"data"`
+}
+
+type query struct {
+	Fields          []string            `json:"fields"`
+	From            string              `json:"from"`
+	OrderScoreFirst bool                `json:"order_score_first"`
+	OrderRandomSeed string              `json:"order_random_seed"`
+	Highlight       bool                `json:"highlight"`
+	Pn              int                 `json:"pn"`
+	Ps              int                 `json:"ps"`
+	Order           []map[string]string `json:"order,omitempty"`
+	Where           where               `json:"where,omitempty"`
+}
+
+func (q *query) string() (string, error) {
+	var (
+		sli []string
+		m   = make(map[string]bool)
+	)
+	for _, i := range strings.Split(q.From, ",") {
+		if m[i] {
+			continue
+		}
+		m[i] = true
+		sli = append(sli, i)
+	}
+	q.From = strings.Join(sli, ",")
+	bs, err := json.Marshal(q)
+	if err != nil {
+		return "", err
+	}
+	return string(bs), nil
+}
+
+type where struct {
+	GroupBy  string                      `json:"group_by,omitempty"`
+	Like     []whereLike                 `json:"like,omitempty"`
+	Eq       map[string]interface{}      `json:"eq,omitempty"`
+	Or       map[string]interface{}      `json:"or,omitempty"`
+	In       map[string][]interface{}    `json:"in,omitempty"`
+	Range    map[string]string           `json:"range,omitempty"`
+	Combo    []*Combo                    `json:"combo,omitempty"`
+	Not      map[notType]map[string]bool `json:"not,omitempty"`
+	Enhanced []interface{}               `json:"enhanced,omitempty"`
+}
+
+type whereLike struct {
+	Fields []string  `json:"kw_fields"`
+	Words  []string  `json:"kw"`
+	Or     bool      `json:"or"`
+	Level  likeLevel `json:"level"`
+}
+
+// Combo mix eq & in & range
+type Combo struct {
+	EQ       []map[string]interface{}   `json:"eq,omitempty"`
+	In       []map[string][]interface{} `json:"in,omitempty"`
+	Range    []map[string]string        `json:"range,omitempty"`
+	NotEQ    []map[string]interface{}   `json:"not_eq,omitempty"`
+	NotIn    []map[string][]interface{} `json:"not_in,omitempty"`
+	NotRange []map[string]string        `json:"not_range,omitempty"`
+	Min      struct {
+		EQ       int `json:"eq,omitempty"`
+		In       int `json:"in,omitempty"`
+		Range    int `json:"range,omitempty"`
+		NotEQ    int `json:"not_eq,omitempty"`
+		NotIn    int `json:"not_in,omitempty"`
+		NotRange int `json:"not_range,omitempty"`
+		Min      int `json:"min"`
+	} `json:"min"`
+}
+
+// ComboEQ .
+func (cmb *Combo) ComboEQ(eq []map[string]interface{}) *Combo {
+	cmb.EQ = append(cmb.EQ, eq...)
+	return cmb
+}
+
+// ComboRange .
+func (cmb *Combo) ComboRange(r []map[string]string) *Combo {
+	cmb.Range = append(cmb.Range, r...)
+	return cmb
+}
+
+// ComboIn .
+func (cmb *Combo) ComboIn(in []map[string][]interface{}) *Combo {
+	cmb.In = append(cmb.In, in...)
+	return cmb
+}
+
+// ComboNotEQ .
+func (cmb *Combo) ComboNotEQ(eq []map[string]interface{}) *Combo {
+	cmb.NotEQ = append(cmb.NotEQ, eq...)
+	return cmb
+}
+
+// ComboNotRange .
+func (cmb *Combo) ComboNotRange(r []map[string]string) *Combo {
+	cmb.NotRange = append(cmb.NotRange, r...)
+	return cmb
+}
+
+// ComboNotIn .
+func (cmb *Combo) ComboNotIn(in []map[string][]interface{}) *Combo {
+	cmb.NotIn = append(cmb.NotIn, in...)
+	return cmb
+}
+
+// MinEQ .
+func (cmb *Combo) MinEQ(min int) *Combo {
+	cmb.Min.EQ = min
+	return cmb
+}
+
+// MinIn .
+func (cmb *Combo) MinIn(min int) *Combo {
+	cmb.Min.In = min
+	return cmb
+}
+
+// MinRange .
+func (cmb *Combo) MinRange(min int) *Combo {
+	cmb.Min.Range = min
+	return cmb
+}
+
+// MinNotEQ .
+func (cmb *Combo) MinNotEQ(min int) *Combo {
+	cmb.Min.NotEQ = min
+	return cmb
+}
+
+// MinNotIn .
+func (cmb *Combo) MinNotIn(min int) *Combo {
+	cmb.Min.NotIn = min
+	return cmb
+}
+
+// MinNotRange .
+func (cmb *Combo) MinNotRange(min int) *Combo {
+	cmb.Min.NotRange = min
+	return cmb
+}
+
+// MinAll .
+func (cmb *Combo) MinAll(min int) *Combo {
+	cmb.Min.Min = min
+	return cmb
+}
+
+type groupBy struct {
+	Mode  enhancedType        `json:"mode"`
+	Field string              `json:"field"`
+	Order []map[string]string `json:"order"`
+}
+
+type enhance struct {
+	Mode  enhancedType        `json:"mode"`
+	Field string              `json:"field"`
+	Order []map[string]string `json:"order,omitempty"`
+	Size  int                 `json:"size,omitempty"`
+}
+
+// Elastic clastic instance
+type Elastic struct {
+	c      *Config
+	client *httpx.Client
+}
+
+// NewElastic .
+func NewElastic(c *Config) *Elastic {
+	if c == nil {
+		c = &Config{
+			Host:       _defaultHost,
+			HTTPClient: _defaultHTTPClient,
+		}
+	}
+	return &Elastic{
+		c:      c,
+		client: httpx.NewClient(c.HTTPClient),
+	}
+}
+
+// Request request to elastic
+type Request struct {
+	*Elastic
+	q        query
+	business string
+}
+
+// NewRequest new a request every search query
+func (e *Elastic) NewRequest(business string) *Request {
+	return &Request{
+		Elastic:  e,
+		business: business,
+		q: query{
+			Fields:          []string{},
+			Highlight:       false,
+			OrderScoreFirst: true,
+			OrderRandomSeed: "",
+			Pn:              1,
+			Ps:              10,
+		},
+	}
+}
+
+// Fields add query fields
+func (r *Request) Fields(fields ...string) *Request {
+	r.q.Fields = append(r.q.Fields, fields...)
+	return r
+}
+
+// Index add query index
+func (r *Request) Index(indexes ...string) *Request {
+	r.q.From = strings.Join(indexes, ",")
+	return r
+}
+
+// IndexByMod index by mod
+func (r *Request) IndexByMod(prefix string, val, mod int64) *Request {
+	tmp := mod - 1
+	var digit int
+	for tmp > 0 {
+		tmp /= 10
+		digit++
+	}
+	format := fmt.Sprintf("%s_%%0%dd", prefix, digit)
+	r.q.From = fmt.Sprintf(format, val%mod)
+	return r
+}
+
+// IndexByTime index by time
+func (r *Request) IndexByTime(prefix string, typ indexType, begin, end time.Time) *Request {
+	var (
+		buf     bytes.Buffer
+		index   string
+		indexes = make(map[string]struct{})
+	)
+	for {
+		year := begin.Format("2006")
+		month := begin.Format("01")
+		switch typ {
+		case IndexTypeYear:
+			index = strings.Join([]string{prefix, year}, "_")
+		case IndexTypeMonth:
+			index = strings.Join([]string{prefix, year, month}, "_")
+		case IndexTypeDay:
+			day := begin.Format("02")
+			index = strings.Join([]string{prefix, year, month, day}, "_")
+		case IndexTypeWeek:
+			index = strings.Join([]string{prefix, year, month, weeks[begin.Day()/8]}, "_")
+		}
+		if begin.After(end) && begin.Day() != end.Day() {
+			break
+		}
+		indexes[index] = struct{}{}
+		begin = begin.AddDate(0, 0, 1)
+	}
+	for i := range indexes {
+		buf.WriteString(i)
+		buf.WriteString(",")
+	}
+	r.q.From = strings.TrimSuffix(buf.String(), ",")
+	return r
+}
+
+// OrderScoreFirst switch for order score first
+func (r *Request) OrderScoreFirst(v bool) *Request {
+	r.q.OrderScoreFirst = v
+	return r
+}
+
+// OrderRandomSeed switch for order random
+func (r *Request) OrderRandomSeed(v string) *Request {
+	r.q.OrderRandomSeed = v
+	return r
+}
+
+// Highlight switch from highlight
+func (r *Request) Highlight(v bool) *Request {
+	r.q.Highlight = v
+	return r
+}
+
+// Pn page number
+func (r *Request) Pn(v int) *Request {
+	r.q.Pn = v
+	return r
+}
+
+// Ps page size
+func (r *Request) Ps(v int) *Request {
+	r.q.Ps = v
+	return r
+}
+
+// Order filed sort
+func (r *Request) Order(field, sort string) *Request {
+	if sort != OrderAsc {
+		sort = OrderDesc
+	}
+	r.q.Order = append(r.q.Order, map[string]string{field: sort})
+	return r
+}
+
+// WhereEq where qual
+func (r *Request) WhereEq(field string, eq interface{}) *Request {
+	if r.q.Where.Eq == nil {
+		r.q.Where.Eq = make(map[string]interface{})
+	}
+	r.q.Where.Eq[field] = eq
+	return r
+}
+
+// WhereOr where or
+func (r *Request) WhereOr(field string, or interface{}) *Request {
+	if r.q.Where.Or == nil {
+		r.q.Where.Or = make(map[string]interface{})
+	}
+	r.q.Where.Or[field] = or
+	return r
+}
+
+// WhereIn where in
+func (r *Request) WhereIn(field string, in interface{}) *Request {
+	if r.q.Where.In == nil {
+		r.q.Where.In = make(map[string][]interface{})
+	}
+	switch v := in.(type) {
+	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, string:
+		r.q.Where.In[field] = append(r.q.Where.In[field], v)
+	case []int:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []int64:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []string:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []int8:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []int16:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []int32:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []uint:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []uint8:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []uint16:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []uint32:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	case []uint64:
+		for _, i := range v {
+			r.q.Where.In[field] = append(r.q.Where.In[field], i)
+		}
+	}
+	return r
+}
+
+// WhereRange where range
+func (r *Request) WhereRange(field string, start, end interface{}, scope rangeScope) *Request {
+	if r.q.Where.Range == nil {
+		r.q.Where.Range = make(map[string]string)
+	}
+	if start == nil {
+		start = ""
+	}
+	if end == nil {
+		end = ""
+	}
+	switch scope {
+	case RangeScopeLoRo:
+		r.q.Where.Range[field] = fmt.Sprintf("(%v,%v)", start, end)
+	case RangeScopeLoRc:
+		r.q.Where.Range[field] = fmt.Sprintf("(%v,%v]", start, end)
+	case RangeScopeLcRo:
+		r.q.Where.Range[field] = fmt.Sprintf("[%v,%v)", start, end)
+	case RangeScopeLcRc:
+		r.q.Where.Range[field] = fmt.Sprintf("[%v,%v]", start, end)
+	}
+	return r
+}
+
+// WhereNot where not
+func (r *Request) WhereNot(typ notType, fields ...string) *Request {
+	if r.q.Where.Not == nil {
+		r.q.Where.Not = make(map[notType]map[string]bool)
+	}
+	if r.q.Where.Not[typ] == nil {
+		r.q.Where.Not[typ] = make(map[string]bool)
+	}
+	for _, v := range fields {
+		r.q.Where.Not[typ][v] = true
+	}
+	return r
+}
+
+// WhereLike where like
+func (r *Request) WhereLike(fields, words []string, or bool, level likeLevel) *Request {
+	if len(fields) == 0 || len(words) == 0 {
+		return r
+	}
+	l := whereLike{Fields: fields, Words: words, Or: or, Level: level}
+	r.q.Where.Like = append(r.q.Where.Like, l)
+	return r
+}
+
+// WhereCombo where combo
+func (r *Request) WhereCombo(cmb ...*Combo) *Request {
+	r.q.Where.Combo = append(r.q.Where.Combo, cmb...)
+	return r
+}
+
+// GroupBy where group by
+func (r *Request) GroupBy(mode enhancedType, field string, order []map[string]string) *Request {
+	for _, i := range order {
+		for k, v := range i {
+			if v != OrderAsc {
+				i[k] = OrderDesc
+			}
+		}
+	}
+	r.q.Where.Enhanced = append(r.q.Where.Enhanced, groupBy{Mode: mode, Field: field, Order: order})
+	return r
+}
+
+// Sum where enhance sum
+func (r *Request) Sum(field string) *Request {
+	r.q.Where.Enhanced = append(r.q.Where.Enhanced, enhance{Mode: EnhancedModeSum, Field: field})
+	return r
+}
+
+// Scan parse the query response data
+func (r *Request) Scan(ctx context.Context, result interface{}) (err error) {
+	q, err := r.q.string()
+	if err != nil {
+		return
+	}
+	params := url.Values{}
+	params.Add("business", r.business)
+	params.Add("query", q)
+	response := new(response)
+	if err = r.client.Get(ctx, r.c.Host+_pathQuery, "", params, &response); err != nil {
+		return
+	}
+	if !ecode.Int(response.Code).Equal(ecode.OK) {
+		return ecode.Int(response.Code)
+	}
+	err = errors.Wrapf(json.Unmarshal(response.Data, &result), "scan(%s)", response.Data)
+	return
+}
+
+// Params get query parameters
+func (r *Request) Params() string {
+	q, _ := r.q.string()
+	return fmt.Sprintf("business=%s&query=%s", r.business, q)
+}

library/database/elastic/query_test.go

@@ -0,0 +1,222 @@
+package elastic
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/library/queue/databus/report"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_UserActionLog(t *testing.T) {
+	Convey("elastic user action log", t, func() {
+		type page struct {
+			Num   int `json:"num"`
+			Size  int `json:"size"`
+			Total int `json:"total"`
+		}
+		var res struct {
+			Page   *page                   `json:"page"`
+			Result []*report.UserActionLog `json:"result"`
+		}
+		r := NewElastic(nil).NewRequest("log_user_action").Index("log_user_action_21_2018_06_1623")
+		err := r.Scan(context.TODO(), &res)
+		So(err, ShouldBeNil)
+		t.Logf("query page(%+v)", res.Page)
+		t.Logf("query result(%v)", len(res.Result))
+	})
+}
+
+func Test_Scan(t *testing.T) {
+	Convey("scan", t, func() {
+		type page struct {
+			Num   int `json:"num"`
+			Size  int `json:"size"`
+			Total int `json:"total"`
+		}
+		type ticketData struct {
+			ID       int    `json:"id"`
+			Name     string `json:"name"`
+			City     int    `json:"city"`
+			Province int    `json:"province"`
+		}
+		var res struct {
+			Page   *page         `json:"page"`
+			Result []*ticketData `json:"result"`
+		}
+		r := NewElastic(nil).NewRequest("ticket_venue").Fields("id", "city").Index("ticket_venue").WhereEq("city", 310100).Order("ctime", OrderDesc).Order("id", OrderAsc).
+			WhereOr("province", 310000).WhereRange("id", 1, 2000000, RangeScopeLcRc).WhereLike([]string{"name"}, []string{"梅赛德斯奔驰文化中心"}, true, LikeLevelHigh)
+		err := r.Scan(context.TODO(), &res)
+		So(err, ShouldBeNil)
+		t.Logf("query page(%+v)", res.Page)
+		t.Logf("query result(%v)", len(res.Result))
+	})
+}
+
+func Test_Index(t *testing.T) {
+	Convey("example index by mod", t, func() {
+		oid := int64(8888888)
+		r := NewElastic(nil).NewRequest("LOG_USER_COIN")
+		r.IndexByMod("log_user_action", oid, 100)
+		q, _ := r.q.string()
+		So(q, ShouldContainSubstring, `"from":"log_user_action_88"`)
+	})
+	Convey("example index by time", t, func() {
+		Convey("indexByTime by week", func() {
+			// 按周划分索引的方式
+			r := NewElastic(nil).NewRequest("LOG_USER_COIN")
+			r.IndexByTime("log_user_action", IndexTypeWeek, time.Date(2018, 8, 25, 2, 0, 0, 0, time.Local), time.Date(2018, 9, 24, 2, 0, 0, 0, time.Local))
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_08_2431")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_0107")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_0815")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_1623")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_2431")
+		})
+		Convey("indexByTime by year", func() {
+			// 按年划分索引的方式
+			r := NewElastic(nil).NewRequest("LOG_USER_COIN")
+			r.IndexByTime("log_user_action", IndexTypeYear, time.Date(2017, 8, 25, 2, 0, 0, 0, time.Local), time.Date(2018, 9, 24, 2, 0, 0, 0, time.Local))
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2017")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018")
+		})
+		Convey("indexByTime by month", func() {
+			// 按月划分索引的方式
+			r := NewElastic(nil).NewRequest("LOG_USER_COIN")
+			r.IndexByTime("log_user_action", IndexTypeMonth, time.Date(2018, 8, 25, 2, 0, 0, 0, time.Local), time.Date(2018, 9, 24, 2, 0, 0, 0, time.Local))
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_08")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09")
+		})
+		Convey("indexByTime by day", func() {
+			// 按天划分索引的方式
+			r := NewElastic(nil).NewRequest("LOG_USER_COIN")
+			r.IndexByTime("log_user_action", IndexTypeDay, time.Date(2018, 9, 23, 4, 0, 0, 0, time.Local), time.Date(2018, 9, 24, 2, 0, 0, 0, time.Local))
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_23")
+			So(r.q.From, ShouldContainSubstring, "log_user_action_2018_09_24")
+		})
+	})
+}
+
+func Test_Combo_Simple(t *testing.T) {
+	Convey("combo", t, func() {
+		e := NewElastic(nil)
+		// 实现: (tid in (2,3,4,5,9,20,21,22)) && (tid in (35,38)) && (tid in (15,17,18))
+		cmbA := &Combo{}
+		cmbA.ComboIn([]map[string][]interface{}{
+			{"tid": {2, 3, 4, 5, 9, 20, 21, 22}},
+		}).MinIn(1).MinAll(1)
+
+		cmbB := &Combo{}
+		cmbB.ComboIn([]map[string][]interface{}{
+			{"tid": {35, 38}},
+		}).MinIn(1).MinAll(1)
+
+		cmbC := &Combo{}
+		cmbC.ComboIn([]map[string][]interface{}{
+			{"tid": {15, 17, 18}},
+		}).MinIn(1).MinAll(1)
+
+		r := e.NewRequest("").WhereCombo(cmbA, cmbB, cmbC)
+		q, _ := r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"combo":[{"eq":null,"in":[{"tid":[2,3,4,5,9,20,21,22]}],"range":null,"min":{"eq":0,"in":1,"range":0,"min":1}},{"eq":null,"in":[{"tid":[35,38]}],"range":null,"min":{"eq":0,"in":1,"range":0,"min":1}},{"eq":null,"in":[{"tid":[15,17,18]}],"range":null,"min":{"eq":0,"in":1,"range":0,"min":1}}]}`)
+	})
+}
+
+func Test_Combo_Complicated(t *testing.T) {
+	Convey("combo not", t, func() {
+		e := NewElastic(nil)
+		cmb := &Combo{}
+		cmb.ComboIn([]map[string][]interface{}{
+			{"tid": {1, 2}},
+			{"tid_type": {2, 3}},
+		}).ComboRange([]map[string]string{
+			{"id": "(10,20)"},
+		}).ComboNotEQ([]map[string]interface{}{
+			{"aid": 122},
+			{"id": 677},
+		}).MinIn(1).MinRange(1).MinNotEQ(1).MinAll(1)
+		r := e.NewRequest("").WhereCombo(cmb)
+		q, _ := r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"combo":[{"in":[{"tid":[1,2]},{"tid_type":[2,3]}],"range":[{"id":"(10,20)"}],"not_eq":[{"aid":122},{"id":677}],"min":{"in":1,"range":1,"not_eq":1,"min":1}}]}}`)
+	})
+	Convey("combo", t, func() {
+		e := NewElastic(nil)
+		// 实现:
+		// (aid=122 or id=677) && (tid in (1,2,3,21) or tid_type in (1,2,3)) && (id > 10) &&
+		// (aid=88 or fid=99) && (mid in (11,33) or id in (22,33)) && (2 < cid <= 10  || sid > 10)
+		cmbA := &Combo{}
+		cmbA.ComboEQ([]map[string]interface{}{
+			{"aid": 122},
+			{"id": 677},
+		}).ComboIn([]map[string][]interface{}{
+			{"tid": {1, 2, 3, 21}},
+			{"tid_type": {1, 2, 3}},
+		}).ComboRange([]map[string]string{
+			{"id": "(10,)"},
+		}).MinIn(1).MinEQ(1).MinRange(1).MinAll(1)
+
+		cmbB := &Combo{}
+		cmbB.ComboEQ([]map[string]interface{}{
+			{"aid": 88},
+			{"fid": 99},
+		}).ComboIn([]map[string][]interface{}{
+			{"mid": {11, 33}},
+			{"id": {22, 33}},
+		}).ComboRange([]map[string]string{
+			{"cid": "(2,4]"},
+			{"sid": "(10,)"},
+		}).MinEQ(1).MinIn(2).MinRange(1).MinAll(1)
+
+		r := e.NewRequest("").WhereCombo(cmbA, cmbB)
+		q, _ := r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"combo":[{"eq":[{"aid":122},{"id":677}],"in":[{"tid":[1,2,3,21]},{"tid_type":[1,2,3]}],"range":[{"id":"(10,)"}],"min":{"eq":1,"in":1,"range":1,"min":1}},{"eq":[{"aid":88},{"fid":99}],"in":[{"mid":[11,33]},{"id":[22,33]}],"range":[{"cid":"(2,4]"},{"sid":"(10,)"}],"min":{"eq":1,"in":2,"range":1,"min":1}}]}`)
+	})
+}
+
+func Test_Query(t *testing.T) {
+	Convey("query conditions", t, func() {
+		e := NewElastic(nil)
+
+		r := e.NewRequest("").WhereEq("city", 310100)
+		q, _ := r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"eq":{"city":310100}}`)
+
+		r = e.NewRequest("").WhereLike([]string{"a", "b"}, []string{"c", "d"}, true, LikeLevelHigh).WhereLike([]string{"e", "f"}, []string{"g", "h"}, false, LikeLevelMiddle)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"like":[{"kw_fields":["a","b"],"kw":["c","d"],"or":true,"level":"high"},{"kw_fields":["e","f"],"kw":["g","h"],"or":false,"level":"middle"}]}`)
+
+		r = e.NewRequest("").WhereNot(NotTypeEq, "province")
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"not":{"eq":{"province":true}}}`)
+
+		r = e.NewRequest("").WhereRange("id", 100, 500, RangeScopeLcRo).WhereRange("date", "2018-08-08 08:08:08", "2019-09-09 09:09:09", RangeScopeLoRo)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"range":{"date":"(2018-08-08 08:08:08,2019-09-09 09:09:09)","id":"[100,500)"}}`)
+
+		r = e.NewRequest("").WhereOr("city", 100000)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"or":{"city":100000}}`)
+
+		ids := []int64{100, 200, 300}
+		r = e.NewRequest("").WhereIn("city", ids)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"in":{"city":[100,200,300]}}`)
+
+		strs := []string{"a"}
+		r = e.NewRequest("").WhereIn("name", strs)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"in":{"name":["a"]}}`)
+
+		field := "a"
+		order := []map[string]string{{"a": "asc"}}
+		r = e.NewRequest("").GroupBy(EnhancedModeGroupBy, field, order)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"enhanced":[{"mode":"group_by","field":"a","order":[{"a":"asc"}]}]}`)
+
+		field = "a"
+		r = e.NewRequest("").Sum(field)
+		q, _ = r.q.string()
+		So(q, ShouldContainSubstring, `"where":{"enhanced":[{"mode":"sum","field":"a"}]}`)
+	})
+}

library/database/elastic/update.go

@@ -0,0 +1,108 @@
+package elastic
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"go-common/library/ecode"
+)
+
+// Update elastic upsert
+type Update struct {
+	*Elastic
+	business string
+	data     map[string][]interface{}
+	insert   bool
+}
+
+// NewUpdate new a request every update
+func (e *Elastic) NewUpdate(business string) *Update {
+	return &Update{
+		Elastic:  e,
+		business: business,
+		data:     make(map[string][]interface{}),
+	}
+}
+
+// IndexByMod index by mod
+func (us *Update) IndexByMod(prefix string, val, mod int64) string {
+	tmp := mod - 1
+	var digit int
+	for tmp > 0 {
+		tmp /= 10
+		digit++
+	}
+	format := fmt.Sprintf("%s_%%0%dd", prefix, digit)
+	return fmt.Sprintf(format, val%mod)
+}
+
+// IndexByTime index by time
+func (us *Update) IndexByTime(prefix string, typ indexType, t time.Time) (index string) {
+	year := t.Format("2006")
+	month := t.Format("01")
+	switch typ {
+	case IndexTypeYear:
+		index = strings.Join([]string{prefix, year}, "_")
+	case IndexTypeMonth:
+		index = strings.Join([]string{prefix, year, month}, "_")
+	case IndexTypeDay:
+		day := t.Format("02")
+		index = strings.Join([]string{prefix, year, month, day}, "_")
+	case IndexTypeWeek:
+		index = strings.Join([]string{prefix, year, month, weeks[t.Day()/8]}, "_")
+	}
+	return
+}
+
+// AddData add data items to request 'data' param
+func (us *Update) AddData(index string, data interface{}) *Update {
+	if data == nil {
+		return us
+	}
+	us.data[index] = append(us.data[index], data)
+	return us
+}
+
+// HasData weather data is empty or not
+func (us *Update) HasData() bool {
+	if us.data == nil {
+		return false
+	}
+	return len(us.data) > 0
+}
+
+// Insert set insert flag, it means 'replace'
+func (us *Update) Insert() *Update {
+	us.insert = true
+	return us
+}
+
+// Do post a request
+func (us *Update) Do(ctx context.Context) (err error) {
+	data, err := json.Marshal(us.data)
+	if err != nil {
+		return
+	}
+	params := url.Values{}
+	params.Add("business", us.business)
+	params.Add("data", string(data))
+	params.Add("insert", fmt.Sprintf("%t", us.insert))
+	response := new(response)
+	if err = us.client.Post(ctx, us.c.Host+_pathUpsert, "", params, &response); err != nil {
+		return
+	}
+	if !ecode.Int(response.Code).Equal(ecode.OK) {
+		err = ecode.Int(response.Code)
+	}
+	return
+}
+
+// Params get query parameters
+func (us *Update) Params() string {
+	data, _ := json.Marshal(us.data)
+	return fmt.Sprintf("business=%s&insert=%t&data=%s", us.business, us.insert, data)
+}

library/database/elastic/update_test.go

@@ -0,0 +1,119 @@
+package elastic
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Upsert(t *testing.T) {
+	Convey("upsert", t, func() {
+		e := NewElastic(nil)
+		us := e.NewUpdate("articledata").Insert()
+		data := map[string]int{"id": 552, "share": 321}
+		us.AddData("articledata", data)
+		data = map[string]int{"id": 558, "share": 1137}
+		us.AddData("articledata", data)
+		t.Logf("params(%s)", us.Params())
+		err := us.Do(context.Background())
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_Upsert_ByMod(t *testing.T) {
+	Convey("upsert", t, func() {
+		data := make([]map[string]interface{}, 0)
+		d := map[string]interface{}{"id": 22, "share": 22, "oid": int64(2266)}
+		data = append(data, d)
+		d = map[string]interface{}{"id": 33, "share": 33, "oid": int64(3388)}
+		data = append(data, d)
+
+		e := NewElastic(nil)
+		us := e.NewUpdate("reply").Insert()
+		for _, v := range data {
+			oid, ok := v["oid"]
+			if !ok {
+				continue
+			}
+			oidReal, ok := oid.(int64) //must as same as interface type
+			if !ok {
+				continue
+			}
+			us.AddData(us.IndexByMod("reply_record", oidReal, 100), v)
+		}
+		t.Logf("params(%s)", us.Params())
+		if us.HasData() {
+			err := us.Do(context.Background())
+			So(err, ShouldBeNil)
+		} else {
+			So(us.HasData(), ShouldBeTrue)
+		}
+	})
+}
+
+func Test_Update_Index(t *testing.T) {
+	Convey("test update index", t, func() {
+		e := NewElastic(nil)
+		us := e.NewUpdate("reply").Insert()
+		Convey("example by mod 100", func() {
+			oid := int64(808)
+			index := us.IndexByMod("reply", oid, 100)
+			So(index, ShouldEqual, "reply_08")
+		})
+		Convey("example by mod 1000", func() {
+			oid := int64(808)
+			index := us.IndexByMod("reply", oid, 1000)
+			So(index, ShouldEqual, "reply_808")
+		})
+		Convey("example by mod 10000", func() {
+			oid := int64(808)
+			index := us.IndexByMod("reply", oid, 10000)
+			So(index, ShouldEqual, "reply_0808")
+		})
+		Convey("example by mod oid 0", func() {
+			oid := int64(0)
+			index := us.IndexByMod("reply", oid, 100)
+			So(index, ShouldEqual, "reply_00")
+		})
+		Convey("example by mod oid 20", func() {
+			oid := int64(808)
+			index := us.IndexByMod("reply", oid, 20)
+			So(index, ShouldEqual, "reply_08")
+		})
+	})
+}
+
+func Test_Upsert_ByTime(t *testing.T) {
+	Convey("upsert", t, func() {
+		data := make([]map[string]interface{}, 0)
+		d := map[string]interface{}{"id": 22, "share": 22, "ctime": time.Now().AddDate(-2, 0, 0)}
+		data = append(data, d)
+		d = map[string]interface{}{"id": 33, "share": 33, "ctime": time.Now().AddDate(-3, 0, 0)}
+		data = append(data, d)
+
+		e := NewElastic(nil)
+		us := e.NewUpdate("reply_list").Insert()
+		for _, v := range data {
+			ctime, ok := v["ctime"]
+			if !ok {
+				continue
+			}
+			ctimeReal, ok := ctime.(time.Time) //must as same as interface type
+			if !ok {
+				continue
+			}
+			indexName := us.IndexByTime("reply_list_hot", IndexTypeYear, ctimeReal)
+			v["ctime"] = ctimeReal.Format("2006-01-02 15:04:05")
+			us.AddData(indexName, v)
+		}
+		t.Logf("params(%s)", us.Params())
+		if us.HasData() {
+			err := us.Do(context.Background())
+			So(err, ShouldBeNil)
+		} else {
+			So(us.HasData(), ShouldBeTrue)
+		}
+	})
+}