506 lines
15 KiB
Go
506 lines
15 KiB
Go
|
package dao
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bytes"
|
|||
|
|
"context"
|
|||
|
|
"encoding/json"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"strings"
|
|||
|
|
|
|||
|
|
"go-common/app/admin/main/search/model"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
|
|||
|
|
"gopkg.in/olivere/elastic.v5"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// UpdateMapBulk (Deprecated).
|
|||
|
|
func (d *Dao) UpdateMapBulk(c context.Context, esName string, bulkData []BulkMapItem) (err error) {
|
|||
|
|
bulkRequest := d.esPool[esName].Bulk()
|
|||
|
|
for _, b := range bulkData {
|
|||
|
|
request := elastic.NewBulkUpdateRequest().Index(b.IndexName()).Type(b.IndexType()).Id(b.IndexID()).Doc(b.PField()).DocAsUpsert(true)
|
|||
|
|
bulkRequest.Add(request)
|
|||
|
|
}
|
|||
|
|
if _, err = bulkRequest.Do(c); err != nil {
|
|||
|
|
log.Error("esName(%s) bulk error(%v)", esName, err)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// UpdateBulk (Deprecated).
|
|||
|
|
func (d *Dao) UpdateBulk(c context.Context, esName string, bulkData []BulkItem) (err error) {
|
|||
|
|
bulkRequest := d.esPool[esName].Bulk()
|
|||
|
|
for _, b := range bulkData {
|
|||
|
|
request := elastic.NewBulkUpdateRequest().Index(b.IndexName()).Type(b.IndexType()).Id(b.IndexID()).Doc(b).DocAsUpsert(true)
|
|||
|
|
bulkRequest.Add(request)
|
|||
|
|
}
|
|||
|
|
if _, err = bulkRequest.Do(c); err != nil {
|
|||
|
|
log.Error("esName(%s) bulk error(%v)", esName, err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// UpsertBulk 为了替换UpdateMapBulk和UpdateBulk .
|
|||
|
|
func (d *Dao) UpsertBulk(c context.Context, esCluster string, up *model.UpsertParams) (err error) {
|
|||
|
|
es, ok := d.esPool[esCluster]
|
|||
|
|
if !ok {
|
|||
|
|
log.Error("esCluster(%s) not exists", esCluster)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
bulkRequest := es.Bulk()
|
|||
|
|
for _, b := range up.UpsertBody {
|
|||
|
|
request := elastic.NewBulkUpdateRequest().Index(b.IndexName).Type(b.IndexType).Id(b.IndexID).Doc(b.Doc)
|
|||
|
|
if up.Insert {
|
|||
|
|
request.DocAsUpsert(true)
|
|||
|
|
}
|
|||
|
|
//fmt.Println(request)
|
|||
|
|
bulkRequest.Add(request)
|
|||
|
|
}
|
|||
|
|
if _, err = bulkRequest.Do(c); err != nil {
|
|||
|
|
log.Error("esCluster(%s) bulk error(%v)", esCluster, err)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// searchResult get result from ES. (Deprecated) v3迁移完要删掉.
|
|||
|
|
func (d *Dao) searchResult(c context.Context, esClusterName, indexName string, query elastic.Query, bsp *model.BasicSearchParams) (res *model.SearchResult, err error) {
|
|||
|
|
res = &model.SearchResult{Debug: ""}
|
|||
|
|
if bsp.Debug {
|
|||
|
|
if src, e := query.Source(); e == nil {
|
|||
|
|
if data, er := json.Marshal(src); er == nil {
|
|||
|
|
res = &model.SearchResult{Debug: string(data)}
|
|||
|
|
} else {
|
|||
|
|
err = er
|
|||
|
|
log.Error("searchResult query.Source.json.Marshal error(%v)", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
err = e
|
|||
|
|
log.Error("searchResult query.Source error(%v)", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if _, ok := d.esPool[esClusterName]; !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:集群不存在%s", esClusterName), "s.dao.searchResult indexName:%s", indexName)
|
|||
|
|
res = &model.SearchResult{Debug: fmt.Sprintf("es:集群不存在%s, %s", esClusterName, res.Debug)}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
// multi sort
|
|||
|
|
sorterSlice := []elastic.Sorter{}
|
|||
|
|
if bsp.KW != "" && bsp.ScoreFirst {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for i, d := range bsp.Order {
|
|||
|
|
if len(bsp.Sort) < i+1 {
|
|||
|
|
if bsp.Sort[0] == "desc" {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(d).Desc())
|
|||
|
|
} else {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(d).Asc())
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
if bsp.Sort[i] == "desc" {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(d).Desc())
|
|||
|
|
} else {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(d).Asc())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if bsp.KW != "" && !bsp.ScoreFirst {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
// source
|
|||
|
|
fsc := elastic.NewFetchSourceContext(true).Include(bsp.Source...)
|
|||
|
|
// highlight
|
|||
|
|
hl := elastic.NewHighlight()
|
|||
|
|
if bsp.Highlight && len(bsp.KwFields) > 0 {
|
|||
|
|
for _, v := range bsp.KwFields {
|
|||
|
|
hl = hl.Fields(elastic.NewHighlighterField(v))
|
|||
|
|
}
|
|||
|
|
hl = hl.PreTags("<em class=\"keyword\">").PostTags("</em>")
|
|||
|
|
}
|
|||
|
|
// from + size = 10,000
|
|||
|
|
from := (bsp.Pn - 1) * bsp.Ps
|
|||
|
|
size := bsp.Ps
|
|||
|
|
if (from + size) > 10000 {
|
|||
|
|
from = 10000 - size
|
|||
|
|
}
|
|||
|
|
// do
|
|||
|
|
searchResult, err := d.esPool[esClusterName].
|
|||
|
|
Search().Index(indexName).
|
|||
|
|
Highlight(hl).
|
|||
|
|
Query(query).
|
|||
|
|
SortBy(sorterSlice...).
|
|||
|
|
From(from).
|
|||
|
|
Size(size).
|
|||
|
|
Pretty(true).
|
|||
|
|
FetchSourceContext(fsc).
|
|||
|
|
Do(context.Background())
|
|||
|
|
if err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:执行查询失败%s ", esClusterName), "%v", err)
|
|||
|
|
res = &model.SearchResult{Debug: res.Debug + "es:执行查询失败"}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
var data []json.RawMessage
|
|||
|
|
b := bytes.Buffer{}
|
|||
|
|
b.WriteString("{")
|
|||
|
|
b.WriteString("}")
|
|||
|
|
for _, hit := range searchResult.Hits.Hits {
|
|||
|
|
var t json.RawMessage
|
|||
|
|
e := json.Unmarshal(*hit.Source, &t)
|
|||
|
|
if e != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:%s 索引有脏数据", esClusterName), "s.dao.SearchArchiveCheck(%d,%d) error(%v) ", bsp.Pn*bsp.Ps, bsp.Ps, e)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
data = append(data, t)
|
|||
|
|
// highlight
|
|||
|
|
if len(hit.Highlight) > 0 {
|
|||
|
|
b, _ := json.Marshal(hit.Highlight)
|
|||
|
|
h := []byte(string(b))
|
|||
|
|
data = append(data, h)
|
|||
|
|
} else if bsp.Highlight {
|
|||
|
|
data = append(data, b.Bytes()) //保证在高亮情况下，肯定有一对数据
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(data) == 0 {
|
|||
|
|
data = []json.RawMessage{}
|
|||
|
|
}
|
|||
|
|
res = &model.SearchResult{
|
|||
|
|
Order: strings.Join(bsp.Order, ","),
|
|||
|
|
Sort: strings.Join(bsp.Sort, ","),
|
|||
|
|
Result: data,
|
|||
|
|
Debug: res.Debug,
|
|||
|
|
Page: &model.Page{
|
|||
|
|
Pn: bsp.Pn,
|
|||
|
|
Ps: bsp.Ps,
|
|||
|
|
Total: searchResult.Hits.TotalHits,
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// QueryResult query result from ES.
|
|||
|
|
func (d *Dao) QueryResult(c context.Context, query elastic.Query, sp *model.QueryParams, qbDebug *model.QueryDebugResult) (res *model.QueryResult, qrDebug *model.QueryDebugResult, err error) {
|
|||
|
|
qrDebug = &model.QueryDebugResult{}
|
|||
|
|
if qbDebug != nil {
|
|||
|
|
qrDebug = qbDebug
|
|||
|
|
}
|
|||
|
|
esCluster := sp.AppIDConf.ESCluster
|
|||
|
|
if _, ok := d.esPool[esCluster]; !ok {
|
|||
|
|
qrDebug.AddErrMsg("es:集群不存在" + esCluster)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if sp.DebugLevel != 0 {
|
|||
|
|
qrDebug.Mapping, err = d.esPool[esCluster].GetMapping().Index(sp.QueryBody.From).Do(context.Background())
|
|||
|
|
}
|
|||
|
|
// 低级别debug，在dsl执行前退出
|
|||
|
|
if sp.DebugLevel == 1 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
// multi sort
|
|||
|
|
sorterSlice := []elastic.Sorter{}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && sp.QueryBody.OrderScoreFirst { // like 长度 > 0，但里面是空的也是个问题
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for _, i := range sp.QueryBody.Order {
|
|||
|
|
for k, v := range i {
|
|||
|
|
if v == "asc" {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Asc())
|
|||
|
|
} else {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Desc())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && sp.QueryBody.OrderScoreFirst {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
// source
|
|||
|
|
fsc := elastic.NewFetchSourceContext(true).Include(sp.QueryBody.Fields...)
|
|||
|
|
// highlight
|
|||
|
|
hl := elastic.NewHighlight()
|
|||
|
|
if sp.QueryBody.Highlight && len(sp.QueryBody.Where.Like) > 0 {
|
|||
|
|
for _, v := range sp.QueryBody.Where.Like {
|
|||
|
|
for _, field := range v.KWFields {
|
|||
|
|
hl = hl.Fields(elastic.NewHighlighterField(field))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
hl = hl.PreTags("<em class=\"keyword\">").PostTags("</em>")
|
|||
|
|
}
|
|||
|
|
// from + size = 10,000
|
|||
|
|
maxRows := 10000
|
|||
|
|
if b, ok := model.PermConf["oht"][sp.Business]; ok && b == "true" {
|
|||
|
|
maxRows = 100000
|
|||
|
|
}
|
|||
|
|
from := (sp.QueryBody.Pn - 1) * sp.QueryBody.Ps
|
|||
|
|
size := sp.QueryBody.Ps
|
|||
|
|
if (from + size) > maxRows {
|
|||
|
|
from = maxRows - size
|
|||
|
|
}
|
|||
|
|
// Scroll
|
|||
|
|
if sp.QueryBody.Scroll == true {
|
|||
|
|
var (
|
|||
|
|
tList []json.RawMessage
|
|||
|
|
tLen int
|
|||
|
|
ScrollID = ""
|
|||
|
|
)
|
|||
|
|
res = &model.QueryResult{}
|
|||
|
|
esCluster := sp.AppIDConf.ESCluster
|
|||
|
|
eSearch, ok := d.esPool[esCluster]
|
|||
|
|
if !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:集群不存在%s", esCluster), "s.dao.searchResult indexName:%s", esCluster)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
fsc := elastic.NewFetchSourceContext(true).Include(sp.QueryBody.Fields...)
|
|||
|
|
// multi sort
|
|||
|
|
sorterSlice := []elastic.Sorter{}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && sp.QueryBody.OrderScoreFirst { // like 长度 > 0，但里面是空的也是个问题
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for _, i := range sp.QueryBody.Order {
|
|||
|
|
for k, v := range i {
|
|||
|
|
if v == "asc" {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Asc())
|
|||
|
|
} else {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Desc())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && !sp.QueryBody.OrderScoreFirst {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for {
|
|||
|
|
searchResult, err := eSearch.Scroll().Index(sp.QueryBody.From).
|
|||
|
|
Query(query).FetchSourceContext(fsc).Size(sp.QueryBody.Ps).Scroll("1m").ScrollId(ScrollID).SortBy(sorterSlice...).Do(c)
|
|||
|
|
if err == io.EOF {
|
|||
|
|
break
|
|||
|
|
} else if err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:执行查询失败%s ", "Scroll"), "es:执行查询失败%v", err)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
ScrollID = searchResult.ScrollId
|
|||
|
|
for _, hit := range searchResult.Hits.Hits {
|
|||
|
|
var t json.RawMessage
|
|||
|
|
if err = json.Unmarshal(*hit.Source, &t); err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "Scroll"), "es:Unmarshal%v", err)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
tList = append(tList, t)
|
|||
|
|
tLen++
|
|||
|
|
if tLen >= sp.QueryBody.Pn*sp.QueryBody.Ps {
|
|||
|
|
goto ClearScroll
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
ClearScroll:
|
|||
|
|
go eSearch.ClearScroll().ScrollId(ScrollID).Do(context.Background())
|
|||
|
|
if res.Result, err = json.Marshal(tList); err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "Scroll"), "es:Unmarshal%v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
// do
|
|||
|
|
searchPrepare := d.esPool[esCluster].
|
|||
|
|
Search().Index(sp.QueryBody.From).
|
|||
|
|
Highlight(hl).
|
|||
|
|
Query(query).
|
|||
|
|
SortBy(sorterSlice...).
|
|||
|
|
From(from).
|
|||
|
|
Size(size).
|
|||
|
|
FetchSourceContext(fsc).IgnoreUnavailable(true).AllowNoIndices(true)
|
|||
|
|
if ec, ok := model.PermConf["es_cache"][sp.Business]; ok && ec == "true" {
|
|||
|
|
searchPrepare.RequestCache(true)
|
|||
|
|
}
|
|||
|
|
if rt, ok := model.PermConf["routing"][sp.Business]; ok {
|
|||
|
|
routing := make([]string, 0, 1)
|
|||
|
|
if sp.QueryBody.Where.EQ != nil {
|
|||
|
|
if eq, ok := sp.QueryBody.Where.EQ[rt]; ok {
|
|||
|
|
routing = append(routing, fmt.Sprintf("%v", eq))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if sp.QueryBody.Where.In != nil {
|
|||
|
|
if in, ok := sp.QueryBody.Where.In[rt]; ok {
|
|||
|
|
for _, v := range in {
|
|||
|
|
routing = append(routing, fmt.Sprintf("%v", v))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(routing) == 0 {
|
|||
|
|
qrDebug.AddErrMsg("es:路由不存在" + rt)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
searchPrepare.Routing(routing...)
|
|||
|
|
}
|
|||
|
|
if sp.DebugLevel == 2 {
|
|||
|
|
searchPrepare.Profile(true)
|
|||
|
|
}
|
|||
|
|
// Enhanced
|
|||
|
|
for _, v := range sp.QueryBody.Where.Enhanced {
|
|||
|
|
aggKey := v.Mode + "_" + v.Field
|
|||
|
|
switch v.Mode {
|
|||
|
|
case model.EnhancedModeGroupBy:
|
|||
|
|
aggs := elastic.NewTermsAggregation()
|
|||
|
|
aggs = aggs.Field(v.Field).Size(1000) //要和业务方确定具体值
|
|||
|
|
searchPrepare.Aggregation(aggKey, aggs)
|
|||
|
|
|
|||
|
|
case model.EnhancedModeCollapse, model.EnhancedModeDistinct:
|
|||
|
|
collapse := elastic.NewCollapseBuilder(v.Field).MaxConcurrentGroupRequests(1)
|
|||
|
|
innerHit := elastic.NewInnerHit().Name("last_one").Size(1)
|
|||
|
|
for _, v := range v.Order {
|
|||
|
|
for field, sort := range v {
|
|||
|
|
if sort == "desc" {
|
|||
|
|
innerHit.Sort(field, false)
|
|||
|
|
} else {
|
|||
|
|
innerHit.Sort(field, true)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(v.Order) > 0 {
|
|||
|
|
collapse.InnerHit(innerHit)
|
|||
|
|
}
|
|||
|
|
searchPrepare.Collapse(collapse)
|
|||
|
|
case model.EnhancedModeSum:
|
|||
|
|
aggs := elastic.NewSumAggregation()
|
|||
|
|
aggs = aggs.Field(v.Field)
|
|||
|
|
searchPrepare.Aggregation(aggKey, aggs)
|
|||
|
|
case model.EnhancedModeDistinctCount:
|
|||
|
|
aggs := elastic.NewCardinalityAggregation()
|
|||
|
|
aggs = aggs.Field(v.Field)
|
|||
|
|
searchPrepare.Aggregation(aggKey, aggs)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
searchResult, err := searchPrepare.Do(context.Background())
|
|||
|
|
if err != nil {
|
|||
|
|
qrDebug.AddErrMsg(fmt.Sprintf("es:执行查询失败%s. %v", esCluster, err))
|
|||
|
|
PromError(fmt.Sprintf("es:执行查询失败%s ", esCluster), "%v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
// data
|
|||
|
|
data := json.RawMessage{}
|
|||
|
|
docHits := []json.RawMessage{}
|
|||
|
|
docBuckets := map[string][]map[string]*json.RawMessage{}
|
|||
|
|
b := bytes.Buffer{}
|
|||
|
|
b.WriteString("{")
|
|||
|
|
b.WriteString("}")
|
|||
|
|
for _, hit := range searchResult.Hits.Hits {
|
|||
|
|
var t json.RawMessage
|
|||
|
|
e := json.Unmarshal(*hit.Source, &t)
|
|||
|
|
if e != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:%s 索引有脏数据", esCluster), "s.dao.SearchArchiveCheck(%d,%d) error(%v) ", sp.QueryBody.Pn*sp.QueryBody.Ps, sp.QueryBody.Ps, e)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
docHits = append(docHits, t)
|
|||
|
|
// highlight
|
|||
|
|
if len(hit.Highlight) > 0 {
|
|||
|
|
b, _ := json.Marshal(hit.Highlight)
|
|||
|
|
docHits = append(docHits, b)
|
|||
|
|
} else if sp.QueryBody.Highlight {
|
|||
|
|
docHits = append(docHits, b.Bytes()) //保证在高亮情况下，肯定有一对数据
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(docHits) > 0 {
|
|||
|
|
if doc, er := json.Marshal(docHits); er != nil {
|
|||
|
|
qrDebug.AddErrMsg(fmt.Sprintf("es:Unmarshal docHits es:Unmarshal%v ", er))
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "docHits"), "es:Unmarshal%v", er)
|
|||
|
|
} else {
|
|||
|
|
data = doc
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
h := bytes.Buffer{}
|
|||
|
|
h.WriteString("[")
|
|||
|
|
h.WriteString("]")
|
|||
|
|
data = h.Bytes()
|
|||
|
|
}
|
|||
|
|
// data overwrite
|
|||
|
|
for _, v := range sp.QueryBody.Where.Enhanced {
|
|||
|
|
key := v.Mode + "_" + v.Field
|
|||
|
|
switch v.Mode {
|
|||
|
|
case model.EnhancedModeGroupBy:
|
|||
|
|
result, ok := searchResult.Aggregations.Terms(key)
|
|||
|
|
if !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", key), "es:Unmarshal%v", err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
for _, b := range result.Buckets {
|
|||
|
|
docBuckets[key] = append(docBuckets[key], b.Aggregations)
|
|||
|
|
}
|
|||
|
|
data = b.Bytes() //保证无数据情况下，有正常返回
|
|||
|
|
case model.EnhancedModeSum:
|
|||
|
|
result, ok := searchResult.Aggregations.Sum(key)
|
|||
|
|
if !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", key), "es:Unmarshal%v", err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
docBuckets[key] = append(docBuckets[key], result.Aggregations)
|
|||
|
|
data = b.Bytes() //保证无数据情况下，有正常返回
|
|||
|
|
case model.EnhancedModeDistinctCount:
|
|||
|
|
result, ok := searchResult.Aggregations.Cardinality(key)
|
|||
|
|
if !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", key), "es:Unmarshal%v", err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
docBuckets[key] = append(docBuckets[key], result.Aggregations)
|
|||
|
|
data = b.Bytes() //保证无数据情况下，有正常返回
|
|||
|
|
default:
|
|||
|
|
// other modes...
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(docBuckets) > 0 {
|
|||
|
|
if doc, er := json.Marshal(docBuckets); er != nil {
|
|||
|
|
qrDebug.AddErrMsg(fmt.Sprintf("es:Unmarshal docBuckets es:Unmarshal%v", er))
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "docBuckets"), "es:Unmarshal%v", er)
|
|||
|
|
} else {
|
|||
|
|
data = doc
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
order := []string{}
|
|||
|
|
sort := []string{}
|
|||
|
|
for _, i := range sp.QueryBody.Order {
|
|||
|
|
for k, v := range i {
|
|||
|
|
order = append(order, k)
|
|||
|
|
sort = append(sort, v)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
res = &model.QueryResult{
|
|||
|
|
Order: strings.Join(order, ","),
|
|||
|
|
Sort: strings.Join(sort, ","),
|
|||
|
|
Result: data,
|
|||
|
|
Page: &model.Page{
|
|||
|
|
Pn: sp.QueryBody.Pn,
|
|||
|
|
Ps: sp.QueryBody.Ps,
|
|||
|
|
Total: searchResult.Hits.TotalHits,
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
//（默认的debug）高级别debug，在dsl执行后退出
|
|||
|
|
if sp.DebugLevel == 2 {
|
|||
|
|
qrDebug.Profile = searchResult.Profile
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// BulkIndex .
|
|||
|
|
func (d *Dao) BulkIndex(c context.Context, esName string, bulkData []BulkItem) (err error) {
|
|||
|
|
bulkRequest := d.esPool[esName].Bulk()
|
|||
|
|
for _, b := range bulkData {
|
|||
|
|
request := elastic.NewBulkIndexRequest().Index(b.IndexName()).Type(b.IndexType()).Id(b.IndexID()).Doc(b)
|
|||
|
|
bulkRequest.Add(request)
|
|||
|
|
}
|
|||
|
|
if _, err = bulkRequest.Do(c); err != nil {
|
|||
|
|
log.Error("esName(%s) bulk error(%v)", esName, err)
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ExistIndex .
|
|||
|
|
func (d *Dao) ExistIndex(c context.Context, esClusterName, indexName string) (exist bool, err error) {
|
|||
|
|
if _, ok := d.esPool[esClusterName]; !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:集群不存在%s", esClusterName), "s.dao.searchResult indexName:%s", indexName)
|
|||
|
|
err = fmt.Errorf("集群不存在")
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
exist, err = d.esPool[esClusterName].IndexExists(indexName).Do(c)
|
|||
|
|
return
|
|||
|
|
}
|