444 lines
12 KiB
Go
444 lines
12 KiB
Go
|
package dao
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"encoding/json"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"strings"
|
|||
|
|
|
|||
|
|
"go-common/app/admin/main/search/model"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
|
|||
|
|
"gopkg.in/olivere/elastic.v5"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
const (
|
|||
|
|
_queryConfSQL = `select appid,es_name,index_prefix,index_type,index_id,index_mapping,query_max_indexes from digger_app`
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// QueryConf query conf
|
|||
|
|
func (d *Dao) QueryConf(ctx context.Context) (res map[string]*model.QueryConfDetail, err error) {
|
|||
|
|
rows, err := d.queryConfStmt.Query(ctx)
|
|||
|
|
if err != nil {
|
|||
|
|
log.Error("d.queryConfStmt.Query() error(%v)", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
defer rows.Close()
|
|||
|
|
res = make(map[string]*model.QueryConfDetail)
|
|||
|
|
for rows.Next() {
|
|||
|
|
var (
|
|||
|
|
appid string
|
|||
|
|
qcd = new(model.QueryConfDetail)
|
|||
|
|
)
|
|||
|
|
if err = rows.Scan(&appid, &qcd.ESCluster, &qcd.IndexPrefix, &qcd.IndexType, &qcd.IndexID, &qcd.IndexMapping, &qcd.MaxIndicesNum); err != nil {
|
|||
|
|
log.Error("d.QueryConf() rows.Scan() error(%v)", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
res[appid] = qcd
|
|||
|
|
}
|
|||
|
|
err = rows.Err()
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
type querysModel struct {
|
|||
|
|
field string
|
|||
|
|
whereKind string
|
|||
|
|
esQuery elastic.Query
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// QueryBasic 其中boolQuery方便定制化业务传参过来.
|
|||
|
|
func (d *Dao) QueryBasic(c context.Context, sp *model.QueryParams) (mixedQuery *elastic.BoolQuery, qbDebug *model.QueryDebugResult) {
|
|||
|
|
mixedQuery = elastic.NewBoolQuery()
|
|||
|
|
qbDebug = &model.QueryDebugResult{}
|
|||
|
|
querys := []*querysModel{}
|
|||
|
|
netstedQuerys := map[string]*elastic.BoolQuery{} // key: path value: boolQuery
|
|||
|
|
//fields
|
|||
|
|
if len(sp.QueryBody.Fields) == 0 {
|
|||
|
|
sp.QueryBody.Fields = []string{}
|
|||
|
|
}
|
|||
|
|
//from done
|
|||
|
|
//where
|
|||
|
|
if sp.QueryBody.Where == nil {
|
|||
|
|
sp.QueryBody.Where = &model.QueryBodyWhere{} //要给个默认值
|
|||
|
|
}
|
|||
|
|
//where - eq
|
|||
|
|
for k, v := range sp.QueryBody.Where.EQ {
|
|||
|
|
querys = append(querys, &querysModel{
|
|||
|
|
field: k,
|
|||
|
|
whereKind: "eq",
|
|||
|
|
esQuery: elastic.NewTermQuery(k, v),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
//where - or
|
|||
|
|
for k, v := range sp.QueryBody.Where.Or {
|
|||
|
|
querys = append(querys, &querysModel{
|
|||
|
|
field: k,
|
|||
|
|
whereKind: "or",
|
|||
|
|
esQuery: elastic.NewTermQuery(k, v),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
//where - in
|
|||
|
|
for k, v := range sp.QueryBody.Where.In {
|
|||
|
|
if len(v) > 1024 {
|
|||
|
|
e := fmt.Sprintf("where in 超过1024 business(%s) error(%v)", sp.Business, v)
|
|||
|
|
log.Error(e)
|
|||
|
|
qbDebug.AddErrMsg(e)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
querys = append(querys, &querysModel{
|
|||
|
|
field: k,
|
|||
|
|
whereKind: "in",
|
|||
|
|
esQuery: elastic.NewTermsQuery(k, v...),
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
//where - range
|
|||
|
|
ranges, err := d.queryBasicRange(sp.QueryBody.Where.Range)
|
|||
|
|
if err != nil {
|
|||
|
|
qbDebug.AddErrMsg(err.Error())
|
|||
|
|
}
|
|||
|
|
for k, v := range ranges {
|
|||
|
|
querys = append(querys, &querysModel{
|
|||
|
|
field: k,
|
|||
|
|
whereKind: "range",
|
|||
|
|
esQuery: v,
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
//where - combo
|
|||
|
|
for _, v := range sp.QueryBody.Where.Combo {
|
|||
|
|
//外面用bool+should+minimum包裹
|
|||
|
|
combo := elastic.NewBoolQuery()
|
|||
|
|
//里面每个子项也是bool+should+minimum
|
|||
|
|
cmbEQ := elastic.NewBoolQuery()
|
|||
|
|
cmbIn := elastic.NewBoolQuery()
|
|||
|
|
cmbRange := elastic.NewBoolQuery()
|
|||
|
|
cmbNotEQ := elastic.NewBoolQuery()
|
|||
|
|
cmbNotIn := elastic.NewBoolQuery()
|
|||
|
|
cmbNotRange := elastic.NewBoolQuery()
|
|||
|
|
//所有的minumum
|
|||
|
|
if v.Min.Min == 0 {
|
|||
|
|
v.Min.Min = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.EQ == 0 {
|
|||
|
|
v.Min.EQ = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.In == 0 {
|
|||
|
|
v.Min.In = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.Range == 0 {
|
|||
|
|
v.Min.Range = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.NotEQ == 0 {
|
|||
|
|
v.Min.NotEQ = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.NotIn == 0 {
|
|||
|
|
v.Min.NotIn = 1
|
|||
|
|
}
|
|||
|
|
if v.Min.NotRange == 0 {
|
|||
|
|
v.Min.NotRange = 1
|
|||
|
|
}
|
|||
|
|
//子项should
|
|||
|
|
for _, vEQ := range v.EQ {
|
|||
|
|
for eqK, eqV := range vEQ {
|
|||
|
|
cmbEQ.Should(elastic.NewTermQuery(eqK, eqV))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, vIn := range v.In {
|
|||
|
|
for inK, inV := range vIn {
|
|||
|
|
cmbIn.Should(elastic.NewTermsQuery(inK, inV...))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, vRange := range v.Range {
|
|||
|
|
ranges, _ := d.queryBasicRange(vRange)
|
|||
|
|
for _, rangeV := range ranges {
|
|||
|
|
cmbRange.Should(rangeV)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, notEQ := range v.NotEQ {
|
|||
|
|
for k, v := range notEQ {
|
|||
|
|
cmbNotEQ.Should(elastic.NewTermQuery(k, v))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, notIn := range v.NotIn {
|
|||
|
|
for k, v := range notIn {
|
|||
|
|
cmbNotIn.Should(elastic.NewTermsQuery(k, v...))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for _, notRange := range v.NotRange {
|
|||
|
|
ranges, _ := d.queryBasicRange(notRange)
|
|||
|
|
for _, v := range ranges {
|
|||
|
|
cmbNotRange.Should(v)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
//子项minimum
|
|||
|
|
if len(v.EQ) > 0 {
|
|||
|
|
combo.Should(cmbEQ.MinimumNumberShouldMatch(v.Min.EQ))
|
|||
|
|
}
|
|||
|
|
if len(v.In) > 0 {
|
|||
|
|
combo.Should(cmbIn.MinimumNumberShouldMatch(v.Min.In))
|
|||
|
|
}
|
|||
|
|
if len(v.Range) > 0 {
|
|||
|
|
combo.Should(cmbRange.MinimumNumberShouldMatch(v.Min.Range))
|
|||
|
|
}
|
|||
|
|
if len(v.NotEQ) > 0 {
|
|||
|
|
combo.MustNot(elastic.NewBoolQuery().Should(cmbNotEQ.MinimumNumberShouldMatch(v.Min.NotEQ)))
|
|||
|
|
}
|
|||
|
|
if len(v.NotIn) > 0 {
|
|||
|
|
combo.MustNot(elastic.NewBoolQuery().Should(cmbNotIn.MinimumNumberShouldMatch(v.Min.NotIn)))
|
|||
|
|
}
|
|||
|
|
if len(v.NotRange) > 0 {
|
|||
|
|
combo.MustNot(elastic.NewBoolQuery().Should(cmbNotRange.MinimumNumberShouldMatch(v.Min.NotRange)))
|
|||
|
|
}
|
|||
|
|
//合并子项
|
|||
|
|
mixedQuery.Filter(combo.MinimumNumberShouldMatch(v.Min.Min))
|
|||
|
|
}
|
|||
|
|
//where - like
|
|||
|
|
like, err := d.queryBasicLike(sp.QueryBody.Where.Like, sp.Business)
|
|||
|
|
if err != nil {
|
|||
|
|
qbDebug.AddErrMsg(err.Error())
|
|||
|
|
}
|
|||
|
|
for _, v := range like {
|
|||
|
|
querys = append(querys, &querysModel{
|
|||
|
|
whereKind: "like",
|
|||
|
|
esQuery: v,
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
//mixedQuery
|
|||
|
|
for _, q := range querys {
|
|||
|
|
// like TODO like的map型字段也要支持must not和 nested
|
|||
|
|
if q.field == "" && q.whereKind == "like" {
|
|||
|
|
mixedQuery.Must(q.esQuery)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
if q.field == "" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// prepare nested 一个DSL只能出现一个nested，不然会有问题
|
|||
|
|
if mapField := strings.Split(q.field, "."); len(mapField) > 1 && mapField[0] != "" {
|
|||
|
|
if _, ok := netstedQuerys[mapField[0]]; !ok {
|
|||
|
|
netstedQuerys[mapField[0]] = elastic.NewBoolQuery()
|
|||
|
|
}
|
|||
|
|
if bl, ok := sp.QueryBody.Where.Not[q.whereKind][q.field]; ok && bl {
|
|||
|
|
// mixedQuery.Must(elastic.NewNestedQuery(mapField[0], elastic.NewBoolQuery().MustNot(q.esQuery)))
|
|||
|
|
netstedQuerys[mapField[0]].MustNot(q.esQuery)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// mixedQuery.Must(elastic.NewNestedQuery(mapField[0], elastic.NewBoolQuery().Must(q.esQuery)))
|
|||
|
|
netstedQuerys[mapField[0]].Must(q.esQuery)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// must not
|
|||
|
|
if bl, ok := sp.QueryBody.Where.Not[q.whereKind][q.field]; ok && bl {
|
|||
|
|
mixedQuery.MustNot(q.esQuery)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// should
|
|||
|
|
if q.whereKind == "or" {
|
|||
|
|
mixedQuery.Should(q.esQuery)
|
|||
|
|
mixedQuery.MinimumShouldMatch("1") // 暂时为1
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// default
|
|||
|
|
mixedQuery.Filter(q.esQuery)
|
|||
|
|
// random order with seed
|
|||
|
|
if sp.QueryBody.OrderRandomSeed != "" {
|
|||
|
|
random := elastic.NewRandomFunction().Seed(sp.QueryBody.OrderRandomSeed)
|
|||
|
|
score := elastic.NewFunctionScoreQuery().Add(elastic.NewBoolQuery(), random)
|
|||
|
|
mixedQuery.Must(score)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// insert nested
|
|||
|
|
for k, n := range netstedQuerys {
|
|||
|
|
mixedQuery.Must(elastic.NewNestedQuery(k, n))
|
|||
|
|
}
|
|||
|
|
// DSL
|
|||
|
|
if sp.DebugLevel != 0 {
|
|||
|
|
if src, e := mixedQuery.Source(); e == nil {
|
|||
|
|
if data, er := json.Marshal(src); er == nil {
|
|||
|
|
qbDebug.DSL = string(data)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// queryBasicRange .
|
|||
|
|
func (d *Dao) queryBasicRange(rangeMap map[string]string) (rangeQuery map[string]*elastic.RangeQuery, err error) {
|
|||
|
|
rangeQuery = make(map[string]*elastic.RangeQuery)
|
|||
|
|
for k, v := range rangeMap {
|
|||
|
|
if r := strings.Trim(v, " "); r != "" {
|
|||
|
|
if rs := []rune(r); len(rs) > 3 {
|
|||
|
|
firstStr := string(rs[0:1])
|
|||
|
|
endStr := string(rs[len(rs)-1:])
|
|||
|
|
rangeStr := strings.Trim(v, "[]() ")
|
|||
|
|
FromTo := strings.Split(rangeStr, ",")
|
|||
|
|
if len(FromTo) != 2 {
|
|||
|
|
err = fmt.Errorf("sp.QueryBody.Where.Range Fromto err")
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
rQuery := elastic.NewRangeQuery(k)
|
|||
|
|
rc := 0
|
|||
|
|
if firstStr == "(" && strings.Trim(FromTo[0], " ") != "" {
|
|||
|
|
rQuery.Gt(strings.Trim(FromTo[0], " "))
|
|||
|
|
rc++
|
|||
|
|
}
|
|||
|
|
if firstStr == "[" && strings.Trim(FromTo[0], " ") != "" {
|
|||
|
|
rQuery.Gte(strings.Trim(FromTo[0], " "))
|
|||
|
|
rc++
|
|||
|
|
}
|
|||
|
|
if endStr == ")" && strings.Trim(FromTo[1], " ") != "" {
|
|||
|
|
rQuery.Lt(strings.Trim(FromTo[1], " "))
|
|||
|
|
rc++
|
|||
|
|
}
|
|||
|
|
if endStr == "]" && strings.Trim(FromTo[1], " ") != "" {
|
|||
|
|
rQuery.Lte(strings.Trim(FromTo[1], " "))
|
|||
|
|
rc++
|
|||
|
|
}
|
|||
|
|
if rc == 0 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
rangeQuery[k] = rQuery
|
|||
|
|
} else {
|
|||
|
|
// 范围格式有问题
|
|||
|
|
err = fmt.Errorf("sp.QueryBody.Where.Range range format err. error(%v)", v)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (d *Dao) queryBasicLike(likeMap []model.QueryBodyWhereLike, business string) (likeQuery []elastic.Query, err error) {
|
|||
|
|
for _, v := range likeMap {
|
|||
|
|
if len(v.KW) == 0 {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
switch v.Level {
|
|||
|
|
case model.LikeLevelHigh:
|
|||
|
|
kw := []string{}
|
|||
|
|
r := []rune(v.KW[0])
|
|||
|
|
for i := 0; i < len(r); i++ {
|
|||
|
|
if k := string(r[i : i+1]); !strings.ContainsAny(k, "~[](){}^?:\"\\/!+-=&* ") { //去掉特殊符号
|
|||
|
|
kw = append(kw, k)
|
|||
|
|
} else if len(kw) > 1 && kw[len(kw)-1:][0] != "*" {
|
|||
|
|
kw = append(kw, "*", " ", "*")
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(kw) == 0 || strings.Join(kw, "") == "* *" {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
qs := elastic.NewQueryStringQuery("*" + strings.Trim(strings.Join(kw, ""), "* ") + "*").AllowLeadingWildcard(true) //默认是or
|
|||
|
|
if !v.Or {
|
|||
|
|
qs.DefaultOperator("AND")
|
|||
|
|
}
|
|||
|
|
for _, v := range v.KWFields {
|
|||
|
|
qs.Field(v)
|
|||
|
|
}
|
|||
|
|
likeQuery = append(likeQuery, qs)
|
|||
|
|
case model.LikeLevelMiddel:
|
|||
|
|
// 单个字要特殊处理
|
|||
|
|
if r := []rune(v.KW[0]); len(r) == 1 && len(v.KW) == 1 {
|
|||
|
|
qs := elastic.NewQueryStringQuery("*" + string(r[:]) + "*").AllowLeadingWildcard(true) //默认是or
|
|||
|
|
if !v.Or {
|
|||
|
|
qs.DefaultOperator("AND")
|
|||
|
|
}
|
|||
|
|
for _, v := range v.KWFields {
|
|||
|
|
qs.Field(v)
|
|||
|
|
}
|
|||
|
|
likeQuery = append(likeQuery, qs)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
// 自定义analyzer时，multi_match无法使用minimum_should_match，默认为至少一个满足，导致结果集还是很大
|
|||
|
|
// ngram(2,2)
|
|||
|
|
for _, kw := range v.KW {
|
|||
|
|
rn := []rune(kw)
|
|||
|
|
for i := 0; i+1 < len(rn); i++ {
|
|||
|
|
kwStr := string(rn[i : i+2])
|
|||
|
|
for _, kwField := range v.KWFields {
|
|||
|
|
likeQuery = append(likeQuery, elastic.NewTermQuery(kwField, kwStr))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
case "", model.LikeLevelLow:
|
|||
|
|
qs := elastic.NewMultiMatchQuery(strings.Join(v.KW, " "), v.KWFields...).Type("best_fields").TieBreaker(0.6).MinimumShouldMatch("90%") //默认是and
|
|||
|
|
// TODO 业务自定义match
|
|||
|
|
if business == "copyright" {
|
|||
|
|
qs.MinimumShouldMatch("10%")
|
|||
|
|
}
|
|||
|
|
if business == "academy_archive" {
|
|||
|
|
qs.MinimumShouldMatch("50%")
|
|||
|
|
}
|
|||
|
|
if v.Or {
|
|||
|
|
qs.Operator("OR")
|
|||
|
|
}
|
|||
|
|
likeQuery = append(likeQuery, qs)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (d *Dao) Scroll(c context.Context, sp *model.QueryParams) (res *model.QueryResult, debug *model.QueryDebugResult, err error) {
|
|||
|
|
var (
|
|||
|
|
tList []json.RawMessage
|
|||
|
|
tLen int
|
|||
|
|
ScrollID = ""
|
|||
|
|
)
|
|||
|
|
res = &model.QueryResult{}
|
|||
|
|
esCluster := sp.AppIDConf.ESCluster
|
|||
|
|
query, _ := d.QueryBasic(c, sp)
|
|||
|
|
eSearch, ok := d.esPool[esCluster]
|
|||
|
|
if !ok {
|
|||
|
|
PromError(fmt.Sprintf("es:集群不存在%s", esCluster), "s.dao.searchResult indexName:%s", esCluster)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
fsc := elastic.NewFetchSourceContext(true).Include(sp.QueryBody.Fields...)
|
|||
|
|
// multi sort
|
|||
|
|
sorterSlice := []elastic.Sorter{}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && sp.QueryBody.OrderScoreFirst { // like 长度 > 0，但里面是空的也是个问题
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for _, i := range sp.QueryBody.Order {
|
|||
|
|
for k, v := range i {
|
|||
|
|
if v == "asc" {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Asc())
|
|||
|
|
} else {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewFieldSort(k).Desc())
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if len(sp.QueryBody.Where.Like) > 0 && !sp.QueryBody.OrderScoreFirst {
|
|||
|
|
sorterSlice = append(sorterSlice, elastic.NewScoreSort().Desc())
|
|||
|
|
}
|
|||
|
|
for {
|
|||
|
|
searchResult, err := eSearch.Scroll().Index(sp.QueryBody.From).Type("base").
|
|||
|
|
Query(query).FetchSourceContext(fsc).Size(5000).Scroll("1m").ScrollId(ScrollID).SortBy(sorterSlice...).Do(c)
|
|||
|
|
if err == io.EOF {
|
|||
|
|
break
|
|||
|
|
} else if err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:执行查询失败%s ", "Scroll"), "es:执行查询失败%v", err)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
ScrollID = searchResult.ScrollId
|
|||
|
|
for _, hit := range searchResult.Hits.Hits {
|
|||
|
|
var t json.RawMessage
|
|||
|
|
if err = json.Unmarshal(*hit.Source, &t); err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "Scroll"), "es:Unmarshal%v", err)
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
tList = append(tList, t)
|
|||
|
|
tLen++
|
|||
|
|
if tLen >= sp.QueryBody.Pn*sp.QueryBody.Ps {
|
|||
|
|
goto ClearScroll
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
ClearScroll:
|
|||
|
|
go eSearch.ClearScroll().ScrollId(ScrollID).Do(context.Background())
|
|||
|
|
if res.Result, err = json.Marshal(tList); err != nil {
|
|||
|
|
PromError(fmt.Sprintf("es:Unmarshal%s ", "Scroll"), "es:Unmarshal%v", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|