Create & Init Project...

app/admin/main/search/service/BUILD

@@ -0,0 +1,64 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "archive_test.go",
+        "log_test.go",
+        "mng_v2_test.go",
+        "service_test.go",
+        "update_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/search/conf:go_default_library",
+        "//app/admin/main/search/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "archive.go",
+        "log.go",
+        "mng.go",
+        "mng_v2.go",
+        "query.go",
+        "service.go",
+        "update.go",
+        "upsert.go",
+    ],
+    importpath = "go-common/app/admin/main/search/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/search/conf:go_default_library",
+        "//app/admin/main/search/dao:go_default_library",
+        "//app/admin/main/search/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+    ],
+)

app/admin/main/search/service/archive.go

@@ -0,0 +1,46 @@
+package service
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/admin/main/search/dao"
+	"go-common/app/admin/main/search/model"
+	"go-common/library/ecode"
+)
+
+// ArchiveCheck gets archive check.
+func (s *Service) ArchiveCheck(c context.Context, sp *model.ArchiveCheckParams) (res *model.SearchResult, err error) {
+	if res, err = s.dao.ArchiveCheck(c, sp); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索archivecheck失败", sp.Bsp.AppID), "s.dao.SearchArchiveCheck(%v) error(%v) ", sp, err)
+		err = ecode.SearchArchiveCheckFailed
+	}
+	return
+}
+
+// Video gets video relation.
+func (s *Service) Video(c context.Context, sp *model.VideoParams) (res *model.SearchResult, err error) {
+	if res, err = s.dao.Video(c, sp); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索video失败", sp.Bsp.AppID), "s.dao.Video(%v) error(%v) ", sp, err)
+		err = ecode.SearchVideoFailed
+	}
+	return
+}
+
+// TaskQa .
+func (s *Service) TaskQa(c context.Context, sp *model.TaskQa) (res *model.SearchResult, err error) {
+	if res, err = s.dao.TaskQa(c, sp); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索TaskQa失败", sp.Bsp.AppID), "s.dao.TaskQa(%v) error(%v) ", sp, err)
+		err = ecode.SearchVideoFailed
+	}
+	return
+}
+
+// ArchiveCommerce .
+func (s *Service) ArchiveCommerce(c context.Context, sp *model.ArchiveCommerce) (res *model.SearchResult, err error) {
+	if res, err = s.dao.ArchiveCommerce(c, sp); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索ArchiveCommerce失败", sp.Bsp.AppID), "s.dao.TaskQa(%v) error(%v) ", sp, err)
+		err = ecode.SearchVideoFailed
+	}
+	return
+}

app/admin/main/search/service/archive_test.go

@@ -0,0 +1 @@
+package service

app/admin/main/search/service/log.go

@@ -0,0 +1,229 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"go-common/app/admin/main/search/dao"
+	"go-common/app/admin/main/search/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+func convert(params map[string][]string) (res map[string][]interface{}) {
+	res = make(map[string][]interface{})
+	for k, v := range params {
+		var arr []interface{}
+		for _, m := range v {
+			if m != "" {
+				for _, u := range strings.Split(m, ",") {
+					arr = append(arr, u)
+				}
+			}
+		}
+		if len(arr) > 0 {
+			res[k] = arr
+		}
+	}
+	return res
+}
+
+// Check .
+func (s *Service) Check(appID string, businessID int) (business *model.Business, ok bool) {
+	business, ok = s.dao.GetLogInfo(appID, businessID)
+	return
+}
+
+func numberToInt64(in map[string]interface{}) (out map[string]interface{}) {
+	var err error
+	out = map[string]interface{}{}
+	for k, v := range in {
+		if integer, ok := v.(json.Number); ok {
+			if out[k], err = integer.Int64(); err != nil {
+				log.Error("service.log.numberToInt64(%v)(%v)", integer, err)
+			}
+		} else {
+			out[k] = v
+		}
+	}
+	return
+}
+
+// 获取部门
+func (s *Service) uDepTs(c context.Context, res *model.SearchResult, sp *model.LogParams) *model.SearchResult {
+	var (
+		uids   []string
+		result []map[string]interface{}
+		err    error
+	)
+	result = []map[string]interface{}{}
+	for _, j := range res.Result {
+		item := map[string]interface{}{}
+		decoder := json.NewDecoder(bytes.NewReader(j))
+		decoder.UseNumber()
+		if err = decoder.Decode(&item); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索LogAudit JSON失败", sp.Bsp.AppID), "s.dao.LogAudit(%v) json error(%v)", sp, err)
+		}
+		item = numberToInt64(item)
+		result = append(result, item)
+		if _, ok := item["uid"]; ok {
+			uids = append(uids, fmt.Sprintf("%v", item["uid"]))
+		}
+	}
+	var depRs = &model.UDepTsData{
+		Data: map[string]string{},
+	}
+	if len(uids) > 0 {
+		if depRs, err = s.dao.UDepTs(c, uids); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索LogAudit失败", sp.Bsp.AppID), "s.dao.LogAudit(%v) error(%v)", sp, err)
+			err = ecode.SearchLogAuditFailed
+			return res
+		}
+	}
+	for i, j := range result {
+		result[i]["department"] = ""
+		if _, ok := j["uid"]; ok {
+			if m, sok := depRs.Data[fmt.Sprintf("%v", j["uid"])]; sok {
+				result[i]["department"] = m
+			}
+		}
+		if res.Result[i], err = json.Marshal(j); err != nil {
+			log.Error("s.dao.LogAudit(%v) json res(%v)", err, j)
+		}
+	}
+	return res
+}
+
+// 获取IP地址
+func (s *Service) IP(c context.Context, res *model.SearchResult, sp *model.LogParams) *model.SearchResult {
+	var (
+		ip     []string
+		result []map[string]interface{}
+		err    error
+	)
+	result = []map[string]interface{}{}
+	for _, j := range res.Result {
+		item := map[string]interface{}{}
+		decoder := json.NewDecoder(bytes.NewReader(j))
+		decoder.UseNumber()
+		if err = decoder.Decode(&item); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索LogUserAction JSON失败", sp.Bsp.AppID), "s.dao.LogUserAction(%v) json error(%v)", sp, err)
+		}
+		item = numberToInt64(item)
+		result = append(result, item)
+		if _, ok := item["ip"]; ok {
+			if v := fmt.Sprintf("%v", item["ip"]); v != "" {
+				ip = append(ip, v)
+			}
+		}
+	}
+	var ipData = &model.IPData{
+		Data: map[string]struct {
+			Country  string `json:"country"`
+			Province string `json:"province"`
+			City     string `json:"city"`
+			Isp      string `json:"isp"`
+		}{},
+	}
+	if len(ip) > 0 {
+		if ipData, err = s.dao.IP(c, ip); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索LogUserAction失败", sp.Bsp.AppID), "s.dao.LogUserAction(%v) error(%v)", sp, err)
+			err = ecode.SearchLogAuditFailed
+			return res
+		}
+	}
+	for i, j := range result {
+		result[i]["location"] = ""
+		if _, ok := j["ip"]; ok {
+			if m, sok := ipData.Data[fmt.Sprintf("%v", j["ip"])]; sok {
+				location := make([]string, 0, 4)
+				for _, v := range []string{m.Country, m.Province, m.City, m.Isp} {
+					if v != "" {
+						location = append(location, v)
+					}
+				}
+				result[i]["location"] = strings.Join(location, "-")
+			}
+		}
+		if res.Result[i], err = json.Marshal(j); err != nil {
+			log.Error("s.dao.LogUserAction(%v) json res(%v)", err, j)
+		}
+	}
+	return res
+}
+
+// LogAudit .
+func (s *Service) LogAudit(c context.Context, params map[string][]string, sp *model.LogParams, business *model.Business) (res *model.SearchResult, err error) {
+	p := convert(params)
+	if res, err = s.dao.LogAudit(c, p, sp, business); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索LogAudit失败", sp.Bsp.AppID), "s.dao.LogAudit(%v) error(%v)", sp, err)
+		err = ecode.SearchLogAuditFailed
+		return
+	}
+	res = s.uDepTs(c, res, sp)
+	return
+}
+
+// LogAuditGroupBy .
+func (s *Service) LogAuditGroupBy(c context.Context, params map[string][]string, sp *model.LogParams, business *model.Business) (res *model.SearchResult, err error) {
+	p := convert(params)
+	if v, ok := p["group"]; !ok || len(v) == 0 {
+		err = ecode.RequestErr
+		return
+	}
+	if res, err = s.dao.LogAuditGroupBy(c, p, sp, business); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索LogAuditGroupBy失败", sp.Bsp.AppID), "s.dao.LogAuditGroupBy(%v) error(%v)", sp, err)
+		err = ecode.SearchLogAuditOidFailed
+		return
+	}
+	if res.Page.Total < int64(res.Page.Ps*(res.Page.Pn-1)) || len(res.Result) == 0 {
+		res.Result = []json.RawMessage{}
+	} else if int64(res.Page.Ps*(res.Page.Pn-1)) <= res.Page.Total && res.Page.Total <= int64(res.Page.Ps*res.Page.Pn) {
+		res.Result = res.Result[res.Page.Ps*(res.Page.Pn-1) : res.Page.Total]
+	} else {
+		res.Result = res.Result[res.Page.Ps*(res.Page.Pn-1) : res.Page.Ps*res.Page.Pn]
+	}
+	res = s.uDepTs(c, res, sp)
+	return
+}
+
+// LogAuditDelete .
+func (s *Service) LogAuditDelete(c context.Context, params map[string][]string, sp *model.LogParams, business *model.Business) (res *model.SearchResult, err error) {
+	p := convert(params)
+	if res, err = s.dao.LogAuditDelete(c, p, sp, business); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s LogAuditDelete失败", sp.Bsp.AppID), "s.dao.LogAuditDelete(%v) error(%v)", sp, err)
+		err = ecode.SearchLogAuditFailed
+		return
+	}
+	return
+}
+
+// LogUserAction .
+func (s *Service) LogUserAction(c context.Context, params map[string][]string, sp *model.LogParams, business *model.Business) (res *model.SearchResult, err error) {
+	p := convert(params)
+	if res, err = s.dao.LogUserAction(c, p, sp, business); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索LogUserAction失败", sp.Bsp.AppID), "s.dao.LogUserAction(%v) error(%v)", sp, err)
+		err = ecode.SearchLogUserActionFailed
+		return
+	}
+	res = s.IP(c, res, sp)
+	return
+}
+
+// LogUserActionDelete .
+func (s *Service) LogUserActionDelete(c context.Context, params map[string][]string, sp *model.LogParams, business *model.Business) (res *model.SearchResult, err error) {
+	p := convert(params)
+	if res, err = s.dao.LogUserActionDelete(c, p, sp, business); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s LogUserActionDelete失败", sp.Bsp.AppID), "s.dao.LogUserActionDelete(%v) error(%v)", sp, err)
+		err = ecode.SearchLogAuditFailed
+		return
+	}
+	return
+}
+
+func (s *Service) LogCount(c context.Context, name string, business int, uid interface{}) {
+	s.dao.LogCount(c, name, business, uid)
+}

app/admin/main/search/service/log_test.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/admin/main/search/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_LogAudit(t *testing.T) {
+	var (
+		err error
+		c   = context.Background()
+		p   = &model.LogParams{
+			Bsp: &model.BasicSearchParams{
+				AppID: "log_audit",
+			},
+		}
+		params map[string][]string
+	)
+
+	Convey("LogAudit", t, WithService(func(s *Service) {
+		business, ok := svr.Check("log_audit", 0)
+		if !ok {
+			return
+		}
+		_, err = s.LogAudit(c, params, p, business)
+		So(err, ShouldBeNil)
+	}))
+}
+
+//func Test_LogAuditGroupBy(t *testing.T) {
+//	var (
+//		err error
+//		c   = context.Background()
+//		p   = &model.LogParams{
+//			Bsp: &model.BasicSearchParams{
+//				AppID: "log_audit_group",
+//			},
+//		}
+//		params map[string][]string
+//	)
+//	params = map[string][]string{
+//		"group": {"oid"},
+//	}
+//	Convey("LogAuditGroupBy", t, WithService(func(s *Service) {
+//		indexMapping, indexFmt, ok := svr.Check("log_audit", p.Business)
+//		if !ok {
+//			return
+//		}
+//		_, err = s.LogAuditGroupBy(c, params, p, indexMapping, indexFmt)
+//		Printf("---------%v", err)
+//		So(err, ShouldBeNil)
+//	}))
+//}
+
+func Test_LogUserAction(t *testing.T) {
+	var (
+		err error
+		c   = context.Background()
+		p   = &model.LogParams{
+			Bsp: &model.BasicSearchParams{
+				AppID: "log_user_action",
+			},
+		}
+		params map[string][]string
+	)
+	Convey("LogUserAction", t, WithService(func(s *Service) {
+		business, ok := svr.Check("log_user_action", 0)
+		if !ok {
+			return
+		}
+		_, err = s.LogUserAction(c, params, p, business)
+		So(err, ShouldBeNil)
+	}))
+}

app/admin/main/search/service/mng.go

@@ -0,0 +1,307 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+
+	"go-common/app/admin/main/search/model"
+	"go-common/library/ecode"
+	"go-common/library/sync/errgroup"
+)
+
+// BusinessList .
+func (s *Service) BusinessList(ctx context.Context, name string, pn, ps int) (list []*model.MngBusiness, total int64, err error) {
+	offset := (pn - 1) * ps
+	if list, err = s.dao.BusinessList(ctx, name, offset, ps); err != nil {
+		return
+	}
+	total, err = s.dao.BusinessTotal(ctx, name)
+	return
+}
+
+// BusinessAll .
+func (s *Service) BusinessAll(ctx context.Context) (list []*model.MngBusiness, err error) {
+	list, err = s.dao.BusinessAll(ctx)
+	return
+}
+
+// BusinessInfo .
+func (s *Service) BusinessInfo(ctx context.Context, id int64) (info *model.MngBusiness, err error) {
+	info, err = s.dao.BusinessInfo(ctx, id)
+	return
+}
+
+// AddBusiness .
+func (s *Service) AddBusiness(ctx context.Context, b *model.MngBusiness) (id int64, err error) {
+	info, err := s.dao.BusinessInfoByName(ctx, b.Name)
+	if err != nil {
+		return
+	}
+	if info != nil {
+		err = ecode.SearchBusinessExistErr
+		return
+	}
+	id, err = s.dao.AddBusiness(ctx, b)
+	return
+}
+
+// UpdateBusiness .
+func (s *Service) UpdateBusiness(ctx context.Context, b *model.MngBusiness) (err error) {
+	err = s.dao.UpdateBusiness(ctx, b)
+	return
+}
+
+// UpdateBusinessApp .
+func (s *Service) UpdateBusinessApp(ctx context.Context, business, app, incrWay string, isJob, incrOpen bool) (err error) {
+	info, err := s.dao.BusinessInfoByName(ctx, business)
+	if err != nil {
+		return
+	}
+	var exist bool
+	for k, v := range info.Apps {
+		if v.AppID == app {
+			exist = true
+			if !isJob {
+				info.Apps = append(info.Apps[:k], info.Apps[k+1:]...)
+				break
+			}
+			v.IncrWay = incrWay
+			v.IncrOpen = incrOpen
+		}
+	}
+	if !exist {
+		info.Apps = append(info.Apps, &model.MngBusinessApp{AppID: app, IncrWay: incrWay, IncrOpen: incrOpen})
+	}
+	bs, err := json.Marshal(info.Apps)
+	if err != nil {
+		return
+	}
+	info.AppsJSON = string(bs)
+	err = s.dao.UpdateBusiness(ctx, info)
+	return
+}
+
+// AssetList .
+func (s *Service) AssetList(ctx context.Context, typ int, name string, pn, ps int) (list []*model.MngAsset, total int64, err error) {
+	offset := (pn - 1) * ps
+	if list, err = s.dao.AssetList(ctx, typ, name, offset, ps); err != nil {
+		return
+	}
+	total, err = s.dao.AssetTotal(ctx, typ, name)
+	return
+}
+
+// AssetAll .
+func (s *Service) AssetAll(ctx context.Context) (list []*model.MngAsset, err error) {
+	list, err = s.dao.AssetAll(ctx)
+	return
+}
+
+// AssetInfo .
+func (s *Service) AssetInfo(ctx context.Context, id int64) (info *model.MngAsset, err error) {
+	info, err = s.dao.AssetInfo(ctx, id)
+	return
+}
+
+// AddAsset .
+func (s *Service) AddAsset(ctx context.Context, a *model.MngAsset) (id int64, err error) {
+	info, err := s.dao.AssetInfoByName(ctx, a.Name)
+	if err != nil {
+		return
+	}
+	if info != nil {
+		err = ecode.SearchAssetExistErr
+		return
+	}
+	id, err = s.dao.AddAsset(ctx, a)
+	return
+}
+
+// UpdateAsset .
+func (s *Service) UpdateAsset(ctx context.Context, a *model.MngAsset) (err error) {
+	if err = s.dao.UpdateAsset(ctx, a); err != nil {
+		return
+	}
+	if a.Type == model.MngAssetTypeDatabus {
+		if a.Config == "" {
+			return
+		}
+		v := new(model.MngAssetDatabus)
+		if err = json.Unmarshal([]byte(a.Config), &v); err != nil {
+			return
+		}
+		err = s.dao.UpdateAppAssetDatabus(ctx, a.Name, v)
+		return
+	}
+	if a.Type == model.MngAssetTypeTable {
+		if a.Config == "" {
+			return
+		}
+		v := new(model.MngAssetTable)
+		if err = json.Unmarshal([]byte(a.Config), &v); err != nil {
+			return
+		}
+		err = s.dao.UpdateAppAssetTable(ctx, a.Name, v)
+		return
+	}
+	return
+}
+
+// AppList .
+func (s *Service) AppList(ctx context.Context, business string) (list []*model.MngApp, err error) {
+	list, err = s.dao.AppList(ctx, business)
+	return
+}
+
+// AppInfo .
+func (s *Service) AppInfo(ctx context.Context, id int64) (info *model.MngApp, err error) {
+	info, err = s.dao.AppInfo(ctx, id)
+	return
+}
+
+// AddApp .
+func (s *Service) AddApp(ctx context.Context, a *model.MngApp) (id int64, err error) {
+	info, err := s.dao.AppInfoByAppid(ctx, a.AppID)
+	if err != nil {
+		return
+	}
+	if info != nil {
+		err = ecode.SearchAssetExistErr
+		return
+	}
+	id, err = s.dao.AddApp(ctx, a)
+	return
+}
+
+// UpdateApp .
+func (s *Service) UpdateApp(ctx context.Context, a *model.MngApp) (err error) {
+	group := errgroup.Group{}
+	group.Go(func() error {
+		if a.TableName == "" {
+			a.TableFormat = ""
+			a.TablePrefix = ""
+			return nil
+		}
+		tb, e := s.dao.AssetInfoByName(ctx, a.TableName)
+		if e != nil {
+			return e
+		}
+		if tb == nil || tb.Config == "" {
+			return nil
+		}
+		val := new(model.MngAssetTable)
+		if e := json.Unmarshal([]byte(tb.Config), val); e != nil {
+			return e
+		}
+		a.TablePrefix = val.TablePrefix
+		a.TableFormat = val.TableFormat
+		return nil
+	})
+	group.Go(func() error {
+		if a.DatabusName == "" {
+			a.DatabusInfo = ""
+			a.DatabusIndexID = ""
+			return nil
+		}
+		dbus, e := s.dao.AssetInfoByName(ctx, a.DatabusName)
+		if e != nil {
+			return e
+		}
+		if dbus == nil || dbus.Config == "" {
+			return nil
+		}
+		val := new(model.MngAssetDatabus)
+		if e := json.Unmarshal([]byte(dbus.Config), val); e != nil {
+			return e
+		}
+		a.DatabusInfo = val.DatabusInfo
+		a.DatabusIndexID = val.DatabusIndexID
+		return nil
+	})
+	if err = group.Wait(); err != nil {
+		return
+	}
+	err = s.dao.UpdateApp(ctx, a)
+	return
+}
+
+// MngCountList .
+func (s *Service) MngCountList(ctx context.Context) (list []*model.MngCount, err error) {
+	daily := "每日增量"
+	sum := "历史总量"
+	list = []*model.MngCount{
+		// 业务方
+		{Business: "业务方", Type: sum, Name: "业务方历史总量", Chart: "line", Param: "business=app&type=all"},
+		{Business: "业务方", Type: daily, Name: "业务方每日增量", Chart: "line", Param: "business=app&type=inc"},
+		// 视频+稿件
+		{Business: "视频稿件", Type: daily, Name: "archive每日增量", Chart: "line", Param: "business=archive&type=inc"},
+		{Business: "视频稿件", Type: daily, Name: "video每日增量", Chart: "line", Param: "business=archive_video&type=inc"},
+		{Business: "视频稿件", Type: sum, Name: "archive历史总量", Chart: "line", Param: "business=archive&type=all"},
+		{Business: "视频稿件", Type: sum, Name: "video历史总量", Chart: "line", Param: "business=archive_video&type=all"},
+		// 弹幕
+		{Business: "弹幕", Type: daily, Name: "弹幕每日增量", Chart: "line", Param: "business=dm&type=inc"},
+		{Business: "弹幕", Type: daily, Name: "弹幕举报每日增量", Chart: "line", Param: "business=dm_report&type=inc"},
+		{Business: "弹幕", Type: daily, Name: "弹幕监控每日增量", Chart: "line", Param: "business=dm_monitor&type=inc"},
+		{Business: "弹幕", Type: sum, Name: "弹幕历史总量", Chart: "line", Param: "business=dm&type=all"},
+		{Business: "弹幕", Type: sum, Name: "弹幕举报历史总量", Chart: "line", Param: "business=dm_report&type=all"},
+		{Business: "弹幕", Type: sum, Name: "弹幕监控历史总量", Chart: "line", Param: "business=dm_monitor&type=all"},
+		// 评论
+		{Business: "评论", Type: daily, Name: "评论每日增量", Chart: "line", Param: "business=reply&type=inc"},
+		// 日志
+		{Business: "日志", Type: "审核日志", Name: "审核日志每日查询量", Chart: "line", Param: "business=log_audit_access&type=inc"},
+		{Business: "日志", Type: "审核日志", Name: "审核日志昨日查询情况 - 业务维度", Chart: "pie", Param: "business=log_audit_business&type=inc"},
+		{Business: "日志", Type: "审核日志", Name: "审核日志昨日查询情况 - 用户维度", Chart: "pie", Param: "business=log_audit_uid&type=inc"},
+		{Business: "日志", Type: "用户日志", Name: "用户日志每日查询量", Chart: "line", Param: "business=log_user_action_access&type=inc"},
+		{Business: "日志", Type: "用户日志", Name: "用户日志昨日查询情况 - 业务维度", Chart: "pie", Param: "business=log_user_action_business&type=inc"},
+		{Business: "日志", Type: "用户日志", Name: "用户日志昨日查询情况 - 用户维度", Chart: "pie", Param: "business=log_user_action_uid&type=inc"},
+		// 用户
+		{Business: "用户", Type: sum, Name: "用户历史总量", Chart: "line", Param: "business=user&type=all"},
+		// 专栏
+		{Business: "专栏", Type: daily, Name: "专栏每日增量", Chart: "line", Param: "business=article&type=inc"},
+		{Business: "专栏", Type: sum, Name: "专栏历史总量", Chart: "line", Param: "business=article&type=all"},
+	}
+	return list, err
+}
+
+// MngCount .
+func (s *Service) MngCount(ctx context.Context, c *model.MngCount) (list []*model.MngCountRes, err error) {
+	list, err = s.dao.MngCount(ctx, c)
+	return
+}
+
+// MngCount .
+func (s *Service) MngPercent(ctx context.Context, c *model.MngCount) (list []*model.MngPercentRes, err error) {
+	list, err = s.dao.MngPercent(ctx, c)
+	switch c.Business {
+	case "log_audit_business":
+		for k, v := range list {
+			if id, e := strconv.Atoi(v.Name); e == nil {
+				if t, ok := s.dao.GetLogInfo("log_audit", id); ok {
+					list[k].Name = t.Name
+				}
+			}
+		}
+	case "log_user_action_business":
+		for k, v := range list {
+			if id, e := strconv.Atoi(v.Name); e == nil {
+				if t, ok := s.dao.GetLogInfo("log_user_action", id); ok {
+					list[k].Name = t.Name
+				}
+			}
+		}
+	case "log_audit_uid", "log_user_action_uid":
+		uid := []string{}
+		for _, v := range list {
+			uid = append(uid, v.Name)
+		}
+		if data, err := s.dao.Unames(ctx, uid); err == nil {
+			for k, v := range list {
+				if t, ok := data.Data[v.Name]; ok {
+					list[k].Name = t
+				}
+			}
+		}
+	}
+	return
+}

app/admin/main/search/service/mng_v2.go

@@ -0,0 +1,199 @@
+package service
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"go-common/app/admin/main/search/model"
+	"go-common/library/ecode"
+)
+
+// BusinessAllV2 .
+func (s *Service) BusinessAllV2(c context.Context) (list []*model.GFBusiness, err error) {
+	return s.dao.BusinessAllV2(c)
+}
+
+// BusinessInfoV2 .
+func (s *Service) BusinessInfoV2(c context.Context, name string) (info *model.GFBusiness, err error) {
+	return s.dao.BusinessInfoV2(c, name)
+}
+
+// BusinessAdd .
+func (s *Service) BusinessAdd(c context.Context, pid int64, name, description string) (id int64, err error) {
+	return s.dao.BusinessIns(c, pid, name, description)
+}
+
+// BusinessUpdate .
+func (s *Service) BusinessUpdate(c context.Context, name, filed, value string) (id int64, err error) {
+	allowFields := []string{"data_conf", "index_conf", "business_conf", "description", "state"}
+	var allow bool
+	for _, v := range allowFields {
+		if v == filed {
+			allow = true
+		}
+	}
+	if !allow {
+		err = ecode.AccessDenied
+		return
+	}
+	return s.dao.BusinessUpdate(c, name, filed, value)
+}
+
+// AssetDBTables .
+func (s *Service) AssetDBTables(c context.Context) (list []*model.GFAsset, err error) {
+	return s.dao.AssetDBTables(c)
+}
+
+// AssetDBConnect .
+func (s *Service) AssetDBConnect(c context.Context, host, port, user, password string) (dbNames []string, err error) {
+	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/?charset=utf8mb4,utf8", user, password, host, port))
+	if err != nil {
+		return
+	}
+	defer db.Close()
+	rows, err := db.Query("show databases")
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	dbNames = make([]string, 0)
+	for rows.Next() {
+		var name string
+		if err = rows.Scan(&name); err != nil {
+			return
+		}
+		dbNames = append(dbNames, name)
+	}
+	return dbNames, rows.Err()
+}
+
+// AssetDBAdd .
+func (s *Service) AssetDBAdd(c context.Context, name, description, host, port, user, password string) (id int64, err error) {
+	dbnames, err := s.AssetDBConnect(c, host, port, user, password)
+	if err != nil {
+		return
+	}
+	var dbExist bool
+	for _, v := range dbnames {
+		if v == name {
+			dbExist = true
+		}
+	}
+	if !dbExist {
+		err = ecode.AccessDenied
+		return
+	}
+	dsn := fmt.Sprintf(model.DBDsnFormat, user, password, host, port, name)
+	return s.dao.AssetDBIns(c, name, description, dsn)
+}
+
+// AssetTableAdd .
+func (s *Service) AssetTableAdd(c context.Context, db, regex, fields, description string) (id int64, err error) {
+	name := db + "." + regex
+	return s.dao.AssetTableIns(c, name, db, regex, fields, description)
+}
+
+// UpdateAssetTable .
+func (s *Service) UpdateAssetTable(c context.Context, name, fields string) (id int64, err error) {
+	return s.dao.UpdateAssetTable(c, name, fields)
+}
+
+// AssetInfoV2 .
+func (s *Service) AssetInfoV2(c context.Context, name string) (info *model.GFAsset, err error) {
+	return s.dao.Asset(c, name)
+}
+
+// AssetShowTables .
+func (s *Service) AssetShowTables(c context.Context, dbName string) (tables []string, err error) {
+	asset, err := s.dao.Asset(c, dbName)
+	if err != nil {
+		return
+	}
+	db, err := sql.Open("mysql", asset.DSN)
+	if err != nil {
+		return
+	}
+	defer db.Close()
+	rows, err := db.Query("show tables")
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	tables = make([]string, 0)
+	for rows.Next() {
+		var name string
+		if err = rows.Scan(&name); err != nil {
+			return
+		}
+		tables = append(tables, name)
+	}
+	return tables, rows.Err()
+}
+
+// AssetTableFields .
+func (s *Service) AssetTableFields(c context.Context, dbName, regex string) (fs []*model.TableField, count int, err error) {
+	asset, err := s.dao.Asset(c, dbName)
+	if err != nil {
+		return
+	}
+	db, err := sql.Open("mysql", asset.DSN)
+	if err != nil {
+		return
+	}
+	defer db.Close()
+	regex = fmt.Sprintf("^%s$", regex)
+	rows, err := db.Query("SELECT COLUMN_NAME,DATA_TYPE,count(1) FROM information_schema.COLUMNS WHERE table_name REGEXP ? GROUP BY COLUMN_NAME,DATA_TYPE", regex)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	fs = make([]*model.TableField, 0)
+	for rows.Next() {
+		f := new(model.TableField)
+		if err = rows.Scan(&f.Name, &f.Type, &f.Count); err != nil {
+			return nil, 0, err
+		}
+		fs = append(fs, f)
+	}
+	if err = rows.Err(); err != nil {
+		return
+	}
+	if len(fs) == 0 {
+		err = ecode.NothingFound
+		return
+	}
+	for _, f := range fs {
+		if fs[0].Count != f.Count {
+			err = ecode.NothingFound
+			return
+		}
+		count = f.Count
+	}
+	row := db.QueryRow("SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE table_name REGEXP ? AND CONSTRAINT_NAME='PRIMARY' GROUP BY CONSTRAINT_NAME LIMIT 1", regex)
+	var primaryCo string
+	err = row.Scan(&primaryCo)
+	for k, v := range fs {
+		if v.Name == primaryCo {
+			fs[k].Primary = true
+		}
+	}
+	return fs, count, err
+}
+
+// ClusterOwners .
+func (s *Service) ClusterOwners() map[string]string {
+	clusters := s.c.Es
+	res := make(map[string]string)
+	res["default"] = "guanhuaxin,daizhichen,libingqi,zhapuyu"
+	for name, es := range clusters {
+		if es.Owner == "" {
+			continue
+		}
+		if es.Cluster != "" {
+			name = es.Cluster
+		}
+		res[name] = es.Owner
+	}
+	return res
+}

app/admin/main/search/service/mng_v2_test.go

@@ -0,0 +1,71 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_AssetDBConnect(t *testing.T) {
+	var (
+		c        = context.Background()
+		host     = "172.16.33.205"
+		port     = "3308"
+		user     = "test"
+		password = "test"
+	)
+
+	Convey("AssetDBConnect", t, WithService(func(s *Service) {
+		res, err := svr.AssetDBConnect(c, host, port, user, password)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	}))
+}
+
+func Test_AssetDBAdd(t *testing.T) {
+	var (
+		c           = context.Background()
+		name        = "bilibili_reply"
+		description = "评论数据库"
+		host        = "172.16.33.205"
+		port        = "3308"
+		user        = "test"
+		password    = "test"
+	)
+
+	Convey("AssetDBAdd", t, WithService(func(s *Service) {
+		res, err := svr.AssetDBAdd(c, name, description, host, port, user, password)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	}))
+}
+
+func Test_AssetTableFields(t *testing.T) {
+	var (
+		c     = context.Background()
+		db    = "bilibili_reply"
+		regex = "reply_([0-9]{1,3})"
+	)
+
+	Convey("AssetTableFields", t, WithService(func(s *Service) {
+		res, _, err := svr.AssetTableFields(c, db, regex)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	}))
+}
+
+func Test_BusinessUpdate(t *testing.T) {
+	var (
+		c     = context.Background()
+		name  = "dm"
+		field = "description"
+		value = "弹幕11"
+	)
+
+	Convey("BusinessUpdate", t, WithService(func(s *Service) {
+		res, err := svr.BusinessUpdate(c, name, field, value)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+	}))
+}

app/admin/main/search/service/query.go

@@ -0,0 +1,146 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/main/search/dao"
+	"go-common/app/admin/main/search/model"
+	"go-common/library/log"
+)
+
+func (s *Service) loadQueryConfproc() {
+	for {
+		if err := s.loadQueryConf(); err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func (s *Service) loadQueryConf() (err error) {
+	confs, err := s.dao.QueryConf(context.Background())
+	if err != nil {
+		return
+	}
+	if len(confs) > 0 {
+		s.queryConf = confs
+	}
+	return
+}
+
+// CheckQueryConf check query conf
+func (s *Service) CheckQueryConf(c context.Context, sp *model.QueryParams) (err error) {
+	app, ok := s.queryConf[sp.Business]
+	if app2, ok2 := model.QueryConf[sp.Business]; ok2 {
+		app = app2
+		ok = true
+	}
+	if !ok {
+		err = fmt.Errorf("sp.Business(%s) not exist in queryConf", sp.Business)
+		return
+	}
+	if app.ESCluster == "" {
+		err = fmt.Errorf("app(%+v) escluster is empty", app)
+		return
+	}
+	max := 1
+	if app.MaxIndicesNum > 0 {
+		max = app.MaxIndicesNum
+	}
+	indecies := strings.Split(sp.QueryBody.From, ",")
+	if len(indecies) == 0 {
+		err = fmt.Errorf("index name is required")
+		return
+	}
+	if len(indecies) > max {
+		err = fmt.Errorf("too many indecies(%v)", indecies)
+		return
+	}
+	for _, index := range indecies {
+		if !strings.Contains(index, app.IndexPrefix) {
+			err = fmt.Errorf("invalid index name(%s)", index)
+			return
+		}
+	}
+	sp.AppIDConf = app
+	return
+}
+
+// QueryBasic .
+func (s *Service) QueryBasic(c context.Context, sp *model.QueryParams) (res *model.QueryResult, debug *model.QueryDebugResult, err error) {
+	switch sp.Business {
+	case "log_audit":
+		t := strings.Split(sp.QueryBody.From, "_")
+		if len(t) > 2 {
+			logID, err := strconv.Atoi(t[2])
+			if err != nil {
+				log.Error("strconv.Atoi(%s) error(%v)", t[2], err)
+			}
+			logBusiness, ok := s.dao.GetLogInfo(sp.Business, logID)
+			if ok {
+				sp.AppIDConf.ESCluster = logBusiness.IndexCluster
+			}
+		}
+	case "log_user_action":
+		t := strings.Split(sp.QueryBody.From, "_")
+		if len(t) > 3 {
+			logID, err := strconv.Atoi(t[3])
+			if err != nil {
+				log.Error("strconv.Atoi(%s) error(%v)", t[3], err)
+			}
+			logBusiness, ok := s.dao.GetLogInfo(sp.Business, logID)
+			if ok {
+				sp.AppIDConf.ESCluster = logBusiness.IndexCluster
+			}
+		}
+	}
+	bQuery, qbDebug := s.dao.QueryBasic(c, sp)
+	if res, debug, err = s.dao.QueryResult(c, bQuery, sp, qbDebug); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryBasic(%v) error(%v)", sp, err)
+	}
+	return
+}
+
+// QueryExtra .
+func (s *Service) QueryExtra(c context.Context, sp *model.QueryParams) (res *model.QueryResult, debug *model.QueryDebugResult, err error) {
+	switch sp.Business {
+	case "archive_video_score":
+		if res, debug, err = s.dao.ArchiveVideoScore(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra(%v) error(%v)", sp, err)
+		}
+	case "archive_score":
+		if res, debug, err = s.dao.ArchiveScore(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra(%v) error(%v)", sp, err)
+		}
+	case "task_qa_random":
+		if res, debug, err = s.dao.TaskQaRandom(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.TaskQaRandom(%v) error(%v)", sp, err)
+		}
+	case "esports_contests_date":
+		if res, debug, err = s.dao.EsportsContestsDate(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.EsportsContestsDate(%v) error(%v)", sp, err)
+		}
+	case "creative_archive_search":
+		if res, debug, err = s.dao.CreativeArchiveSearch(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.CreativeArchiveSearch(%v) error(%v)", sp, err)
+		}
+	case "creative_archive_staff":
+		if res, debug, err = s.dao.CreativeArchiveStaff(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.CreativeArchiveStaff(%v) error(%v)", sp, err)
+		}
+	case "creative_archive_apply":
+		if res, debug, err = s.dao.CreativeArchiveApply(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.CreativeArchiveApply(%v) error(%v)", sp, err)
+		}
+	case "dm_history":
+		if res, debug, err = s.dao.Scroll(c, sp); err != nil {
+			dao.PromError(fmt.Sprintf("es:%s 搜索失败", sp.Business), "s.dao.QueryExtra.Scroll(%v) error(%v)", sp, err)
+		}
+	}
+	return
+}

app/admin/main/search/service/service.go

@@ -0,0 +1,32 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/main/search/conf"
+	"go-common/app/admin/main/search/dao"
+	"go-common/app/admin/main/search/model"
+)
+
+// Service struct of service.
+type Service struct {
+	c         *conf.Config
+	dao       *dao.Dao
+	queryConf map[string]*model.QueryConfDetail
+}
+
+// New create service instance and return.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		dao: dao.New(c),
+	}
+	s.loadQueryConf()
+	go s.loadQueryConfproc()
+	return
+}
+
+// Ping .
+func (s *Service) Ping(c context.Context) error {
+	return s.dao.Ping(c)
+}

app/admin/main/search/service/service_test.go

@@ -0,0 +1,35 @@
+package service
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/admin/main/search/conf"
+)
+
+var (
+	svr *Service
+)
+
+func TestMain(m *testing.M) {
+	var (
+		err error
+	)
+	dir, _ := filepath.Abs("../cmd/search-admin-test.toml")
+	if err = flag.Set("conf", dir); err != nil {
+		panic(err)
+	}
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	svr = New(conf.Conf)
+	os.Exit(m.Run())
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		f(svr)
+	}
+}

app/admin/main/search/service/update.go

@@ -0,0 +1,33 @@
+package service
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/admin/main/search/dao"
+	"go-common/library/ecode"
+)
+
+// Update update some indices.
+func (s *Service) Update(c context.Context, esName string, bulkData []dao.BulkItem) (err error) {
+	if err = s.dao.UpdateBulk(c, esName, bulkData); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 更新失败", esName), "s.dao.updateBulk error(%v) ", err)
+		err = ecode.SearchUpdateIndexFailed
+	}
+	return
+}
+
+// MapUpdate map update.
+func (s *Service) MapUpdate(c context.Context, p []dao.BulkMapItem) (err error) {
+	err = s.dao.UpdateMapBulk(c, "ssd_archive", p)
+	return
+}
+
+// Index .
+func (s *Service) Index(c context.Context, esName string, bulkData []dao.BulkItem) (err error) {
+	if err = s.dao.BulkIndex(c, esName, bulkData); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 写入失败", esName), "s.dao.BulkIndex error(%v) ", err)
+		err = ecode.SearchUpdateIndexFailed
+	}
+	return
+}

app/admin/main/search/service/update_test.go

@@ -0,0 +1 @@
+package service

app/admin/main/search/service/upsert.go

@@ -0,0 +1,51 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go-common/app/admin/main/search/dao"
+	"go-common/app/admin/main/search/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// Upsert upsert docs.
+func (s *Service) Upsert(c context.Context, up *model.UpsertParams, dataBody map[string][]model.MapData) (err error) {
+	app, ok := s.queryConf[up.Business]
+	if app2, ok2 := model.QueryConf[up.Business]; ok2 {
+		app = app2
+		ok = true
+	}
+	if !ok {
+		err = fmt.Errorf("up.Business(%s) not exists in queryConf", up.Business)
+		return
+	}
+	if app.ESCluster == "" {
+		err = fmt.Errorf("app(%+v) escluster is empty", app)
+		return
+	}
+	// dataBody to upsertBody
+	up.UpsertBody = []model.UpsertBody{}
+	for indexName, docs := range dataBody {
+		if !strings.Contains(indexName, app.IndexPrefix) {
+			log.Error("invalid indexName (%s)", indexName)
+			continue
+		}
+		for _, doc := range docs {
+			indexID := doc.StrID(app.IndexID)
+			//TODO 提前告知base不对
+			if indexID == "" {
+				continue
+			}
+			upsert := model.UpsertBody{IndexName: indexName, IndexType: app.IndexType, IndexID: indexID, Doc: doc}
+			up.UpsertBody = append(up.UpsertBody, upsert)
+		}
+	}
+	if err = s.dao.UpsertBulk(c, app.ESCluster, up); err != nil {
+		dao.PromError(fmt.Sprintf("es:%s 更新失败", app.ESCluster), "s.dao.UpsertBulk error(%v) ", err)
+		err = ecode.SearchUpdateIndexFailed
+	}
+	return
+}