Create & Init Project...

app/job/main/search/model/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config_asset.go",
+        "config_attrs.go",
+        "config_business.go",
+        "config_offset.go",
+        "es.go",
+        "map_data.go",
+        "model.go",
+    ],
+    importpath = "go-common/app/job/main/search/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/search/model/config_asset.go

@@ -0,0 +1,9 @@
+package model
+
+// SQLAsset .
+type SQLAsset struct {
+	ID   int64
+	Name string
+	Type int
+	Src  string
+}

app/job/main/search/model/config_attrs.go

@@ -0,0 +1,131 @@
+package model
+
+// SQLAttrs get attrs from db
+type SQLAttrs struct {
+	AppID            string
+	DBName           string
+	ESName           string
+	DtbName          string
+	TablePrefix      string
+	TableFormat      string
+	IndexAliasPrefix string //写和读数据时，指向的索引。是别名索引
+	IndexVersion     string //创建索引时，指向的索引。这里是实体索引版本号
+	IndexFormat      string
+	IndexType        string
+	IndexID          string
+	IndexMapping     string
+	DataIndexSuffix  string
+	DataExtraInfo    string
+	ReviewNum        int64
+	ReviewTime       int64
+	Sleep            float64
+	Size             int
+	Business         string
+	DataFields       string
+	SQLByID          string
+	SQLByMTime       string
+	SQLByIDMTime     string
+	DatabusInfo      string
+	DatabusIndexID   string
+}
+
+// Attrs parse AppAttrs
+type Attrs struct {
+	Business   string
+	AppID      string
+	DBName     string
+	ESName     string
+	DtbName    string
+	Table      *AttrTable
+	Index      *AttrIndex
+	DataSQL    *AttrDataSQL
+	DataExtras []AttrDataExtra //appID需要关联其他库的数据
+	Databus    *AttrDatabus
+	Other      *AttrOther
+}
+
+// AttrTable .
+type AttrTable struct {
+	TablePrefix string
+	TableFormat string
+	TableSplit  string
+	TableFrom   int
+	TableTo     int
+	TableZero   string
+	TableFixed  bool
+}
+
+// AttrIndex .
+type AttrIndex struct {
+	IndexAliasPrefix  string //写和读数据时，指向的索引。是别名索引
+	IndexEntityPrefix string //创建索引时，指向的索引。是实体索引名
+	IndexFormat       string
+	IndexSplit        string
+	IndexFrom         int
+	IndexTo           int
+	IndexType         string
+	IndexID           string
+	IndexMapping      string
+	IndexZero         string
+	IndexFixed        bool
+}
+
+// AttrDataSQL .
+type AttrDataSQL struct {
+	DataIndexSuffix       string //索引数据归属
+	DataFields            string
+	DataFieldsV2          map[string]AttrDataFields //存放json转换得到的data_fields字段信息, 替换老的DataFields
+	DataIndexFields       []string                  //来自DataFields左数第一位
+	DataIndexRemoveFields []string                  //ES需要移除的字段
+	DataIndexFormatFields map[string]string         //ES每个字段的格式化，如int64,time,int等
+	DataDtbFields         map[string][]string       //databus的字段对应的es字段, TODO 改成 map[string]map[string]bool 或 map[string][]string，应对一个数据库字段用在多个es字段
+	DataExtraInfo         string
+	SQLFields             string //来自DataFields左数第二位，含表名和字段的alias以及mysql函数等其他表达式
+	SQLByID               string //因为有left join的缘故，顾提供完整sql（抛除字段部分，下同）
+	SQLByMTime            string
+	SQLByIDMTime          string
+}
+
+// AttrDataFields .
+type AttrDataFields struct {
+	ESField string `json:"es"`
+	Field   string `json:"field"`
+	SQL     string `json:"sql"`
+	Expect  string `json:"expect"`
+	Stored  string `json:"stored"`
+	InDtb   string `json:"in_dtb"`
+}
+
+// AttrDataExtra .
+type AttrDataExtra struct {
+	Type         string            `json:"type"`
+	Tag          string            `json:"tag"`
+	Condition    map[string]string `json:"condition"`
+	SliceField   string            `json:"slice_field"` // 逗号分隔，支持多个字段
+	DBName       string            `json:"dbname"`
+	Table        string            `json:"table"`
+	TableFormat  string            `json:"table_format"`
+	InField      string            `json:"in_field"`
+	FieldsStr    string            `json:"fields_str"`
+	Fields       []string          `json:"fields"`
+	RemoveFields []string          `json:"remove_fields"`
+	SQL          string            `json:"sql"`
+}
+
+// AttrDatabus .
+type AttrDatabus struct {
+	DatabusInfo string
+	Ticker      int    // 定时时间(毫秒)
+	AggCount    int    // 聚合数量
+	Databus     string // databus Map key
+	PrimaryID   string // 主表索引id
+	RelatedID   string // 关联表索引id
+}
+
+// AttrOther .
+type AttrOther struct {
+	ReviewNum  int64
+	ReviewTime int64
+	Sleep      float64
+	Size       int
+}

app/job/main/search/model/config_business.go

@@ -0,0 +1,38 @@
+package model
+
+import (
+	"go-common/library/database/sql"
+)
+
+// SQLBusiness single table offset
+type SQLBusiness struct {
+	Business string
+	AppIds   string
+	AssetDB  string
+	AssetES  string
+	AssetDtb string
+}
+
+// Bsn single table offset
+type Bsn struct {
+	Business string
+	AppInfo  []BsnAppInfo
+	AssetDB  map[string]*sql.Config
+	AssetES  []string
+	//AssetDtb []AssetDtb
+}
+
+// BsnAppInfo .
+type BsnAppInfo struct {
+	AppID       string `json:"appid"`
+	IncrWay     string `json:"incr_way"`
+	IncrOpen    bool   `json:"incr_open"`
+	RecoverLock bool
+}
+
+// AssetDtb .
+// type AssetDtb struct {
+// 	dtb   map[string]*databus.Config
+// 	size  int
+// 	sleep int
+// }

app/job/main/search/model/config_offset.go

@@ -0,0 +1,128 @@
+package model
+
+import (
+	"time"
+)
+
+// LoopOffset single table offset
+type LoopOffset struct {
+	IsLoop          bool
+	OffsetID        int64
+	OffsetTime      string
+	TempOffsetID    int64
+	TempOffsetTime  string
+	RecoverID       int64
+	RecoverTime     string
+	TempRecoverID   int64
+	TempRecoverTime string
+	ReviewID        int64
+	ReviewTime      int64
+}
+
+// SetLoop .
+func (lo *LoopOffset) SetLoop(isLoop bool) {
+	lo.IsLoop = isLoop
+}
+
+// SetReview .
+func (lo *LoopOffset) SetReview(rid int64, rtime int64) {
+	lo.ReviewID = rid
+	lo.ReviewTime = rtime
+}
+
+// SetOffset .
+func (lo *LoopOffset) SetOffset(id int64, t string) {
+	if id != 0 {
+		lo.OffsetID = id
+	}
+	if t != "" {
+		lo.OffsetTime = t
+		if !lo.IsLoop {
+			if local, err := time.LoadLocation("Local"); err == nil {
+				if t2, e := time.ParseInLocation("2006-01-02 15:04:05", t, local); e == nil && t2.Unix()-lo.ReviewTime > 0 {
+					lo.OffsetTime = time.Unix(t2.Unix()-lo.ReviewTime, 0).Format("2006-01-02 15:04:05") //往前推ReviewTime
+				}
+			}
+		}
+	}
+}
+
+// SetTempOffset .
+func (lo *LoopOffset) SetTempOffset(id int64, time string) {
+	if id != 0 {
+		lo.TempOffsetID = id
+	}
+	if time != "" {
+		lo.TempOffsetTime = time
+	}
+}
+
+// SetRecoverOffset .
+func (lo *LoopOffset) SetRecoverOffset(recoverID int64, recoverTime string) {
+	if recoverID >= 0 {
+		lo.RecoverID = recoverID
+	}
+	if recoverTime != "" {
+		lo.RecoverTime = recoverTime
+	}
+}
+
+// SetRecoverTempOffset .
+func (lo *LoopOffset) SetRecoverTempOffset(recoverID int64, recoverTime string) {
+	if recoverID >= 0 {
+		lo.TempRecoverID = recoverID
+	}
+	if recoverTime != "" {
+		lo.TempRecoverTime = recoverTime
+	}
+}
+
+// LoopOffsets more tables offset
+type LoopOffsets map[int]*LoopOffset
+
+// SetLoops .
+func (los LoopOffsets) SetLoops(i int, isLoop bool) {
+	if _, ok := los[i]; ok {
+		los[i].IsLoop = isLoop
+	}
+}
+
+// SetOffsets .
+func (los LoopOffsets) SetOffsets(i int, id int64, time string) {
+	if id != 0 {
+		los[i].OffsetID = id
+	}
+	if time != "" {
+		los[i].OffsetTime = time
+	}
+}
+
+// SetTempOffsets .
+func (los LoopOffsets) SetTempOffsets(i int, id int64, time string) {
+	if id != 0 {
+		los[i].TempOffsetID = id
+	}
+	if time != "" {
+		los[i].TempOffsetTime = time
+	}
+}
+
+// SetRecoverOffsets .
+func (los LoopOffsets) SetRecoverOffsets(i int, recoverID int64, recoverTime string) {
+	if recoverID >= 0 {
+		los[i].RecoverID = recoverID
+	}
+	if recoverTime != "" {
+		los[i].RecoverTime = recoverTime
+	}
+}
+
+// SetRecoverTempOffsets .
+func (los LoopOffsets) SetRecoverTempOffsets(i int, recoverID int64, recoverTime string) {
+	if recoverID >= 0 {
+		los[i].TempRecoverID = recoverID
+	}
+	if recoverTime != "" {
+		los[i].TempRecoverTime = recoverTime
+	}
+}

app/job/main/search/model/es.go

@@ -0,0 +1,35 @@
+package model
+
+import (
+	"time"
+
+	xtime "go-common/library/time"
+)
+
+// ES .
+type ES struct {
+	Addr string
+}
+
+// Offset .
+type Offset struct {
+	OffID      int64
+	OffTime    xtime.Time
+	ReviewID   int64
+	ReviewTime int64
+}
+
+// OffsetID .
+func (o *Offset) OffsetID() int64 {
+	return o.OffID - o.ReviewID
+}
+
+// OffsetTime .
+func (o *Offset) OffsetTime() string {
+	return time.Unix(int64(o.OffTime), 0).Format("2006-01-02 15:04:05")
+}
+
+// var .
+var (
+	ExistsAction = map[string]bool{"repair": true, "all": true}
+)

app/job/main/search/model/map_data.go

@@ -0,0 +1,222 @@
+package model
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/library/log"
+)
+
+// MapData .
+type MapData map[string]interface{}
+
+// StrID .
+func (m MapData) StrID(attrs *Attrs) (string, bool) {
+	if attrs.Index.IndexID == "UUID" {
+		return "", true
+	}
+	var data []interface{}
+	arr := strings.Split(attrs.Index.IndexID, ",")
+	arrLen := len(arr)
+	if arrLen >= 2 {
+		for _, v := range arr[1:] {
+			if item, ok := m[v].(*interface{}); ok {
+				data = append(data, item)
+				continue
+			}
+			if item, ok := m[v].(interface{}); ok {
+				data = append(data, item)
+				continue
+			}
+			log.Error("model.map_data.StrID err (%v)", v)
+			return "", false
+		}
+		return fmt.Sprintf(arr[0], data...), true
+	}
+	return "", false
+}
+
+// Index .
+func (m MapData) Index(attrs *Attrs) (indexAliasName, indexEntityName string) {
+	switch attrs.Index.IndexSplit {
+	case "single":
+		indexAliasName = attrs.Index.IndexAliasPrefix
+		indexEntityName = attrs.Index.IndexEntityPrefix
+	case "int":
+		if attrs.DataSQL.DataIndexSuffix != "" {
+			s := strings.Split(attrs.DataSQL.DataIndexSuffix, ";")
+			v := strings.Split(s[1], ":")
+			if id, ok := m[v[0]].(*interface{}); ok {
+				// indexAliasName = fmt.Sprintf("%s%d", attrs.Index.IndexAliasPrefix, (*id).(int64)%100) // mod
+				divisor, _ := strconv.ParseInt(v[2], 10, 64)
+				indexAliasName = fmt.Sprintf("%s"+s[0], attrs.Index.IndexAliasPrefix, (*id).(int64)%divisor)
+				indexEntityName = fmt.Sprintf("%s"+s[0], attrs.Index.IndexEntityPrefix, (*id).(int64)%divisor)
+			}
+			if id, ok := m[v[0]].(interface{}); ok {
+				divisor, _ := strconv.ParseInt(v[2], 10, 64)
+				indexAliasName = fmt.Sprintf("%s"+s[0], attrs.Index.IndexAliasPrefix, id.(int64)%divisor)
+				indexEntityName = fmt.Sprintf("%s"+s[0], attrs.Index.IndexEntityPrefix, id.(int64)%divisor)
+			}
+		}
+	}
+	//fmt.Println("indexname", indexAliasName, indexEntityName)
+	return
+}
+
+// DtbIndex .
+// func (m MapData) DtbIndex(attrs *Attrs) (indexName string) {
+// 	if attrs.Index.IndexZero == "0" {
+// 		indexName = attrs.Index.IndexAliasPrefix
+// 		return
+// 	}
+// 	if attrs.DataSQL.DataIndexSuffix != "" {
+// 		s := strings.Split(attrs.DataSQL.DataIndexSuffix, ";")
+// 		v := strings.Split(s[1], ":")
+// 		divisor, _ := strconv.ParseInt(v[2], 10, 64)
+// 		indexName = fmt.Sprintf("%s"+s[0], attrs.Index.IndexAliasPrefix, int64(m[v[0]].(float64))%divisor)
+// 	}
+// 	return
+// }
+
+// PrimaryID .
+func (m MapData) PrimaryID() int64 {
+	if m["_id"] != nil {
+		if id, ok := m["_id"].(*interface{}); ok {
+			return (*id).(int64)
+		}
+	}
+	return 0
+}
+
+// StrMTime .
+func (m MapData) StrMTime() string {
+	if m["_mtime"] != nil {
+		if mtime, ok := m["_mtime"].(*interface{}); ok {
+			return (*mtime).(time.Time).Format("2006-01-02 15:04:05")
+		} else if mtime, ok := m["_mtime"].(string); ok {
+			return mtime
+		}
+	}
+	return ""
+}
+
+// StrCTime .
+func (m MapData) StrCTime() string {
+	if m["ctime"] != nil {
+		if ctime, ok := m["ctime"].(*interface{}); ok {
+			return (*ctime).(time.Time).Format("2006-01-02")
+		} else if ctime, ok := m["ctime"].(string); ok {
+			return ctime
+		}
+	}
+	return ""
+}
+
+// InetNtoA int64 to string ip.
+func (m MapData) InetNtoA(ip int64) string {
+	return fmt.Sprintf("%d.%d.%d.%d",
+		byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
+}
+
+// TransData transfer address into value
+func (m MapData) TransData(attr *Attrs) {
+	for k, v := range m {
+		// transfer automaticlly
+		if v2, ok := v.(*interface{}); ok {
+			switch (*v2).(type) {
+			case time.Time:
+				m[k] = (*v2).(time.Time).Format("2006-01-02 15:04:05")
+			case []uint, []uint8, []uint16, []uint32, []uint64:
+				m[k] = string((*v2).([]byte))
+			case int, int8, int16, int32, int64: // 一定要，用于extra_data查询
+				m[k] = (*v2).(int64)
+			case nil:
+				m[k] = int64(0) //给个默认值，当查到为null时
+			default:
+				// other types
+			}
+		}
+		// transfer again by custom
+		if t, ok := attr.DataSQL.DataIndexFormatFields[k]; ok {
+			if v3, ok := v.(*interface{}); ok {
+				switch t {
+				case "ip":
+					if *v3 == nil {
+						*v3 = int64(0)
+					}
+					ipFormat := m.InetNtoA((*v3).(int64))
+					m[k+"_format"] = ipFormat
+				case "arr":
+					var arr []int
+					binaryAttributes := strconv.FormatInt((*v3).(int64), 2)
+					for i := len(binaryAttributes) - 1; i >= 0; i-- {
+						b := fmt.Sprintf("%c", binaryAttributes[i])
+						if b == "1" {
+							arr = append(arr, len(binaryAttributes)-i)
+						}
+					}
+					m[k+"_format"] = arr
+				case "bin":
+					var arr []int
+					binaryAttributes := strconv.FormatInt((*v3).(int64), 2)
+					for i := len(binaryAttributes) - 1; i >= 0; i-- {
+						b := fmt.Sprintf("%c", binaryAttributes[i])
+						if b == "1" {
+							arr = append(arr, len(binaryAttributes)-i)
+						}
+					}
+					m[k] = arr
+				case "array_json":
+					var arr []int64
+					arr = []int64{}
+					json.Unmarshal([]byte((*v3).([]uint8)), &arr) //如果不是json就是空数组
+					// println(len(arr))
+					m[k] = arr
+				case "day":
+					m[k] = (*v3).(time.Time).Format("2006-01-02")
+				case "workflow":
+					delete(m, k)
+				default:
+					// other types
+				}
+			}
+		}
+	}
+}
+
+// TransDtb transfer databus fields into es fields
+func (m MapData) TransDtb(attr *Attrs) {
+	// TODO	注释要打开，不然无法移除不要的dtb字段
+	// for k := range m {
+	// 	if _, ok := attr.DataSQL.DataDtbFields[k]; !ok {
+	// 		if k == "index_name" {
+	// 			continue
+	// 		}
+	// 		delete(m, k)
+	// 	}
+	// }
+	res := map[string]interface{}{}
+	for k, dv := range attr.DataSQL.DataDtbFields {
+		for _, dk := range dv {
+			if v, ok := m[k]; ok {
+				switch v.(type) {
+				case float64:
+					res[dk] = int64(v.(float64))
+				default:
+					res[dk] = v
+				}
+			}
+		}
+	}
+	for k := range res {
+		m[k] = res[k]
+	}
+	id, okID := attr.DataSQL.DataFieldsV2["_id"]
+	key, okKey := attr.DataSQL.DataDtbFields[id.Field]
+	if len(key) >= 1 && okID && okKey {
+		m["_id"] = m[key[0]]
+	}
+}

app/job/main/search/model/model.go

@@ -0,0 +1,46 @@
+package model
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// Stat all data statistics
+type Stat struct {
+	Counts int `json:"counts"`
+}
+
+// Message canal binlog message.
+type Message struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// DatabusPool poll from db.
+var DatabusPool = []string{"dm", "dmreport_new"}
+
+// JSONTime .
+type JSONTime time.Time
+
+// UnmarshalJSON .
+func (p *JSONTime) UnmarshalJSON(data []byte) error {
+	local, err := time.ParseInLocation(`"2006-01-02 15:04:05"`, string(data), time.Local)
+	*p = JSONTime(local)
+	return err
+}
+
+// MarshalJSON .
+func (p JSONTime) MarshalJSON() ([]byte, error) {
+	data := make([]byte, 0)
+	data = append(data, '"')
+	data = time.Time(p).AppendFormat(data, "2006-01-02 15:04:05")
+	data = append(data, '"')
+	return data, nil
+}
+
+// String .
+func (p JSONTime) String() string {
+	return time.Time(p).Format("2006-01-02 15:04:05")
+}