Create & Init Project...

app/job/main/search/service/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "all_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/search/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "all.go",
+        "incr.go",
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/search/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/search/conf:go_default_library",
+        "//app/job/main/search/dao:go_default_library",
+        "//app/job/main/search/dao/base:go_default_library",
+        "//app/job/main/search/model:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/search/service/all.go

@@ -0,0 +1,68 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/search/model"
+	"go-common/library/log"
+)
+
+// all all data to es
+func (s *Service) all(c context.Context, appid string, writeEntityIndex bool) {
+	var stat = new(model.Stat)
+	app := s.base.D.AppPool[appid]
+	app.InitIndex(c)
+	app.InitOffset(c)
+	//app.Offset(c)
+	app.Sleep(c)
+	for {
+		start := 0
+		length, err := app.AllMessages(c)
+		if err != nil {
+			log.Error("AllMessages error(%v)", err)
+			app.Sleep(c)
+			continue
+		}
+		for {
+			end := start + _bulkSize
+			diff := length - start
+			if diff > _bulkSize {
+				if err := app.BulkIndex(c, start, end, writeEntityIndex); err != nil {
+					log.Error("es:BulkIndex error(%v)", err)
+					app.Sleep(c)
+					continue
+				}
+				start = end
+			} else if diff > 0 && diff <= _bulkSize {
+				if err := app.BulkIndex(c, start, length, writeEntityIndex); err != nil {
+					log.Error("BulkIndex error(%v)", err)
+					app.Sleep(c)
+					continue
+				}
+				if err := app.Commit(c); err != nil {
+					log.Error("UpdateOffsetID error(%v)", err)
+					app.Sleep(c)
+					continue
+				}
+				app.Sleep(c)
+				break
+			} else {
+				app.Sleep(c)
+				break
+			}
+		}
+		stat.Counts += length
+		s.updateStat(appid, stat)
+		if length < app.Size(c) {
+			switch appid {
+			case "pgc_media", "esports", "esports_contests", "academy_archive", "esports_fav_all", "activity_all":
+				app.SetRecover(c, 0, "", 0)
+				app.Sleep(c)
+				continue
+			}
+			break
+		}
+		app.Sleep(c)
+	}
+	log.Info("appid:%s, all data to es successful!!!", appid)
+}

app/job/main/search/service/all_test.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Stat(t *testing.T) {
+
+	var (
+		err error
+		c   = context.TODO()
+	)
+
+	Convey("Stat", t, WithService(func(s *Service) {
+		_, err = s.Stat(c, "music_songs")
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/search/service/incr.go

@@ -0,0 +1,39 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/search/dao"
+)
+
+// incr increment data
+func (s *Service) incr(c context.Context, app dao.App) {
+	app.InitIndex(c)
+	app.Sleep(c)
+	app.Offset(c)
+Loop:
+	for {
+		length, err := app.IncrMessages(c)
+		if err != nil {
+			s.base.D.PromError("IncrMessages", "IncrMessages error(%v)", err)
+			app.Sleep(c)
+			continue
+		}
+		for start := 0; start < length; start += _bulkSize {
+			diff := length - start
+			if diff > _bulkSize {
+				diff = _bulkSize
+			}
+			if err := app.BulkIndex(c, start, start+diff, false); err != nil {
+				s.base.D.PromError("BulkIndex", "BulkIndex error(%v)", err)
+				time.Sleep(120 * time.Second) // 使databus readTimeout重新消费
+				continue Loop
+			}
+		}
+		if err := app.Commit(c); err != nil {
+			s.base.D.PromError("UpdateOffsetID", "UpdateOffsetID error(%v)", err)
+		}
+		app.Sleep(c)
+	}
+}

app/job/main/search/service/service.go

@@ -0,0 +1,88 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/search/conf"
+	"go-common/app/job/main/search/dao/base"
+	"go-common/app/job/main/search/model"
+	"go-common/library/log"
+)
+
+var (
+	ctx = context.TODO()
+)
+
+const (
+	_bulkSize = 5000
+)
+
+// Service .
+type Service struct {
+	c *conf.Config
+	// base
+	base *base.Base
+	//mutex
+	mutex *sync.RWMutex
+	// stats
+	stats map[string]*model.Stat
+}
+
+// New .
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:     c,
+		base:  base.NewBase(c),
+		mutex: new(sync.RWMutex),
+		stats: make(map[string]*model.Stat),
+	}
+	s.incrproc()
+	return
+}
+
+// incrproc incr data
+func (s *Service) incrproc() {
+	for appid, e := range s.base.D.AppPool {
+		if !s.base.D.BusinessPool[appid].IncrOpen {
+			continue
+		}
+		if e.Business() == s.c.Business.Env && !s.c.Business.Index {
+			go s.incr(ctx, e)
+		}
+	}
+}
+
+// Close .
+func (s *Service) Close() {
+	s.base.D.Close()
+}
+
+// Ping .
+func (s *Service) Ping(c context.Context) error {
+	return s.base.D.Ping(c)
+}
+
+// HTTPAction http action
+func (s *Service) HTTPAction(ctx context.Context, appid, action string, recoverID int64, writeEntityIndex bool) (msg string, err error) {
+	switch action {
+	case "repair":
+	case "all":
+		if _, ok := s.base.D.AppPool[appid]; !ok {
+			msg = "appid不在appPool中"
+			log.Error("AppPool inclueds (%v)", s.base.D.AppPool)
+			return
+		}
+		s.base.D.SetRecover(ctx, appid, recoverID, "", 0)
+		go s.all(context.Background(), appid, writeEntityIndex)
+	default:
+		return
+	}
+	return
+}
+
+// Stat .
+func (s *Service) Stat(ctx context.Context, appid string) (st *model.Stat, err error) {
+	st = s.stat(appid)
+	return
+}

app/job/main/search/service/service_test.go

@@ -0,0 +1,19 @@
+package service
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/search/conf"
+)
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../goconvey.toml")
+		flag.Set("conf", dir)
+		conf.Init()
+		s := New(conf.Conf)
+		// s.dao = dao.New(conf.Conf)
+		f(s)
+	}
+}

app/job/main/search/service/stat.go

@@ -0,0 +1,20 @@
+package service
+
+import (
+	"go-common/app/job/main/search/model"
+)
+
+// stat get stat
+func (s *Service) stat(appid string) (st *model.Stat) {
+	s.mutex.RLock()
+	st = s.stats[appid]
+	s.mutex.RUnlock()
+	return
+}
+
+// updateStat update stat
+func (s *Service) updateStat(appid string, st *model.Stat) {
+	s.mutex.Lock()
+	s.stats[appid] = st
+	s.mutex.Unlock()
+}