Create & Init Project...

app/job/openplatform/open-sug/service/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "databus.go",
+        "service.go",
+        "sug.go",
+    ],
+    importpath = "go-common/app/job/openplatform/open-sug/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/openplatform/open-sug/conf:go_default_library",
+        "//app/job/openplatform/open-sug/dao:go_default_library",
+        "//app/job/openplatform/open-sug/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/open-sug/service/databus.go

@@ -0,0 +1,87 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"go-common/app/job/openplatform/open-sug/conf"
+	"go-common/app/job/openplatform/open-sug/model"
+	"go-common/library/log"
+)
+
+func (s *Service) pgcConsumePROC() {
+	var (
+		msgs = s.pgcSub.Messages()
+		err  error
+	)
+	for {
+		msg, ok := <-msgs
+		if !ok {
+			log.Error("pgc databus Consumer exit")
+			return
+		}
+		s.pgcMsgCnt++
+		msg.Commit()
+		log.Info("message commit key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
+		m := &model.Message{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		func(m *model.Message) {
+			s.wg.Add(1)
+			//defer s.wg.Done()
+			go s.subproc()
+
+			if m.Table == "t_chn_season2" || m.Table == "t_jp_season2" {
+				s.seasonUpdate(m.Action, m.New, m.Old)
+				s.seasonMsgCnt++
+			}
+		}(m)
+		log.Info("pgcConsumeproc table:%s key:%s partition:%d offset:%d", m.Table, msg.Key, msg.Partition, msg.Offset)
+	}
+}
+
+func (s *Service) seasonUpdate(action string, n, o json.RawMessage) {
+	newSeason := new(model.Season)
+	if err := json.Unmarshal(n, newSeason); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", n, err)
+		return
+	}
+	oldSeason := new(model.Season)
+	if err := json.Unmarshal(o, oldSeason); err != nil {
+		log.Error("json.Unmarshal(%s) (%s)error(%v)", string(o), o, err)
+		return
+	}
+	if action == "insert" || action == "update" {
+		if newSeason.FieldDiff(oldSeason) {
+			s.dao.Index(
+				context.TODO(),
+				s.envIndex(s.c.ElasticSearch.Season.Index),
+				s.c.ElasticSearch.Season.Type,
+				strconv.Itoa(newSeason.ID),
+				newSeason.EsFormat(),
+			)
+		}
+	}
+}
+
+func (s *Service) envIndex(index string) string {
+	return fmt.Sprintf("%s_%s", s.c.Env, index)
+}
+
+func (s *Service) existsOrCreate(c conf.EsIndex) {
+	if s.dao.IndexExists(context.TODO(), s.envIndex(c.Index)) {
+		log.Info("索引(%s)已存在", s.envIndex(c.Index))
+		return
+	}
+	if !s.dao.CreateIndex(context.TODO(), s.envIndex(c.Index), c.Mapping) {
+		panic("创建索引失败")
+	}
+}
+
+func (s *Service) subproc() {
+	defer s.wg.Done()
+}

app/job/openplatform/open-sug/service/service.go

@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/openplatform/open-sug/conf"
+	"go-common/app/job/openplatform/open-sug/dao"
+	"go-common/library/queue/databus"
+	"sync"
+	"time"
+)
+
+// Service struct
+type Service struct {
+	c            *conf.Config
+	dao          *dao.Dao
+	pgcSub       *databus.Databus
+	pgcMsgCnt    int64
+	wg           *sync.WaitGroup
+	seasonMsgCnt int64
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:            c,
+		dao:          dao.New(c),
+		wg:           new(sync.WaitGroup),
+		pgcSub:       databus.New(c.PgcSub),
+		pgcMsgCnt:    0,
+		seasonMsgCnt: 0,
+	}
+	s.existsOrCreate(c.ElasticSearch.Season)
+	go s.pgcConsumePROC()
+	go s.fetchData()
+
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}
+
+func (s *Service) fetchData() {
+	for {
+		s.buildData()
+		s.Refresh()
+		s.Filter()
+		time.Sleep(time.Hour * 24)
+	}
+}
+
+// Refresh ...
+func (s *Service) Refresh() {
+	s.dao.ItemSalesMin = make(map[string]int)
+	s.dao.ItemSalesMax = make(map[string]int)
+	s.dao.ItemWishMax = make(map[string]int)
+	s.dao.ItemWishMin = make(map[string]int)
+	s.dao.ItemCommentMax = make(map[string]int)
+	s.dao.ItemCommentMin = make(map[string]int)
+}

app/job/openplatform/open-sug/service/sug.go

@@ -0,0 +1,80 @@
+package service
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/job/openplatform/open-sug/model"
+	"go-common/library/log"
+)
+
+func (s *Service) buildData() (err error) {
+	var (
+		itemList   []*model.Item
+		scoreSlice []model.Score
+	)
+	if itemList, err = s.dao.FetchItem(context.TODO()); err != nil {
+		log.Error("pull project list error (%v)", err)
+		return
+	}
+	for _, item := range itemList {
+		if scoreSlice, err = s.dao.SeasonData(context.TODO(), item); err != nil {
+			log.Error("es item match season fail")
+			continue
+		}
+		if len(scoreSlice) > 0 {
+			if _, err = s.dao.SetItem(context.TODO(), item); err != nil {
+				log.Error("set item redis error(%v)", err)
+				continue
+			}
+			for _, score := range scoreSlice {
+				score.Score = score.Score * s.BuildScore(item, score.SeasonID)
+				if rowNum, _ := s.dao.InsertMatch(context.TODO(), item, score); rowNum > 0 {
+					if _, err = s.dao.SetSug(context.TODO(), score.SeasonID, item.ID, score.Score); err != nil {
+						log.Error("redis dao.SetSug error(%v)", err)
+						continue
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+// BuildScore ...
+func (s *Service) BuildScore(item *model.Item, seasonID string) (normalizationScore float64) {
+	normalizationScore = 0.5*(s.Normalization(item.SalesCount, s.dao.ItemSalesMax[seasonID], s.dao.ItemSalesMin[seasonID])) + 0.2*(s.Normalization(item.CommentCount, s.dao.ItemCommentMax[seasonID], s.dao.ItemCommentMin[seasonID])) + 0.3*(s.Normalization(item.WishCount, s.dao.ItemWishMax[seasonID], s.dao.ItemCommentMin[seasonID]))
+	return
+}
+
+// Normalization ...
+func (s *Service) Normalization(x int, max int, min int) (normalizationScore float64) {
+	if x == 0 {
+		return 0
+	}
+	if float64(max-min) == 0 {
+		return 1
+	}
+	return float64(x-min) / float64(max-min)
+}
+
+// Filter ...
+func (s *Service) Filter() {
+	c := context.TODO()
+	if bindItems, err := s.dao.GetBind(c); err == nil {
+		for _, buildItem := range bindItems {
+			s.dao.GetItem(c, buildItem)
+			log.Info("add filter data items_id(%d) season_id(%d)", buildItem.Item.ID, buildItem.SeasonID)
+			if _, err = s.dao.SetItem(c, buildItem.Item); err != nil {
+				log.Error("set item redis error(%v)", err)
+				continue
+			}
+			log.Info("add sug res items_id(%d) season_id(%d)", buildItem.Item.ID, buildItem.SeasonID)
+			if _, err = s.dao.SetSug(c, strconv.FormatInt(buildItem.SeasonID, 10), buildItem.Item.ID, float64(buildItem.Score)); err != nil {
+				log.Error("redis dao.SetSug error(%v)", err)
+				continue
+			}
+			s.dao.UpdatePic(c, buildItem)
+		}
+	}
+}