Create & Init Project...

app/job/main/up-rating/service/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "base.go",
+        "paramter.go",
+        "past.go",
+        "rating.go",
+        "run.go",
+        "score.go",
+        "service.go",
+        "statistics.go",
+        "task_status.go",
+        "top.go",
+        "trend.go",
+        "trend_heap.go",
+    ],
+    importpath = "go-common/app/job/main/up-rating/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//app/job/main/up-rating/dao:go_default_library",
+        "//app/job/main/up-rating/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/golang.org/x/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/service/base.go

@@ -0,0 +1,72 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// BaseInfoOffEnd get offset and end
+func (s *Service) BaseInfoOffEnd(c context.Context, date time.Time) (offset, end int, err error) {
+	start, err := s.dao.BaseInfoStart(c, date)
+	if err != nil {
+		return
+	}
+	offset = start - 1
+	end, err = s.dao.BaseInfoEnd(c, date)
+	return
+}
+
+// RatingOffEnd get offset and end
+func (s *Service) RatingOffEnd(c context.Context, date time.Time) (offset, end, count int, err error) {
+	start, err := s.dao.RatingStart(c, date)
+	if err != nil {
+		return
+	}
+	offset = start - 1
+	end, err = s.dao.RatingEnd(c, date)
+	if err != nil {
+		return
+	}
+	count, err = s.dao.RatingCount(c, date)
+	return
+}
+
+// BaseInfo get base infos
+func (s *Service) BaseInfo(c context.Context, date time.Time, start, end int, ch chan []*model.BaseInfo) (err error) {
+	defer close(ch)
+	for {
+		var bs []*model.BaseInfo
+		bs, start, err = s.dao.GetBaseInfo(c, date.Month(), start, end, _limit)
+		if err != nil {
+			return
+		}
+		if len(bs) == 0 {
+			break
+		}
+		ch <- bs
+	}
+	return
+}
+
+// BaseTotal get total base
+func (s *Service) BaseTotal(c context.Context, date time.Time) (total map[int64]*model.BaseInfo, err error) {
+	total = make(map[int64]*model.BaseInfo)
+	var id int64
+	for {
+		var bs []*model.BaseInfo
+		bs, err = s.dao.GetBaseTotal(c, date, id, int64(_limit))
+		if err != nil {
+			return
+		}
+		for _, b := range bs {
+			total[b.MID] = b
+		}
+		if len(bs) < _limit {
+			break
+		}
+		id = bs[len(bs)-1].ID
+	}
+	return
+}

app/job/main/up-rating/service/paramter.go

@@ -0,0 +1,33 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+func (s *Service) getAllParamter(c context.Context) (rp *model.RatingParameter, err error) {
+	paramters, err := s.dao.GetAllParamter(c)
+	if err != nil {
+		return
+	}
+	rp = &model.RatingParameter{
+		WDP:      paramters["wdp"],
+		WDC:      paramters["wdc"],
+		WDV:      paramters["wdv"],
+		WMDV:     paramters["wmdv"],
+		WCS:      paramters["wcs"],
+		WCSR:     paramters["wcsr"],
+		WMAAFans: paramters["wmaafans"],
+		WMAHFans: paramters["wmahfans"],
+		WIS:      paramters["wis"],
+		WISR:     paramters["wisr"],
+		HBASE:    paramters["hbase"],
+		HR:       paramters["hr"],
+		HV:       paramters["hv"],
+		HVM:      paramters["hvm"],
+		HL:       paramters["hl"],
+		HLM:      paramters["hlm"],
+	}
+	return
+}

app/job/main/up-rating/service/past.go

@@ -0,0 +1,210 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_layout = "2006-01-02"
+	_limit  = 2000
+)
+
+// RunPastScore run past score by date
+func (s *Service) RunPastScore(c context.Context, date time.Time) (err error) {
+	date = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	times, err := s.getPastRecord(c, date.Format(_layout))
+	if err != nil {
+		log.Error("s.getPastRecord error(%v)", err)
+		return
+	}
+	if times < 0 {
+		log.Info("This month's calculation did not start")
+		return
+	}
+	// 创作力需要计算前22个月的数据
+	if times >= 22 {
+		log.Info("Last month's calculation has end")
+		return
+	}
+	var (
+		readGroup errgroup.Group
+		cw        float64 // 创作力当月权重
+		iw        int64   // 影响力当月权重
+		pastScore []*model.Past
+		pastCh    = make(chan []*model.Rating, _limit)
+	)
+	// 获取前n个月的数据
+	pastDate := date.AddDate(0, -1*(22-times), 0)
+	times++ // update calculate times
+
+	//csr = csm0 + csm1 + ... + csm11 + 11/12 * csm12 + 10/12 * csm13 + ... 1/12 * csm22
+	cw = float64(times) / float64(12)
+	if cw > 1.0 {
+		cw = 1.0
+	}
+	// isr = mfans0 + mfans1 + ... + mfans12
+	iw = int64(float64(times) / float64(12))
+	// get past month data
+	readGroup.Go(func() (err error) {
+		err = s.RatingInfos(c, pastDate, pastCh)
+		if err != nil {
+			log.Error("s.RatingInfos error(%v)", err)
+		}
+		return
+	})
+	// cal past month data
+	readGroup.Go(func() (err error) {
+		pastScore, err = s.calPastScores(c, pastCh, cw, iw)
+		if err != nil {
+			log.Error("s.calPastScores error(%v)", err)
+		}
+		return
+	})
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	err = s.insertPastRecord(c, times, date.Format(_layout))
+	if err != nil {
+		log.Error("s.upPastRecord error(%v)", err)
+		return
+	}
+
+	err = s.batchInsertPastScore(c, pastScore)
+	if err != nil {
+		log.Error("s.batchInsertPastScore error(%v)", err)
+	}
+	return
+}
+
+// InsertPastRecord insert past record
+func (s *Service) InsertPastRecord(c context.Context, date string) (err error) {
+	return s.insertPastRecord(c, 0, date)
+}
+
+func (s *Service) calPastScores(c context.Context, pastRating chan []*model.Rating, cw float64, iw int64) (pastScore []*model.Past, err error) {
+	pastScore = make([]*model.Past, 0)
+	for rating := range pastRating {
+		p := calPastScore(rating, cw, iw)
+		pastScore = append(pastScore, p...)
+	}
+	return
+}
+
+func calPastScore(rating []*model.Rating, cw float64, iw int64) (pastScore []*model.Past) {
+	pastScore = make([]*model.Past, 0, len(rating))
+	for _, r := range rating {
+		pastScore = append(pastScore, &model.Past{
+			MID:                 r.MID,
+			MetaCreativityScore: int64(float64(r.MetaCreativityScore) * cw),
+			MetaInfluenceScore:  r.MetaInfluenceScore * iw,
+			CreditScore:         r.CreditScore,
+		})
+	}
+	return
+}
+
+// get past calculate record
+func (s *Service) getPastRecord(c context.Context, date string) (times int, err error) {
+	return s.dao.GetPastRecord(c, date)
+}
+
+func (s *Service) insertPastRecord(c context.Context, times int, date string) (err error) {
+	_, err = s.dao.InsertPastRecord(c, times, date)
+	return err
+}
+
+func (s *Service) pastInfos(c context.Context) (past map[int64]*model.Past, err error) {
+	past = make(map[int64]*model.Past)
+	var id int64
+	for {
+		var p []*model.Past
+		p, id, err = s.dao.GetPasts(c, id, int64(_limit))
+		if err != nil {
+			return
+		}
+		for i := 0; i < len(p); i++ {
+			past[p[i].MID] = p[i]
+		}
+		if len(p) < _limit {
+			break
+		}
+	}
+	return
+}
+
+func (s *Service) batchInsertPastScore(c context.Context, past []*model.Past) (err error) {
+	var (
+		buff    = make([]*model.Past, 2000)
+		buffEnd = 0
+	)
+	for _, p := range past {
+		buff[buffEnd] = p
+		buffEnd++
+		if buffEnd >= 2000 {
+			values := assemblePastValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertPastScoreStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := assemblePastValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = s.dao.InsertPastScoreStat(c, values)
+	}
+	return
+}
+
+func assemblePastValues(past []*model.Past) (values string) {
+	var buf bytes.Buffer
+	for _, p := range past {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(p.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.MetaCreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.MetaInfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.CreditScore, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) delOldPastInfo(c context.Context, limit int64) (err error) {
+	var rows int64
+	for {
+		rows, err = s.dao.DelPastStat(c, limit)
+		if err != nil {
+			return
+		}
+		if rows < limit {
+			break
+		}
+	}
+	return
+}
+
+// DelPastRecord del past record
+func (s *Service) DelPastRecord(c context.Context, date time.Time) (err error) {
+	_, err = s.dao.DelPastRecord(c, date)
+	return
+}

app/job/main/up-rating/service/rating.go

@@ -0,0 +1,127 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// DelRatings del ratings
+func (s *Service) DelRatings(c context.Context, date time.Time) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelRatings(c, date, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return s.DelPastRecord(c, date)
+}
+
+// RatingFast close chan when part finished
+func (s *Service) RatingFast(c context.Context, date time.Time, start, end int, ch chan []*model.Rating) (err error) {
+	defer close(ch)
+	return s.Ratings(c, date, start, end, ch)
+}
+
+// Ratings chan <- ratings, close chan outside
+func (s *Service) Ratings(c context.Context, date time.Time, start, end int, ch chan []*model.Rating) (err error) {
+	for {
+		var rs []*model.Rating
+		rs, start, err = s.dao.GetRatingsFast(c, date, start, end, _limit)
+		if err != nil {
+			return
+		}
+		if len(rs) == 0 {
+			break
+		}
+		ch <- rs
+	}
+	return
+}
+
+// RatingInfos rating infos
+func (s *Service) RatingInfos(c context.Context, date time.Time, ch chan []*model.Rating) (err error) {
+	defer close(ch)
+	var id int
+	for {
+		var rs []*model.Rating
+		rs, id, err = s.dao.GetRatings(c, date, id, _limit)
+		if err != nil {
+			return
+		}
+		ch <- rs
+		if len(rs) == 0 {
+			break
+		}
+	}
+	return
+}
+
+// BatchInsertRatingStat batch insert rating stat
+func (s *Service) BatchInsertRatingStat(c context.Context, wch chan []*model.Rating, date time.Time) (err error) {
+	var (
+		buff    = make([]*model.Rating, _limit)
+		buffEnd = 0
+	)
+	dateStr := date.Format(_layout)
+	for rs := range wch {
+		for _, r := range rs {
+			buff[buffEnd] = r
+			buffEnd++
+			if buffEnd >= _limit {
+				values := ratingStatValues(buff[:buffEnd], dateStr)
+				buffEnd = 0
+				_, err = s.dao.InsertRatingStat(c, date.Month(), values)
+				if err != nil {
+					return
+				}
+			}
+		}
+		if buffEnd > 0 {
+			values := ratingStatValues(buff[:buffEnd], dateStr)
+			buffEnd = 0
+			_, err = s.dao.InsertRatingStat(c, date.Month(), values)
+		}
+	}
+	return
+}
+
+func ratingStatValues(rs []*model.Rating, date string) (values string) {
+	var buf bytes.Buffer
+	for _, r := range rs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(r.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", date))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MetaCreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MetaInfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MagneticScore, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/up-rating/service/run.go

@@ -0,0 +1,164 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+// Run run scores
+func (s *Service) Run(c context.Context, date time.Time) (err error) {
+	date = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	// get weight parameters
+	params, err := s.getAllParamter(c)
+	if err != nil {
+		log.Error("s.getAllParamter error(%v)", err)
+		return
+	}
+	// get past scores
+	past, err := s.pastInfos(c)
+	if err != nil {
+		log.Error("s.pastInfos error(%v)", err)
+		return
+	}
+	err = s.PrepareData(c, date, past, params)
+	if err != nil {
+		log.Error("s.PrepareDate error(%v)", err)
+		return
+	}
+
+	err = s.CalScores(c, date, past, params)
+	if err != nil {
+		log.Error("s.CalScores error(%v)", err)
+		return
+	}
+
+	err = s.delOldPastInfo(c, int64(_limit))
+	if err != nil {
+		log.Error("s.delOldPastInfo error(%v)", err)
+		return
+	}
+
+	err = s.insertPastRecord(c, 0, date.AddDate(0, 1, 0).Format(_layout))
+	if err != nil {
+		log.Error("s.insertPastRecord error(%v)", err)
+		return
+	}
+
+	log.Info("run data read finished")
+	err = s.InsertTaskStatus(c, 0, 1, date.Format(_layout), "ok")
+	return
+}
+
+// PrepareData prepare old data
+func (s *Service) PrepareData(c context.Context, date time.Time, past map[int64]*model.Past, params *model.RatingParameter) (err error) {
+	var (
+		g         errgroup.Group
+		routines  = s.conf.Con.Concurrent
+		lastMonth = time.Date(date.Year(), date.Month()-1, 1, 0, 0, 0, 0, time.Local)
+	)
+
+	offset, end, _, err := s.RatingOffEnd(c, lastMonth)
+	if err != nil {
+		return
+	}
+	total := end - offset
+	section := (total - total%routines) / routines
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+
+		// read chan: for last ratings
+		rch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			err = s.RatingFast(c, lastMonth, begin, over, rch)
+			if err != nil {
+				log.Error("s.RatingFast error(%v)", err)
+			}
+			return
+		})
+
+		wch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			s.Copy(rch, wch, past, params)
+			return
+		})
+
+		g.Go(func() (err error) {
+			err = s.BatchInsertRatingStat(c, wch, date)
+			if err != nil {
+				log.Error("s.BatchInsertRatingStat error(%v)", err)
+			}
+			return
+		})
+	}
+
+	if err = g.Wait(); err != nil {
+		log.Error("run g.Wait error(%v)", err)
+		return
+	}
+	return
+}
+
+// CalScores cal scores
+func (s *Service) CalScores(c context.Context, date time.Time, past map[int64]*model.Past, params *model.RatingParameter) (err error) {
+	var (
+		routines = s.conf.Con.Concurrent
+		_limit   = s.conf.Con.Limit
+	)
+	t := time.Now().UnixNano()
+	var g errgroup.Group
+	// get id start:end by date
+	offset, end, err := s.BaseInfoOffEnd(c, date)
+	if err != nil {
+		return
+	}
+	total := end - offset
+	section := (total - total%routines) / routines
+	// parallelization and pipeling
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+		// read chan: for origin datas
+		rch := make(chan []*model.BaseInfo, _limit)
+		g.Go(func() (err error) {
+			err = s.BaseInfo(c, date, begin, over, rch)
+			if err != nil {
+				log.Error("s.BaseInfo error(%v)", err)
+			}
+			return
+		})
+
+		// write chan: for calculated results
+		wch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			s.CalScore(rch, wch, params, past, date)
+			return
+		})
+
+		g.Go(func() (err error) {
+			err = s.BatchInsertRatingStat(c, wch, date)
+			if err != nil {
+				log.Error("s.BatchInsertRatingStat error(%v)", err)
+			}
+			return
+		})
+	}
+	if err = g.Wait(); err != nil {
+		log.Error("run g.Wait error(%v)", err)
+		return
+	}
+
+	log.Info("cal time cost:", time.Now().UnixNano()-t)
+	return
+}

app/job/main/up-rating/service/score.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"math"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+	xtime "go-common/library/time"
+)
+
+// Copy copy data from month to month
+func (s *Service) Copy(rch chan []*model.Rating, wch chan []*model.Rating, past map[int64]*model.Past, params *model.RatingParameter) {
+	defer close(wch)
+	for rs := range rch {
+		for _, r := range rs {
+			if _, ok := past[r.MID]; !ok {
+				continue
+			}
+			r.MetaCreativityScore = 0
+			csr := past[r.MID].MetaCreativityScore
+			if csr == 0 {
+				r.CreativityScore = 0
+			} else {
+				r.CreativityScore = int64(math.Min(float64(params.WCS)*math.Log(float64(csr)), float64(params.WCSR)))
+			}
+
+			r.MetaInfluenceScore = 0
+			isr := past[r.MID].MetaInfluenceScore
+			if isr == 0 {
+				r.InfluenceScore = 0
+			} else {
+				r.InfluenceScore = int64(math.Min(float64(params.WIS)*math.Log(float64(isr)), float64(params.WISR)))
+			}
+			r.Date = xtime.Time(time.Date(r.Date.Time().Year(), r.Date.Time().Month()+1, 1, 0, 0, 0, 0, time.Local).Unix())
+			r.MagneticScore = r.CreativityScore + r.InfluenceScore + r.CreditScore
+		}
+		wch <- rs
+	}
+}
+
+// CalScore cal rating score
+func (s *Service) CalScore(rch chan []*model.BaseInfo,
+	wch chan []*model.Rating,
+	params *model.RatingParameter,
+	past map[int64]*model.Past, date time.Time) {
+	defer close(wch)
+	for bs := range rch {
+		m := make([]*model.Rating, 0)
+		for _, b := range bs {
+			if !b.Date.Time().Equal(date) {
+				continue
+			}
+			r := &model.Rating{
+				MID:                 b.MID,
+				TagID:               b.TagID,
+				MetaCreativityScore: calCreativetyMetaScore(b, params),
+				CreativityScore:     calCreativityScore(b, params, past),
+				MetaInfluenceScore:  calInfluenceMetaScore(b, params),
+				InfluenceScore:      calInfluenceScore(b, params, past),
+				CreditScore:         calCreditScore(b, params, past),
+				Date:                b.Date,
+			}
+			r.MagneticScore = r.CreativityScore + r.InfluenceScore + r.CreditScore
+			m = append(m, r)
+		}
+		wch <- m
+	}
+}
+
+func calCreativetyMetaScore(b *model.BaseInfo, params *model.RatingParameter) int64 {
+	// ps: 当月播放分
+	ps := params.WDP*b.PlayIncr + params.WDC*b.CoinIncr
+	// ubs: 当月投稿低保分
+	ubs := params.WDV * int64(math.Min(float64(b.Avs), float64(params.WMDV)))
+	// csm: 当月创作力得分
+	csm := ps + ubs
+	return csm
+}
+
+func calCreativityScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	csm := calCreativetyMetaScore(b, params)
+	// csr: csm + past 创作力原始分
+	var csr int64
+	if _, ok := past[b.MID]; ok {
+		csr = csm + past[b.MID].MetaCreativityScore
+	} else {
+		csr = csm
+	}
+	if csr < 1 {
+		return 0
+	}
+	// cs: 创作力总分
+	cs := math.Min(float64(params.WCS)*math.Log(float64(csr)), float64(params.WCSR))
+	return int64(cs)
+}
+
+func calInfluenceMetaScore(b *model.BaseInfo, params *model.RatingParameter) int64 {
+	// mfans: 当月活跃粉丝数
+	mfans := params.WMAAFans*(b.MAAFans+b.MAHFans) + params.WMAHFans*b.MAHFans
+	return mfans
+}
+
+func calInfluenceScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	mfans := calInfluenceMetaScore(b, params)
+	// isr: 影响力原始分
+	var isr int64
+	if _, ok := past[b.MID]; ok {
+		isr = mfans + past[b.MID].MetaInfluenceScore
+	} else {
+		isr = mfans
+	}
+	if isr < 1 {
+		return 0
+	}
+	// is: up主影响力分
+	is := math.Min(float64(params.WIS)*math.Log(float64(isr)), float64(params.WISR))
+	return int64(is)
+}
+
+func calCreditScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	addScore := min(b.OpenAvs*params.HV, params.HVM)
+	minusScore := min(b.LockedAvs*params.HL, params.HLM)
+	var cs int64
+	if _, ok := past[b.MID]; ok {
+		cs = past[b.MID].CreditScore + addScore - minusScore
+	} else {
+		cs = params.HBASE + addScore - minusScore
+	}
+	if cs < 0 {
+		cs = 0
+	}
+	return min(cs, params.HR)
+}
+
+func min(x, y int64) int64 {
+	if x > y {
+		return y
+	}
+	return x
+}

app/job/main/up-rating/service/service.go

@@ -0,0 +1,35 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/up-rating/conf"
+	"go-common/app/job/main/up-rating/dao"
+	"go-common/library/log"
+)
+
+// Service struct
+type Service struct {
+	conf *conf.Config
+	dao  *dao.Dao
+}
+
+// New fn
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		conf: c,
+		dao:  dao.New(c),
+	}
+	log.Info("service start")
+	return s
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close close the service
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/main/up-rating/service/statistics.go

@@ -0,0 +1,292 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	// TotalType total type
+	TotalType int = iota
+	// CreativeType creative type
+	CreativeType
+	// InfluenceType influence type
+	InfluenceType
+	// CreditType Credit type
+	CreditType
+)
+
+var (
+	_offset         int64 = 10
+	_sumTotal       int64
+	_creativeTotal  int64
+	_influenceTotal int64
+	_creditTotal    int64
+)
+
+func initSection(min, max, section int64, ctype int, tagID int64, date xtime.Time) *model.RatingStatis {
+	return &model.RatingStatis{
+		Section: section,
+		Tips:    fmt.Sprintf("\"%d-%d\"", min, max),
+		TagID:   tagID,
+		CDate:   date,
+		CType:   ctype,
+	}
+}
+
+func initSections(totalScore, tagID int64, ctype int, date xtime.Time) (statis []*model.RatingStatis) {
+	var (
+		idx int64
+	)
+	statis = make([]*model.RatingStatis, totalScore/_offset)
+	for idx*_offset < totalScore {
+		statis[idx] = initSection(idx*_offset, (idx+1)*_offset, idx, ctype, tagID, date)
+		idx++
+	}
+	return
+}
+
+// RunStatistics run up rating statistics
+func (s *Service) RunStatistics(c context.Context, date time.Time) (err error) {
+	err = s.initTotalScore(c)
+	if err != nil {
+		log.Error("s.initTotalScore error(%v)", err)
+		return
+	}
+	err = s.delStatistics(c, date)
+	if err != nil {
+		log.Error("s.delStatistics error(%v)", err)
+		return
+	}
+	err = s.statistics(c, date)
+	if err != nil {
+		log.Error("s.scoreStatistics error(%v)", err)
+	}
+	return
+}
+
+func (s *Service) initTotalScore(c context.Context) (err error) {
+	params, err := s.getAllParamter(c)
+	if err != nil {
+		log.Error("s.getAllParamter error(%v)", err)
+		return
+	}
+	_sumTotal = params.WCSR + params.HR + params.WISR
+	_creativeTotal = params.WCSR
+	_influenceTotal = params.WISR
+	_creditTotal = params.HR
+	return
+}
+
+func (s *Service) delStatistics(c context.Context, date time.Time) (err error) {
+	err = s.delRatingCom(c, "up_rating_statistics", date)
+	if err != nil {
+		return
+	}
+	err = s.delRatingCom(c, "up_rating_top", date)
+	return
+}
+
+// delRatingCom del com
+func (s *Service) delRatingCom(c context.Context, table string, date time.Time) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelRatingCom(c, table, date, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return
+}
+
+func (s *Service) statistics(c context.Context, date time.Time) (err error) {
+	var (
+		readGroup errgroup.Group
+		sourceCh  = make(chan []*model.Rating, _limit)
+		statisCh  = make(chan []*model.Rating, _limit)
+		topCh     = make(chan []*model.Rating, _limit)
+
+		sections  map[int]map[int64][]*model.RatingStatis
+		topRating map[int]map[int64]*RatingHeap
+	)
+	baseInfo, err := s.BaseTotal(c, date)
+	if err != nil {
+		log.Error("s.BaseTotal error(%v)", err)
+		return
+	}
+	// get rating info
+	readGroup.Go(func() (err error) {
+		err = s.RatingInfos(c, date, sourceCh)
+		if err != nil {
+			log.Error("s.RatingInfos error(%v)", err)
+		}
+		return
+	})
+
+	// dispatch
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(topCh)
+			close(statisCh)
+		}()
+		for rating := range sourceCh {
+			statisCh <- rating
+			topCh <- rating
+		}
+		return
+	})
+
+	// top
+	readGroup.Go(func() (err error) {
+		topRating, err = s.ratingTop(c, date, topCh)
+		if err != nil {
+			log.Error("s.RatingTop error(%v)", err)
+		}
+		return
+	})
+	// statis
+	readGroup.Go(func() (err error) {
+		sections, err = s.scoreStatistics(c, date, statisCh, baseInfo)
+		if err != nil {
+			log.Error("s.scoreStatistics error(%v)", err)
+		}
+		return
+	})
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	// persistent
+	var writeGroup errgroup.Group
+
+	//up_rating_statistics
+	writeGroup.Go(func() (err error) {
+		err = s.insertSections(c, sections)
+		if err != nil {
+			log.Error("s.insertSections error(%v)", err)
+		}
+		return
+	})
+	// up_rating_top
+	writeGroup.Go(func() (err error) {
+		_, err = s.insertTopRating(c, date, topRating, baseInfo)
+		if err != nil {
+			log.Error("s.insertSections error(%v)", err)
+		}
+		return
+	})
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}
+
+func (s *Service) scoreStatistics(c context.Context, date time.Time, source chan []*model.Rating, baseInfo map[int64]*model.BaseInfo) (sections map[int]map[int64][]*model.RatingStatis, err error) {
+	sections = make(map[int]map[int64][]*model.RatingStatis) // map[ctype][tagID][]*model.RatingStatis
+	sections[TotalType] = make(map[int64][]*model.RatingStatis)
+	sections[CreativeType] = make(map[int64][]*model.RatingStatis)
+	sections[InfluenceType] = make(map[int64][]*model.RatingStatis)
+	sections[CreditType] = make(map[int64][]*model.RatingStatis)
+	for rating := range source {
+		for _, r := range rating {
+			statisScoreCtype(TotalType, r.CreativityScore+r.InfluenceScore+r.CreditScore, _sumTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(CreativeType, r.CreativityScore, _creativeTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(InfluenceType, r.InfluenceScore, _influenceTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(CreditType, r.CreditScore, _creditTotal, sections, date, r, baseInfo[r.MID])
+		}
+	}
+	return
+}
+
+func statisScoreCtype(ctype int, score, totalScore int64, sections map[int]map[int64][]*model.RatingStatis, date time.Time, rate *model.Rating, base *model.BaseInfo) {
+	if _, ok := sections[ctype][rate.TagID]; !ok {
+		sections[ctype][rate.TagID] = initSections(totalScore, rate.TagID, ctype, xtime.Time(date.Unix()))
+	}
+	idx := score / _offset
+	if idx >= int64(len(sections[ctype][rate.TagID])) {
+		idx = int64(len(sections[ctype][rate.TagID]) - 1)
+	}
+	sections[ctype][rate.TagID][idx].Ups++
+	sections[ctype][rate.TagID][idx].TotalScore += rate.CreativityScore + rate.InfluenceScore + rate.CreditScore
+	sections[ctype][rate.TagID][idx].CreativityScore += rate.CreativityScore
+	sections[ctype][rate.TagID][idx].InfluenceScore += rate.InfluenceScore
+	sections[ctype][rate.TagID][idx].CreditScore += rate.CreditScore
+	if base != nil {
+		sections[ctype][rate.TagID][idx].Fans += base.TotalFans
+		sections[ctype][rate.TagID][idx].Avs += base.TotalAvs
+		sections[ctype][rate.TagID][idx].Coin += base.TotalCoin
+		sections[ctype][rate.TagID][idx].Play += base.TotalPlay
+	}
+}
+
+func (s *Service) insertSections(c context.Context, sections map[int]map[int64][]*model.RatingStatis) (err error) {
+	for ctype, tags := range sections {
+		for tagID, statis := range tags {
+			_, err = s.insertRatingStatis(c, ctype, tagID, statis)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) insertRatingStatis(c context.Context, ctype int, tagID int64, statis []*model.RatingStatis) (rows int64, err error) {
+	return s.dao.InsertRatingStatis(c, assembleRatingStatis(c, ctype, tagID, statis))
+}
+
+func assembleRatingStatis(c context.Context, ctype int, tagID int64, statis []*model.RatingStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, s := range statis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(s.Ups, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Section, 10))
+		buf.WriteByte(',')
+		buf.WriteString(s.Tips)
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.TotalScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Fans, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Avs, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Coin, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Play, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(s.CType))
+		buf.WriteByte(',')
+		buf.WriteString("'" + s.CDate.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/up-rating/service/task_status.go

@@ -0,0 +1,12 @@
+package service
+
+import (
+	"context"
+	"fmt"
+)
+
+// InsertTaskStatus insert task status
+func (s *Service) InsertTaskStatus(c context.Context, ctype, status int, date, message string) (err error) {
+	_, err = s.dao.InsertTaskStatus(c, fmt.Sprintf("(%d, %d, '%s', '%s')", ctype, status, date, message))
+	return
+}

app/job/main/up-rating/service/top.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"bytes"
+	"container/heap"
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// ratingTop get top ups
+func (s *Service) ratingTop(c context.Context, date time.Time, source chan []*model.Rating) (topRating map[int]map[int64]*RatingHeap, err error) {
+	topRating = make(map[int]map[int64]*RatingHeap) // map[ctype][tagID]
+	topRating[CreativeType] = make(map[int64]*RatingHeap)
+	topRating[InfluenceType] = make(map[int64]*RatingHeap)
+	for rating := range source {
+		for _, r := range rating {
+			if _, ok := topRating[CreativeType][r.TagID]; !ok {
+				topRating[CreativeType][r.TagID] = &RatingHeap{}
+			}
+			pushTopRating(topRating[CreativeType][r.TagID], CreativeType, r)
+			if _, ok := topRating[InfluenceType][r.TagID]; !ok {
+				topRating[InfluenceType][r.TagID] = &RatingHeap{}
+			}
+			pushTopRating(topRating[InfluenceType][r.TagID], InfluenceType, r)
+		}
+	}
+	return
+}
+
+func pushTopRating(h *RatingHeap, ctype int, r *model.Rating) {
+	tr := &model.TopRating{
+		MID:   r.MID,
+		CType: ctype,
+		TagID: r.TagID,
+	}
+	switch ctype {
+	case CreativeType:
+		tr.Score = r.CreativityScore
+	case InfluenceType:
+		tr.Score = r.InfluenceScore
+	}
+	heap.Push(h, tr)
+	if h.Len() > 10 {
+		heap.Pop(h)
+	}
+}
+
+func (s *Service) insertTopRating(c context.Context, date time.Time, topRating map[int]map[int64]*RatingHeap, baseInfo map[int64]*model.BaseInfo) (rows int64, err error) {
+	return s.dao.InsertTopRating(c, assemberTopRating(date, topRating, baseInfo))
+}
+
+func assemberTopRating(date time.Time, topRating map[int]map[int64]*RatingHeap, baseInfo map[int64]*model.BaseInfo) (vals string) {
+	var buf bytes.Buffer
+	for _, tagTop := range topRating {
+		for _, h := range tagTop {
+			for h.Len() > 0 {
+				tr := heap.Pop(h).(*model.TopRating)
+				info := baseInfo[tr.MID]
+				if info == nil {
+					info = &model.BaseInfo{}
+				}
+				buf.WriteString("(")
+				buf.WriteString(strconv.FormatInt(tr.MID, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.Itoa(tr.CType))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(tr.TagID, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(tr.Score, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(info.TotalFans, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(info.TotalPlay, 10))
+				buf.WriteByte(',')
+				buf.WriteString("'" + date.Format(_layout) + "'")
+				buf.WriteString(")")
+				buf.WriteByte(',')
+			}
+		}
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+// RatingHeap rating heap for topK
+type RatingHeap []*model.TopRating
+
+// Len len
+func (r RatingHeap) Len() int { return len(r) }
+
+// Less less
+func (r RatingHeap) Less(i, j int) bool { return r[i].Score < r[j].Score }
+
+// Swap swap
+func (r RatingHeap) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+
+// Push push to heap
+func (r *RatingHeap) Push(x interface{}) {
+	*r = append(*r, x.(*model.TopRating))
+}
+
+// Pop pop from heap
+func (r *RatingHeap) Pop() interface{} {
+	old := *r
+	n := len(old)
+	x := old[n-1]
+	*r = old[0 : n-1]
+	return x
+}

app/job/main/up-rating/service/trend.go

@@ -0,0 +1,444 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// CalTrend cal trend
+func (s *Service) CalTrend(c context.Context, date time.Time) (err error) {
+	return s.calTrend(c, date)
+}
+
+func (s *Service) calTrend(c context.Context, date time.Time) (err error) {
+	ds, err := s.getDiffs(c, date)
+	if err != nil {
+		return
+	}
+
+	var (
+		magneticSec  = sections(TotalType)
+		creativeSec  = sections(CreativeType)
+		influenceSec = sections(InfluenceType)
+		creditSec    = sections(CreditType)
+		g            errgroup.Group
+
+		// for concurrent write
+		_ascMagneticCh  = make(chan []*model.Diff, 5)
+		_ascCreativeCh  = make(chan []*model.Diff, 5)
+		_ascInfluenceCh = make(chan []*model.Diff, 5)
+		_ascCreditCh    = make(chan []*model.Diff, 5)
+
+		_descMagneticCh  = make(chan []*model.Diff, 5)
+		_descCreativeCh  = make(chan []*model.Diff, 5)
+		_descInfluenceCh = make(chan []*model.Diff, 5)
+		_descCreditCh    = make(chan []*model.Diff, 5)
+	)
+
+	// magnetic ascend
+	g.Go(func() (err error) {
+		defer close(_ascMagneticCh)
+		mr := classify(ds, TotalType, "asc", magneticSec)
+		push(_ascMagneticCh, mr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascMagneticCh)
+		return
+	})
+
+	// creativity ascend
+	g.Go(func() (err error) {
+		defer close(_ascCreativeCh)
+		cr := classify(ds, CreativeType, "asc", creativeSec)
+		push(_ascCreativeCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascCreativeCh)
+		return
+	})
+
+	// influence ascend
+	g.Go(func() (err error) {
+		defer close(_ascInfluenceCh)
+		ir := classify(ds, InfluenceType, "asc", influenceSec)
+		push(_ascInfluenceCh, ir)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascInfluenceCh)
+		return
+	})
+
+	// credit ascend
+	g.Go(func() (err error) {
+		defer close(_ascCreditCh)
+		cr := classify(ds, CreditType, "asc", creditSec)
+		push(_ascCreditCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascCreditCh)
+		return
+	})
+
+	// magnetic descend
+	g.Go(func() (err error) {
+		defer close(_descMagneticCh)
+		mr := classify(ds, TotalType, "desc", magneticSec)
+		push(_descMagneticCh, mr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descMagneticCh)
+		return
+	})
+
+	// creativity descend
+	g.Go(func() (err error) {
+		defer close(_descCreativeCh)
+		cr := classify(ds, CreativeType, "desc", creativeSec)
+		push(_descCreativeCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descCreativeCh)
+		return
+	})
+
+	// influence descend
+	g.Go(func() (err error) {
+		defer close(_descInfluenceCh)
+		ir := classify(ds, InfluenceType, "desc", influenceSec)
+		push(_descInfluenceCh, ir)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descInfluenceCh)
+		return
+	})
+
+	// credit descend
+	g.Go(func() (err error) {
+		defer close(_descCreditCh)
+		cr := classify(ds, CreditType, "desc", creditSec)
+		push(_descCreditCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descCreditCh)
+		return
+	})
+
+	if err = g.Wait(); err != nil {
+		log.Error("g.Wait error(%v)", err)
+	}
+	return
+}
+
+func push(ch chan []*model.Diff, m map[int64]map[int]Heap) {
+	for _, sm := range m {
+		for _, h := range sm {
+			ch <- h.Result()
+		}
+	}
+}
+
+func (s *Service) getDiffs(c context.Context, date time.Time) (ds map[int64][]*model.Diff, err error) {
+	lastMonth := time.Date(date.Year(), date.Month()-1, 1, 0, 0, 0, 0, time.Local)
+	last, err := s.getR(c, lastMonth)
+	if err != nil {
+		return
+	}
+
+	curMonth := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	cur, err := s.getR(c, curMonth)
+	if err != nil {
+		return
+	}
+	ds = diff(last, cur)
+	return
+}
+
+func sections(ctype int) (ss []*section) {
+	switch ctype {
+	case TotalType:
+		ss = initSec(10, 600)
+	case CreativeType, InfluenceType, CreditType:
+		ss = initSec(10, 200)
+	}
+	return
+}
+
+type section struct {
+	start int64
+	end   int64
+	tips  string
+}
+
+func initSec(c, total int64) (ss []*section) {
+	size := total / c
+	for index := int64(0); index < c; index++ {
+		start := index * size
+		end := start + size - 1
+		if index == c-1 {
+			end = total
+		}
+		sec := &section{
+			start: start,
+			end:   end,
+			tips:  fmt.Sprintf("\"%d-%d\"", start, end),
+		}
+		ss = append(ss, sec)
+	}
+	return
+}
+
+func getSection(score int64, ss []*section) (index int, tips string) {
+	for index, section := range ss {
+		if score >= section.start && score <= section.end {
+			return index, section.tips
+		}
+	}
+	return
+}
+
+func getHeap(order string, ctype int) Heap {
+	if order == "asc" {
+		return &AscHeap{heap: make([]*model.Diff, 0), ctype: ctype}
+	}
+	return &DescHeap{heap: make([]*model.Diff, 0), ctype: ctype}
+}
+
+// map[tag_id]map[section][]diffs
+func classify(ds map[int64][]*model.Diff, ctype int, order string, ss []*section) (m map[int64]map[int]Heap) {
+	m = make(map[int64]map[int]Heap)
+	for tagID, diffs := range ds {
+		for _, diff := range diffs {
+			sec, tips := getSection(diff.GetScore(ctype), ss)
+			// need clone
+			_diff := clone(diff)
+			_diff.Section = sec
+			_diff.CType = ctype
+			_diff.Tips = tips
+			if sm, ok := m[tagID]; ok {
+				if _, ok = sm[sec]; ok {
+					sm[sec].Put(_diff)
+				} else {
+					h := getHeap(order, ctype)
+					h.Put(_diff)
+					sm[sec] = h
+				}
+			} else {
+				h := getHeap(order, ctype)
+				h.Put(_diff)
+				sm := make(map[int]Heap)
+				sm[sec] = h
+				m[tagID] = sm
+			}
+		}
+	}
+	return
+}
+
+func clone(a *model.Diff) *model.Diff {
+	return &model.Diff{
+		MID:             a.MID,
+		MagneticScore:   a.MagneticScore,
+		CreativityScore: a.CreativityScore,
+		InfluenceScore:  a.InfluenceScore,
+		CreditScore:     a.CreditScore,
+		MagneticDiff:    a.MagneticDiff,
+		CreativityDiff:  a.CreativityDiff,
+		InfluenceDiff:   a.InfluenceDiff,
+		CreditDiff:      a.CreditDiff,
+		TotalAvs:        a.TotalAvs,
+		Fans:            a.Fans,
+		TagID:           a.TagID,
+		Date:            a.Date,
+	}
+}
+
+// ds map[tag_id][]*model.Diff
+func diff(last map[int64]*model.Rating, cur map[int64]*model.Rating) (ds map[int64][]*model.Diff) {
+	ds = make(map[int64][]*model.Diff)
+	for mid, r := range cur {
+		if _, ok := last[mid]; !ok {
+			continue
+		}
+		lr := last[mid]
+		diff := &model.Diff{
+			MID:             mid,
+			TagID:           r.TagID,
+			MagneticScore:   r.MagneticScore,
+			CreativityScore: r.CreativityScore,
+			InfluenceScore:  r.InfluenceScore,
+			CreditScore:     r.CreditScore,
+			MagneticDiff:    int(r.MagneticScore - lr.MagneticScore),
+			CreativityDiff:  int(r.CreativityScore - lr.CreativityScore),
+			InfluenceDiff:   int(r.InfluenceScore - lr.InfluenceScore),
+			CreditDiff:      int(r.CreditScore - lr.CreditScore),
+			Date:            r.Date,
+		}
+		if _, ok := ds[diff.TagID]; ok {
+			ds[diff.TagID] = append(ds[diff.TagID], diff)
+		} else {
+			ds[diff.TagID] = []*model.Diff{diff}
+		}
+	}
+	return
+}
+
+// GetR m[mid]*model.Rating
+func (s *Service) getR(c context.Context, date time.Time) (m map[int64]*model.Rating, err error) {
+	var (
+		g        errgroup.Group
+		routines = 5 // 4 core + 1
+		ch       = make(chan []*model.Rating, routines)
+	)
+
+	m = make(map[int64]*model.Rating)
+	offset, end, total, err := s.RatingOffEnd(c, date)
+	if err != nil {
+		return
+	}
+	if total == 0 {
+		return
+	}
+	t := end - offset
+	section := (t - t%routines) / routines
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+		g.Go(func() (err error) {
+			err = s.Ratings(c, date, begin, over, ch)
+			if err != nil {
+				log.Error("get rating infos error(%v)", err)
+			}
+			return
+		})
+	}
+
+	g.Go(func() (err error) {
+	Loop:
+		for rs := range ch {
+			for _, r := range rs {
+				m[r.MID] = r
+			}
+			if len(m) == total {
+				break Loop
+			}
+		}
+		return
+	})
+	if err = g.Wait(); err != nil {
+		log.Error("get rating wait error(%v)", err)
+	}
+	return
+}
+
+// BatchInsertDiffs batch insert diffs
+func (s *Service) batchInsertDiffs(c context.Context, table string, wch chan []*model.Diff) (err error) {
+	var (
+		buff    = make([]*model.Diff, _limit)
+		buffEnd = 0
+	)
+	for ds := range wch {
+		for _, d := range ds {
+			buff[buffEnd] = d
+			buffEnd++
+			if buffEnd >= _limit {
+				values := diffValues(buff[:buffEnd])
+				buffEnd = 0
+				_, err = s.dao.InsertTrend(c, table, values)
+				if err != nil {
+					return
+				}
+			}
+		}
+		if buffEnd > 0 {
+			values := diffValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertTrend(c, table, values)
+		}
+	}
+	return
+}
+
+func diffValues(ds []*model.Diff) (values string) {
+	var buf bytes.Buffer
+	for _, r := range ds {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(r.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CreativityDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.InfluenceDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CreditDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MagneticScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.MagneticDiff))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", r.Date.Time().Format(_layout)))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CType))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.Section))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", r.Tips))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+// DelTrends del trends
+func (s *Service) DelTrends(c context.Context, table string) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelTrend(c, table, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return
+}

app/job/main/up-rating/service/trend_heap.go

@@ -0,0 +1,101 @@
+package service
+
+import (
+	"container/heap"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// Heap for diff topK
+type Heap interface {
+	Put(*model.Diff)
+	Result() []*model.Diff
+}
+
+// AscHeap for asc
+type AscHeap struct {
+	heap  []*model.Diff
+	ctype int
+}
+
+// Put put diff to heap
+func (a *AscHeap) Put(diff *model.Diff) {
+	heap.Push(a, diff)
+	if a.Len() > 200 {
+		heap.Pop(a)
+	}
+}
+
+// Result get result
+func (a *AscHeap) Result() []*model.Diff {
+	return a.heap
+}
+
+// Len len
+func (a *AscHeap) Len() int { return len(a.heap) }
+
+// Less less
+func (a *AscHeap) Less(i, j int) bool {
+	return a.heap[i].GetScore(a.ctype) < a.heap[j].GetScore(a.ctype)
+}
+
+// Swap swap
+func (a *AscHeap) Swap(i, j int) { a.heap[i], a.heap[j] = a.heap[j], a.heap[i] }
+
+// Push push to heap
+func (a *AscHeap) Push(x interface{}) {
+	a.heap = append(a.heap, x.(*model.Diff))
+}
+
+// Pop pop from heap
+func (a *AscHeap) Pop() interface{} {
+	old := a.heap
+	n := len(old)
+	x := old[n-1]
+	a.heap = old[0 : n-1]
+	return x
+}
+
+// DescHeap for desc
+type DescHeap struct {
+	heap  []*model.Diff
+	ctype int
+}
+
+// Put to descHeap
+func (d *DescHeap) Put(diff *model.Diff) {
+	heap.Push(d, diff)
+	if d.Len() > 200 {
+		heap.Pop(d)
+	}
+}
+
+// Result desc heap result
+func (d *DescHeap) Result() []*model.Diff {
+	return d.heap
+}
+
+// Len len
+func (d *DescHeap) Len() int { return len(d.heap) }
+
+// Less less
+func (d *DescHeap) Less(i, j int) bool {
+	return d.heap[i].GetScore(d.ctype) > d.heap[j].GetScore(d.ctype)
+}
+
+// Swap swap
+func (d *DescHeap) Swap(i, j int) { d.heap[i], d.heap[j] = d.heap[j], d.heap[i] }
+
+// Push push to desc heap
+func (d *DescHeap) Push(x interface{}) {
+	d.heap = append(d.heap, x.(*model.Diff))
+}
+
+// Pop pop from desc heap
+func (d *DescHeap) Pop() interface{} {
+	old := d.heap
+	n := len(old)
+	x := old[n-1]
+	d.heap = old[0 : n-1]
+	return x
+}