Create & Init Project...

app/job/main/up-rating/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/up-rating/cmd:all-srcs",
+        "//app/job/main/up-rating/conf:all-srcs",
+        "//app/job/main/up-rating/dao:all-srcs",
+        "//app/job/main/up-rating/http:all-srcs",
+        "//app/job/main/up-rating/model:all-srcs",
+        "//app/job/main/up-rating/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/CHANGELOG.md

@@ -0,0 +1,10 @@
+#### up-rating-job
+
+##### Version 0.0.1
+> 1. 初始化up-rating-job
+
+##### Version 0.0.2
+> 1. 添加新增past_record接口
+
+##### Version 0.0.3
+> 1. 优化statistics接口

app/job/main/up-rating/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+gaopeng
+
+# Author
+gaopeng
+shaozhenyu
+
+# Reviewer
+all

app/job/main/up-rating/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- gaopeng
+- shaozhenyu
+labels:
+- job
+- job/main/up-rating
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- gaopeng
+- shaozhenyu

app/job/main/up-rating/README.md

@@ -0,0 +1,10 @@
+#### up-rating-job
+
+##### 项目简介
+> up-rating-job
+
+##### 编译环境
+> 请只用golang v1.8.x以上版本编译执行
+
+##### 依赖包
+> 1.公共包go-common

app/job/main/up-rating/cmd/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["up-rating-job.toml"],
+    importpath = "go-common/app/job/main/up-rating/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//app/job/main/up-rating/http:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/cmd/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/up-rating/conf"
+	"go-common/app/job/main/up-rating/http"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+
+	http.Init(conf.Conf)
+	signalHandler()
+}
+
+func signalHandler() {
+	var ch = make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		si := <-ch
+		log.Info("up-rating-job get a signal %s", si.String())
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			log.Info("get a signal %s, stop the consume process", si.String())
+			http.Close()
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/up-rating/cmd/up-rating-job.toml

@@ -0,0 +1,42 @@
+[log]
+dir = "/data/log/up-rating-job/"
+
+[mysql]
+[mysql.rating]
+	addr = "172.16.33.205:3308"
+	dsn = "gaopeng:V8ktIf7HzvpPSsZeNy5ClWw1RYFAmUDJ@tcp(172.16.33.205:3308)/bilibili_up_rating?timeout=10s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	active = 30
+	idle = 2
+	idleTimeout = "4h"
+	queryTimeout = "10s"
+	execTimeout = "100s"
+	tranTimeout = "100s"
+	[mysql.rating.breaker]
+	window = "3s"
+	sleep = "100ms"
+	bucket = 10
+	ratio = 0.5
+	request = 100
+
+[app]
+key = "a4bb50e6f2bc2e0f"
+secret = "3507b8667a3d0b6c952013db2b635f3a"
+
+[bm]
+    addr = "0.0.0.0:8414"
+    maxListen = 1000
+    timeout = "3600s"
+
+[httpClient]
+key = "c1a1cb2d89c33794"
+secret = "dda47eeca111e03e6845017505baea13"
+dial = "2s"
+timeout = "3s"
+keepAlive = "60s"
+timer = 10
+[httpClient.breaker]
+window = "10s"
+sleep = "100ms"
+bucket = 10
+ratio = 0.5
+request =  100

app/job/main/up-rating/conf/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/up-rating/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/conf/conf.go

@@ -0,0 +1,103 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/trace"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	// ConfPath local config path
+	confPath string
+	client   *conf.Client
+	// Conf is global config object.
+	Conf = &Config{}
+)
+
+// Config is project all config
+type Config struct {
+	// log
+	Log *log.Config
+	// Mysql
+	MySQL *MySQL
+	// tracer
+	Tracer *trace.Config
+	// http client
+	HTTPClient *bm.ClientConfig
+	// bm
+	BM *bm.ServerConfig
+	// concurrent
+	Con *Concurrent
+}
+
+// MySQL mysql config
+type MySQL struct {
+	Rating *sql.Config
+}
+
+// Concurrent concurrent compute
+type Concurrent struct {
+	Concurrent int
+	Limit      int
+}
+
+// MailAddr mail send addr.
+type MailAddr struct {
+	Type int
+	Addr []string
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init config.
+func Init() (err error) {
+	if confPath != "" {
+		_, err = toml.DecodeFile(confPath, &Conf)
+		return
+	}
+	err = remote()
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/up-rating/dao/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "base.go",
+        "dao.go",
+        "paramter.go",
+        "past.go",
+        "rating.go",
+        "statistics.go",
+        "task_status.go",
+    ],
+    importpath = "go-common/app/job/main/up-rating/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//app/job/main/up-rating/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "base_test.go",
+        "dao_test.go",
+        "paramter_test.go",
+        "past_test.go",
+        "rating_test.go",
+        "statistics_test.go",
+        "task_status_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/job/main/up-rating/dao/base.go

@@ -0,0 +1,85 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+)
+
+const (
+	// get base info from up_level_info
+	_baseInfoSQL  = "SELECT id,mid,tag_id,inc_play,inc_coin,avs,maafans,mahfans,open_avs,lock_avs,cdate FROM up_level_info_%02d WHERE id > ? AND id <= ? LIMIT ?"
+	_baseTotalSQL = "SELECT id,mid,total_fans,total_avs,total_coin,total_play FROM up_level_info_%02d WHERE id > ? AND cdate = '%s' LIMIT ?"
+
+	// get up_level_info start & end
+	_baseInfoStartSQL = "SELECT id FROM up_level_info_%02d WHERE cdate=? ORDER BY id LIMIT 1"
+	_baseInfoEndSQL   = "SELECT id FROM up_level_info_%02d WHERE cdate=? ORDER BY id DESC LIMIT 1"
+)
+
+// GetBaseInfo get rating info
+func (d *Dao) GetBaseInfo(c context.Context, month time.Month, start, end, limit int) (bs []*model.BaseInfo, id int, err error) {
+	rows, err := d.db.Query(c, fmt.Sprintf(_baseInfoSQL, month), start, end, limit)
+	if err != nil {
+		log.Error("d.db.Query Rating Base Info error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		b := &model.BaseInfo{}
+		err = rows.Scan(&id, &b.MID, &b.TagID, &b.PlayIncr, &b.CoinIncr, &b.Avs, &b.MAAFans, &b.MAHFans, &b.OpenAvs, &b.LockedAvs, &b.Date)
+		if err != nil {
+			log.Error("rows scan error(%v)", err)
+			return
+		}
+		bs = append(bs, b)
+	}
+	return
+}
+
+// GetBaseTotal get total
+func (d *Dao) GetBaseTotal(c context.Context, date time.Time, id, limit int64) (bs []*model.BaseInfo, err error) {
+	bs = make([]*model.BaseInfo, 0)
+	rows, err := d.db.Query(c, fmt.Sprintf(_baseTotalSQL, date.Month(), date.Format("2006-01-02")), id, limit)
+	if err != nil {
+		log.Error("d.db.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		b := &model.BaseInfo{}
+		err = rows.Scan(&b.ID, &b.MID, &b.TotalFans, &b.TotalAvs, &b.TotalCoin, &b.TotalPlay)
+		if err != nil {
+			log.Error("rows scan error(%v)", err)
+			return
+		}
+		bs = append(bs, b)
+	}
+	return
+}
+
+// BaseInfoStart get start id by date
+func (d *Dao) BaseInfoStart(c context.Context, date time.Time) (start int, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_baseInfoStartSQL, date.Month()), date)
+	if err = row.Scan(&start); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+	}
+	return
+}
+
+// BaseInfoEnd get end id by date
+func (d *Dao) BaseInfoEnd(c context.Context, date time.Time) (end int, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_baseInfoEndSQL, date.Month()), date)
+	if err = row.Scan(&end); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+	}
+	return
+}

app/job/main/up-rating/dao/base_test.go

@@ -0,0 +1,83 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoGetBaseInfo(t *testing.T) {
+	convey.Convey("GetBaseInfo", t, func(ctx convey.C) {
+		var (
+			c                = context.Background()
+			month time.Month = time.June
+			start            = int(0)
+			end              = int(1000)
+			limit            = int(1)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_level_info_06(mid) VALUES(100) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			bs, id, err := d.GetBaseInfo(c, month, start, end, limit)
+			ctx.Convey("Then err should be nil.bs,id should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(id, convey.ShouldNotBeNil)
+				ctx.So(bs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetBaseTotal(t *testing.T) {
+	convey.Convey("GetBaseTotal", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			date  = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+			id    = int64(0)
+			limit = int64(100)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_level_info_06(mid) VALUES(101) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			bs, err := d.GetBaseTotal(c, date, id, limit)
+			ctx.Convey("Then err should be nil.bs should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(bs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoBaseInfoStart(t *testing.T) {
+	convey.Convey("BaseInfoStart", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_level_info_06(mid) VALUES(102) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			start, err := d.BaseInfoStart(c, date)
+			ctx.Convey("Then err should be nil.start should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(start, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoBaseInfoEnd(t *testing.T) {
+	convey.Convey("BaseInfoEnd", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_level_info_06(mid) VALUES(103) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			end, err := d.BaseInfoEnd(c, date)
+			ctx.Convey("Then err should be nil.end should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(end, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/dao/dao.go

@@ -0,0 +1,44 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/up-rating/conf"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+// Dao dao
+type Dao struct {
+	c  *conf.Config
+	db *sql.DB
+}
+
+// New fn
+func New(c *conf.Config) (d *Dao) {
+	log.Info("dao start")
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.MySQL.Rating),
+	}
+	//d.db.State = prom.LibClient
+	return
+}
+
+// Ping ping health.
+func (d *Dao) Ping(c context.Context) (err error) {
+	return d.db.Ping(c)
+}
+
+// Close close connections of mc, redis, db.
+func (d *Dao) Close() {
+	if d.db != nil {
+		d.db.Close()
+	}
+}
+
+// BeginTran begin transcation
+func (d *Dao) BeginTran(c context.Context) (tx *sql.Tx, err error) {
+	return d.db.Begin(c)
+}

app/job/main/up-rating/dao/dao_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"flag"
+	"go-common/app/job/main/up-rating/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.up-rating-job")
+		flag.Set("conf_token", "b48530d9e9d1a3aa20e078e8c72932d8")
+		flag.Set("tree_id", "65764")
+		flag.Set("conf_version", "0.0.1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/up-rating-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/up-rating/dao/paramter.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/log"
+)
+
+var (
+	_allParamterSQL = "SELECT name, value FROM rating_parameter"
+)
+
+// GetAllParamter get all paramter
+func (d *Dao) GetAllParamter(c context.Context) (paramters map[string]int64, err error) {
+	paramters = make(map[string]int64)
+	rows, err := d.db.Query(c, _allParamterSQL)
+	if err != nil {
+		log.Error("d.db.Query GetAllParamter error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var name string
+		var vaule int64
+		err = rows.Scan(&name, &vaule)
+		if err != nil {
+			log.Error("rows.Scan GetAllParamter error(%v)", err)
+			return
+		}
+		paramters[name] = vaule
+	}
+	return
+}

app/job/main/up-rating/dao/paramter_test.go

@@ -0,0 +1,24 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoGetAllParamter(t *testing.T) {
+	convey.Convey("GetAllParamter", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO rating_parameter(name,value) VALUES('test', 123)")
+			paramters, err := d.GetAllParamter(c)
+			ctx.Convey("Then err should be nil.paramters should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(paramters, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/dao/past.go

@@ -0,0 +1,93 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_pastRatingRecordSQL = "SELECT times FROM past_rating_record WHERE cdate = '%s' AND is_deleted = 0"
+	_pastStatSQL         = "SELECT id,mid,creativity_score,influence_score,credit_score FROM past_score_statistics WHERE id > ? ORDER BY id LIMIT ?"
+
+	// insert
+	_inPastRecordSQL  = "INSERT INTO past_rating_record(times, cdate) VALUES(?,'%s') ON DUPLICATE KEY UPDATE times=VALUES(times)"
+	_pastScoreStatSQL = "INSERT INTO past_score_statistics(mid,creativity_score,influence_score,credit_score) VALUES %s ON DUPLICATE KEY UPDATE creativity_score=creativity_score+VALUES(creativity_score),influence_score=influence_score+VALUES(influence_score),credit_score=VALUES(credit_score)"
+
+	// delete
+	_delPastStatSQL   = "DELETE FROM past_score_statistics ORDER BY ID LIMIT ?"
+	_delPastRecordSQL = "DELETE FROM past_rating_record WHERE cdate='%s'"
+)
+
+// DelPastRecord del past record
+func (d *Dao) DelPastRecord(c context.Context, date time.Time) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_delPastRecordSQL, date.Format(_layout)))
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}
+
+// GetPastRecord batch insert past score stat
+func (d *Dao) GetPastRecord(c context.Context, cdate string) (times int, err error) {
+	err = d.db.QueryRow(c, fmt.Sprintf(_pastRatingRecordSQL, cdate)).Scan(&times)
+	if err == sql.ErrNoRows {
+		err = nil
+		times = -1
+	}
+	return
+}
+
+// InsertPastRecord insert past record date and times
+func (d *Dao) InsertPastRecord(c context.Context, times int, cdate string) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_inPastRecordSQL, cdate), times)
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}
+
+// InsertPastScoreStat batch insert past score stat
+func (d *Dao) InsertPastScoreStat(c context.Context, values string) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_pastScoreStatSQL, values))
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_pastScoreStatSQL, values), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// GetPasts get past statistics
+func (d *Dao) GetPasts(c context.Context, offset, limit int64) (past []*model.Past, last int64, err error) {
+	past = make([]*model.Past, 0, limit)
+	rows, err := d.db.Query(c, _pastStatSQL, offset, limit)
+	if err != nil {
+		log.Error("d.db.Query GetPasts error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		p := &model.Past{}
+		err = rows.Scan(&last, &p.MID, &p.MetaCreativityScore, &p.MetaInfluenceScore, &p.CreditScore)
+		if err != nil {
+			log.Error("rows.Scan GetPasts error(%v)", err)
+			return
+		}
+		past = append(past, p)
+	}
+	return
+}
+
+// DelPastStat del past stat
+func (d *Dao) DelPastStat(c context.Context, limit int64) (rows int64, err error) {
+	res, err := d.db.Exec(c, _delPastStatSQL, limit)
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/up-rating/dao/past_test.go

@@ -0,0 +1,112 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoDelPastRecord(t *testing.T) {
+	convey.Convey("DelPastRecord", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO past_rating_record(times,date) VALUES(1, '2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rows, err := d.DelPastRecord(c, date)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetPastRecord(t *testing.T) {
+	convey.Convey("GetPastRecord", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			cdate = "2018-06-01"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO past_rating_record(times,date) VALUES(1, '2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			times, err := d.GetPastRecord(c, cdate)
+			ctx.Convey("Then err should be nil.times should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(times, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoInsertPastRecord(t *testing.T) {
+	convey.Convey("InsertPastRecord", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			times = int(1)
+			cdate = "2018-06-01"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rows, err := d.InsertPastRecord(c, times, cdate)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoInsertPastScoreStat(t *testing.T) {
+	convey.Convey("InsertPastScoreStat", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			values = "(1, 100, 100, 100)"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			rows, err := d.InsertPastScoreStat(c, values)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetPasts(t *testing.T) {
+	convey.Convey("GetPasts", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			offset = int64(0)
+			limit  = int64(1000)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO past_score_statistics(mid) VALUES(1) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			past, last, err := d.GetPasts(c, offset, limit)
+			ctx.Convey("Then err should be nil.past,last should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(last, convey.ShouldNotBeNil)
+				ctx.So(past, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelPastStat(t *testing.T) {
+	convey.Convey("DelPastStat", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			limit = int64(0)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO past_score_statistics(mid) VALUES(1) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rows, err := d.DelPastStat(c, limit)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/dao/rating.go

@@ -0,0 +1,118 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+)
+
+const (
+	_layout = "2006-01-02"
+	// get up_rating start & end
+	_ratingStartSQL = "SELECT id FROM up_rating_%02d WHERE cdate='%s' ORDER BY id LIMIT 1"
+	_ratingEndSQL   = "SELECT id FROM up_rating_%02d WHERE cdate='%s' ORDER BY id DESC LIMIT 1"
+	_ratingCountSQL = "SELECT COUNT(*) FROM up_rating_%02d WHERE cdate='%s'"
+
+	_ratingSQL     = "SELECT id,mid,tag_id,creativity_score,influence_score,credit_score,meta_creativity_score,meta_influence_score,cdate FROM up_rating_%02d WHERE cdate= '%s' AND id > ? ORDER BY id LIMIT ?"
+	_ratingByIDSQL = "SELECT id,mid,tag_id,creativity_score,influence_score,credit_score,meta_creativity_score,meta_influence_score,magnetic_score,cdate FROM up_rating_%02d WHERE id > ? AND id <= ? LIMIT ?"
+
+	_ratingScoreSQL    = "INSERT INTO up_rating_%02d(mid,tag_id,cdate,creativity_score,influence_score,credit_score,meta_creativity_score,meta_influence_score, magnetic_score) VALUES %s ON DUPLICATE KEY UPDATE tag_id=VALUES(tag_id), cdate=VALUES(cdate), creativity_score=VALUES(creativity_score),influence_score=VALUES(influence_score),credit_score=VALUES(credit_score),meta_creativity_score=VALUES(meta_creativity_score),meta_influence_score=VALUES(meta_influence_score),magnetic_score=VALUES(magnetic_score)"
+	_delRatingScoreSQL = "DELETE FROM up_rating_%02d WHERE cdate='%s' LIMIT ?"
+)
+
+// DelRatings del ratings
+func (d *Dao) DelRatings(c context.Context, date time.Time, limit int) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_delRatingScoreSQL, date.Month(), date.Format(_layout)), limit)
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}
+
+// RatingStart get start id by date
+func (d *Dao) RatingStart(c context.Context, date time.Time) (start int, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_ratingStartSQL, date.Month(), date.Format(_layout)))
+	if err = row.Scan(&start); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+	}
+	return
+}
+
+// RatingEnd get end id by date
+func (d *Dao) RatingEnd(c context.Context, date time.Time) (end int, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_ratingEndSQL, date.Month(), date.Format(_layout)))
+	if err = row.Scan(&end); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+	}
+	return
+}
+
+// RatingCount get end id by date
+func (d *Dao) RatingCount(c context.Context, date time.Time) (count int, err error) {
+	row := d.db.QueryRow(c, fmt.Sprintf(_ratingCountSQL, date.Month(), date.Format(_layout)))
+	if err = row.Scan(&count); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		}
+	}
+	return
+}
+
+// GetRatings get ratings by date
+func (d *Dao) GetRatings(c context.Context, date time.Time, offset, limit int) (rs []*model.Rating, last int, err error) {
+	rows, err := d.db.Query(c, fmt.Sprintf(_ratingSQL, date.Month(), date.Format(_layout)), offset, limit)
+	if err != nil {
+		log.Error("d.db.Query Rating Info error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &model.Rating{}
+		err = rows.Scan(&last, &r.MID, &r.TagID, &r.CreativityScore, &r.InfluenceScore, &r.CreditScore, &r.MetaCreativityScore, &r.MetaInfluenceScore, &r.Date)
+		if err != nil {
+			log.Error("rows scan error(%v)", err)
+			return
+		}
+		rs = append(rs, r)
+	}
+	return
+}
+
+// GetRatingsFast get rating fast
+func (d *Dao) GetRatingsFast(c context.Context, date time.Time, start, end, limit int) (rs []*model.Rating, id int, err error) {
+	rows, err := d.db.Query(c, fmt.Sprintf(_ratingByIDSQL, date.Month()), start, end, limit)
+	if err != nil {
+		log.Error("d.db.Query Rating Info error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &model.Rating{}
+		err = rows.Scan(&id, &r.MID, &r.TagID, &r.CreativityScore, &r.InfluenceScore, &r.CreditScore, &r.MetaCreativityScore, &r.MetaInfluenceScore, &r.MagneticScore, &r.Date)
+		if err != nil {
+			log.Error("rows scan error(%v)", err)
+			return
+		}
+		rs = append(rs, r)
+	}
+	return
+}
+
+// InsertRatingStat batch insert rating score stat
+func (d *Dao) InsertRatingStat(c context.Context, month time.Month, values string) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_ratingScoreSQL, month, values))
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_ratingScoreSQL, month, values), err)
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/up-rating/dao/rating_test.go

@@ -0,0 +1,135 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoDelRatings(t *testing.T) {
+	convey.Convey("DelRatings", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			date  = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+			limit = int(10)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rows, err := d.DelRatings(c, date, limit)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRatingStart(t *testing.T) {
+	convey.Convey("RatingStart", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			start, err := d.RatingStart(c, date)
+			ctx.Convey("Then err should be nil.start should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(start, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRatingEnd(t *testing.T) {
+	convey.Convey("RatingEnd", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			end, err := d.RatingEnd(c, date)
+			ctx.Convey("Then err should be nil.end should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(end, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoRatingCount(t *testing.T) {
+	convey.Convey("RatingCount", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			date = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			count, err := d.RatingCount(c, date)
+			ctx.Convey("Then err should be nil.count should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(count, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetRatings(t *testing.T) {
+	convey.Convey("GetRatings", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			date   = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+			offset = int(0)
+			limit  = int(100)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rs, last, err := d.GetRatings(c, date, offset, limit)
+			ctx.Convey("Then err should be nil.rs,last should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(last, convey.ShouldNotBeNil)
+				ctx.So(rs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGetRatingsFast(t *testing.T) {
+	convey.Convey("GetRatingsFast", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			date  = time.Date(2018, time.June, 1, 0, 0, 0, 0, time.Local)
+			start = int(0)
+			end   = int(10000)
+			limit = int(10)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_06(mid,cdate) VALUES(1,'2018-06-01') ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rs, id, err := d.GetRatingsFast(c, date, start, end, limit)
+			ctx.Convey("Then err should be nil.rs,id should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(id, convey.ShouldNotBeNil)
+				ctx.So(rs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoInsertRatingStat(t *testing.T) {
+	convey.Convey("InsertRatingStat", t, func(ctx convey.C) {
+		var (
+			c                 = context.Background()
+			month  time.Month = time.June
+			values            = "(1,2,'2018-06-01',100,100,100,600,600,300)"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.InsertRatingStat(c, month, values)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/dao/statistics.go

@@ -0,0 +1,76 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	_inRaingStatisticsSQL = "INSERT INTO up_rating_statistics(ups,section,tips,total_score,creativity_score,influence_score,credit_score,fans,avs,coin,play,tag_id,ctype,cdate) VALUES %s"
+	_inAscSQL             = "INSERT INTO up_rating_trend_%s(mid,tag_id,creativity_score,creativity_diff,influence_score,influence_diff,credit_score,credit_diff,magnetic_score,magnetic_diff,date,ctype,section,tips) VALUES %s ON DUPLICATE KEY UPDATE tag_id=VALUES(tag_id),creativity_score=VALUES(creativity_score),influence_score=VALUES(influence_score),credit_score=VALUES(credit_score)"
+	_inRatingTopSQL       = "INSERT INTO up_rating_top(mid,ctype,tag_id,score,fans,play,cdate) VALUES %s"
+
+	_delTrendSQL     = "DELETE FROM up_rating_trend_%s LIMIT ?"
+	_delRatingComSQL = "DELETE FROM %s WHERE cdate='%s' LIMIT ?"
+)
+
+// DelTrend del trend limit x
+func (d *Dao) DelTrend(c context.Context, table string, limit int) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_delTrendSQL, table), limit)
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_delTrendSQL, table), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// InsertRatingStatis batch insert rating statistics
+func (d *Dao) InsertRatingStatis(c context.Context, values string) (rows int64, err error) {
+	if values == "" {
+		return
+	}
+	res, err := d.db.Exec(c, fmt.Sprintf(_inRaingStatisticsSQL, values))
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_inRaingStatisticsSQL, values), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// InsertTrend insert asc values
+func (d *Dao) InsertTrend(c context.Context, table string, values string) (rows int64, err error) {
+	if values == "" {
+		return
+	}
+	res, err := d.db.Exec(c, fmt.Sprintf(_inAscSQL, table, values))
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_inAscSQL, table, values), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// InsertTopRating insert rating top
+func (d *Dao) InsertTopRating(c context.Context, values string) (rows int64, err error) {
+	if values == "" {
+		return
+	}
+	res, err := d.db.Exec(c, fmt.Sprintf(_inRatingTopSQL, values))
+	if err != nil {
+		log.Error("d.db.Exec(%s) error(%v)", fmt.Sprintf(_inRatingTopSQL, values), err)
+		return
+	}
+	return res.RowsAffected()
+}
+
+// DelRatingCom del rating common by date
+func (d *Dao) DelRatingCom(c context.Context, table string, date time.Time, limit int) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_delRatingComSQL, table, date.Format(_layout)), limit)
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/up-rating/dao/statistics_test.go

@@ -0,0 +1,75 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoDelTrend(t *testing.T) {
+	convey.Convey("DelTrend", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			table = "asc"
+			limit = int(100)
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "INSERT INTO up_rating_trend_asc(mid,tag_id) VALUES(1,2) ON DUPLICATE KEY UPDATE mid=VALUES(mid)")
+			rows, err := d.DelTrend(c, table, limit)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(rows, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoInsertRatingStatis(t *testing.T) {
+	convey.Convey("InsertRatingStatis", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			values = "(1,1,'10-20',160,60,50,50,100,100,100,200,2,1,'2018-06-01')"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "DELETE FROM up_rating_statistics WHERE tag_id=2")
+			_, err := d.InsertRatingStatis(c, values)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoInsertTopRating(t *testing.T) {
+	convey.Convey("InsertTopRating", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			values = "(1,2,3,100,100,100,'2018-06-01')"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.db.Exec(c, "DELETE FROM up_rating_top WHERE mid=1")
+			_, err := d.InsertTopRating(c, values)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoDelRatingCom(t *testing.T) {
+	convey.Convey("DelRatingCom", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			table = "up_rating_top"
+			date  = time.Now()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.DelRatingCom(c, table, date, 2000)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/dao/task_status.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+)
+
+const (
+	// insert
+	_inTaskStatusSQL = "INSERT INTO task_status(type, status, date, message) VALUES %s ON DUPLICATE KEY UPDATE status=VALUES(status), message=VALUES(message)"
+)
+
+// InsertTaskStatus insert task status
+func (d *Dao) InsertTaskStatus(c context.Context, val string) (rows int64, err error) {
+	if val == "" {
+		return
+	}
+	res, err := d.db.Exec(c, fmt.Sprintf(_inTaskStatusSQL, val))
+	if err != nil {
+		return
+	}
+	return res.RowsAffected()
+}

app/job/main/up-rating/dao/task_status_test.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoInsertTaskStatus(t *testing.T) {
+	convey.Convey("InsertTaskStatus", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			val = "(1,2,'2018-06-01','test')"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.InsertTaskStatus(c, val)
+			ctx.Convey("Then err should be nil.rows should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/job/main/up-rating/http/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "http.go",
+        "web.go",
+    ],
+    importpath = "go-common/app/job/main/up-rating/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//app/job/main/up-rating/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/http/http.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/up-rating/conf"
+	"go-common/app/job/main/up-rating/service"
+
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	svr *service.Service
+)
+
+// Init init http router.
+func Init(c *conf.Config) {
+	svr = service.New(conf.Conf)
+	// bm
+	engine := bm.DefaultServer(c.BM)
+	initRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func initRouter(e *bm.Engine) {
+	e.Ping(ping)
+	mr := e.Group("/x/internal/job/up-rating")
+	mr.POST("/past/score", pastScore)
+	mr.POST("/past/record", pastRecord)
+	mr.POST("/score", score)
+	mr.POST("/score/del", delScore)
+	mr.POST("/statistics", statistics)
+	mr.POST("/trend", trend)
+	mr.POST("/trend/del", delTrends)
+	mr.POST("/task/status", taskStatus)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := svr.Ping(c); err != nil {
+		log.Error("service ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+// Close http close
+func Close() {
+	svr.Close()
+}

app/job/main/up-rating/http/web.go

@@ -0,0 +1,149 @@
+package http
+
+import (
+	"time"
+
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+func pastScore(c *bm.Context) {
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	date, err := time.Parse("2006-01-02", v.Date)
+	if err != nil {
+		log.Error("pastScore date error(date:%s)", v.Date)
+		return
+	}
+	err = svr.RunPastScore(c, date)
+	if err != nil {
+		log.Error("svr.RunPastScore error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func pastRecord(c *bm.Context) {
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	err := svr.InsertPastRecord(c, v.Date)
+	if err != nil {
+		log.Error("svr.InsertPastRecord error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func score(c *bm.Context) {
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	date, err := time.Parse("2006-01-02", v.Date)
+	if err != nil {
+		log.Error("score date error(date:%s)", v.Date)
+		return
+	}
+	err = svr.Run(c, date)
+	if err != nil {
+		log.Error("svr.Run error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func delTrends(c *bm.Context) {
+	v := new(struct {
+		Table string `form:"table" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	err := svr.DelTrends(c, v.Table)
+	if err != nil {
+		log.Error("svr.DelTrends error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func delScore(c *bm.Context) {
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	date, err := time.Parse("2006-01-02", v.Date)
+	if err != nil {
+		log.Error("del score date error(date:%s)", v.Date)
+		return
+	}
+	err = svr.DelRatings(c, date)
+	if err != nil {
+		log.Error("svr.DelRatings error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func statistics(c *bm.Context) {
+	var err error
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err = c.Bind(v); err != nil {
+		return
+	}
+	date, err := time.Parse("2006-01-02", v.Date)
+	if err != nil {
+		log.Error("statistics date error(date:%s)", v.Date)
+		return
+	}
+	err = svr.RunStatistics(c, date)
+	if err != nil {
+		log.Error("svr.RunStatistics error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func trend(c *bm.Context) {
+	v := new(struct {
+		Date string `form:"date" validate:"required"`
+	})
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	date, err := time.Parse("2006-01-02", v.Date)
+	if err != nil {
+		log.Error("trend date error(date:%s)", v.Date)
+		return
+	}
+	err = svr.CalTrend(c, date)
+	if err != nil {
+		log.Error("svr.Trend error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func taskStatus(c *bm.Context) {
+	var err error
+	v := new(struct {
+		Date    string `form:"date" validate:"required"`
+		Type    int    `form:"type"`
+		Status  int    `form:"status"`
+		Message string `form:"message"`
+	})
+	if err = c.Bind(v); err != nil {
+		return
+	}
+	if err = svr.InsertTaskStatus(c, v.Type, v.Status, v.Date, v.Message); err != nil {
+		log.Error("svr.UpdateTaskStatus error(%v)", err)
+	}
+	c.JSON(nil, err)
+}

app/job/main/up-rating/model/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "rating.go",
+        "statistics.go",
+    ],
+    importpath = "go-common/app/job/main/up-rating/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/model/rating.go

@@ -0,0 +1,65 @@
+package model
+
+import "go-common/library/time"
+
+// BaseInfo up rating info
+type BaseInfo struct {
+	ID        int64
+	MID       int64
+	TagID     int64
+	PlayIncr  int64
+	CoinIncr  int64
+	Avs       int64
+	MAAFans   int64
+	MAHFans   int64
+	OpenAvs   int64
+	LockedAvs int64
+	Date      time.Time
+	TotalFans int64
+	TotalAvs  int64
+	TotalCoin int64
+	TotalPlay int64
+}
+
+// RatingParameter rating parameter
+type RatingParameter struct {
+	WDP      int64 // dp weight
+	WDC      int64 // dc weight
+	WDV      int64 // dv weight
+	WMDV     int64 // mdv weight
+	WCS      int64
+	WCSR     int64
+	WMAAFans int64
+	WMAHFans int64
+	WIS      int64
+	WISR     int64
+	// 信用分
+	HBASE int64
+	HR    int64
+	HV    int64
+	HVM   int64
+	HL    int64
+	HLM   int64
+}
+
+// Rating rating
+type Rating struct {
+	MID                 int64
+	TagID               int64
+	MetaCreativityScore int64
+	CreativityScore     int64
+	MetaInfluenceScore  int64
+	InfluenceScore      int64
+	CreditScore         int64
+	MagneticScore       int64
+	Score               int64
+	Date                time.Time
+}
+
+// Past past stat
+type Past struct {
+	MID                 int64
+	MetaCreativityScore int64
+	MetaInfluenceScore  int64
+	CreditScore         int64
+}

app/job/main/up-rating/model/statistics.go

@@ -0,0 +1,77 @@
+package model
+
+import "go-common/library/time"
+
+// RatingStatis rating statistics
+type RatingStatis struct {
+	Ups             int64
+	Section         int64
+	Tips            string
+	TotalScore      int64
+	CreativityScore int64
+	InfluenceScore  int64
+	CreditScore     int64
+	Fans            int64
+	Avs             int64
+	Coin            int64
+	Play            int64
+	CDate           time.Time
+	TagID           int64
+	CType           int
+}
+
+// Diff rating diff
+type Diff struct {
+	MID             int64
+	MagneticScore   int64
+	CreativityScore int64
+	InfluenceScore  int64
+	CreditScore     int64
+	MagneticDiff    int
+	CreativityDiff  int
+	InfluenceDiff   int
+	CreditDiff      int
+	TotalAvs        int64
+	Fans            int64
+	TagID           int64
+	CType           int
+	Section         int
+	Tips            string
+	Date            time.Time
+}
+
+// TopRating top rating
+type TopRating struct {
+	MID   int64
+	CType int
+	TagID int64
+	Score int64
+	Play  int64
+	Fans  int64
+}
+
+const (
+	// MAGNETIC magnetic ctype
+	MAGNETIC = iota
+	// CREATIVITY creativity ctype
+	CREATIVITY
+	// INFLUENCE influence ctype
+	INFLUENCE
+	// CREDIT influence ctype
+	CREDIT
+)
+
+// GetScore get score
+func (a *Diff) GetScore(ctype int) (score int64) {
+	switch ctype {
+	case MAGNETIC:
+		return a.MagneticScore
+	case CREATIVITY:
+		return a.CreativityScore
+	case INFLUENCE:
+		return a.InfluenceScore
+	case CREDIT:
+		return a.CreditScore
+	}
+	return
+}

app/job/main/up-rating/service/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "base.go",
+        "paramter.go",
+        "past.go",
+        "rating.go",
+        "run.go",
+        "score.go",
+        "service.go",
+        "statistics.go",
+        "task_status.go",
+        "top.go",
+        "trend.go",
+        "trend_heap.go",
+    ],
+    importpath = "go-common/app/job/main/up-rating/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/up-rating/conf:go_default_library",
+        "//app/job/main/up-rating/dao:go_default_library",
+        "//app/job/main/up-rating/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/golang.org/x/sync/errgroup:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/up-rating/service/base.go

@@ -0,0 +1,72 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// BaseInfoOffEnd get offset and end
+func (s *Service) BaseInfoOffEnd(c context.Context, date time.Time) (offset, end int, err error) {
+	start, err := s.dao.BaseInfoStart(c, date)
+	if err != nil {
+		return
+	}
+	offset = start - 1
+	end, err = s.dao.BaseInfoEnd(c, date)
+	return
+}
+
+// RatingOffEnd get offset and end
+func (s *Service) RatingOffEnd(c context.Context, date time.Time) (offset, end, count int, err error) {
+	start, err := s.dao.RatingStart(c, date)
+	if err != nil {
+		return
+	}
+	offset = start - 1
+	end, err = s.dao.RatingEnd(c, date)
+	if err != nil {
+		return
+	}
+	count, err = s.dao.RatingCount(c, date)
+	return
+}
+
+// BaseInfo get base infos
+func (s *Service) BaseInfo(c context.Context, date time.Time, start, end int, ch chan []*model.BaseInfo) (err error) {
+	defer close(ch)
+	for {
+		var bs []*model.BaseInfo
+		bs, start, err = s.dao.GetBaseInfo(c, date.Month(), start, end, _limit)
+		if err != nil {
+			return
+		}
+		if len(bs) == 0 {
+			break
+		}
+		ch <- bs
+	}
+	return
+}
+
+// BaseTotal get total base
+func (s *Service) BaseTotal(c context.Context, date time.Time) (total map[int64]*model.BaseInfo, err error) {
+	total = make(map[int64]*model.BaseInfo)
+	var id int64
+	for {
+		var bs []*model.BaseInfo
+		bs, err = s.dao.GetBaseTotal(c, date, id, int64(_limit))
+		if err != nil {
+			return
+		}
+		for _, b := range bs {
+			total[b.MID] = b
+		}
+		if len(bs) < _limit {
+			break
+		}
+		id = bs[len(bs)-1].ID
+	}
+	return
+}

app/job/main/up-rating/service/paramter.go

@@ -0,0 +1,33 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+func (s *Service) getAllParamter(c context.Context) (rp *model.RatingParameter, err error) {
+	paramters, err := s.dao.GetAllParamter(c)
+	if err != nil {
+		return
+	}
+	rp = &model.RatingParameter{
+		WDP:      paramters["wdp"],
+		WDC:      paramters["wdc"],
+		WDV:      paramters["wdv"],
+		WMDV:     paramters["wmdv"],
+		WCS:      paramters["wcs"],
+		WCSR:     paramters["wcsr"],
+		WMAAFans: paramters["wmaafans"],
+		WMAHFans: paramters["wmahfans"],
+		WIS:      paramters["wis"],
+		WISR:     paramters["wisr"],
+		HBASE:    paramters["hbase"],
+		HR:       paramters["hr"],
+		HV:       paramters["hv"],
+		HVM:      paramters["hvm"],
+		HL:       paramters["hl"],
+		HLM:      paramters["hlm"],
+	}
+	return
+}

app/job/main/up-rating/service/past.go

@@ -0,0 +1,210 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	_layout = "2006-01-02"
+	_limit  = 2000
+)
+
+// RunPastScore run past score by date
+func (s *Service) RunPastScore(c context.Context, date time.Time) (err error) {
+	date = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	times, err := s.getPastRecord(c, date.Format(_layout))
+	if err != nil {
+		log.Error("s.getPastRecord error(%v)", err)
+		return
+	}
+	if times < 0 {
+		log.Info("This month's calculation did not start")
+		return
+	}
+	// 创作力需要计算前22个月的数据
+	if times >= 22 {
+		log.Info("Last month's calculation has end")
+		return
+	}
+	var (
+		readGroup errgroup.Group
+		cw        float64 // 创作力当月权重
+		iw        int64   // 影响力当月权重
+		pastScore []*model.Past
+		pastCh    = make(chan []*model.Rating, _limit)
+	)
+	// 获取前n个月的数据
+	pastDate := date.AddDate(0, -1*(22-times), 0)
+	times++ // update calculate times
+
+	//csr = csm0 + csm1 + ... + csm11 + 11/12 * csm12 + 10/12 * csm13 + ... 1/12 * csm22
+	cw = float64(times) / float64(12)
+	if cw > 1.0 {
+		cw = 1.0
+	}
+	// isr = mfans0 + mfans1 + ... + mfans12
+	iw = int64(float64(times) / float64(12))
+	// get past month data
+	readGroup.Go(func() (err error) {
+		err = s.RatingInfos(c, pastDate, pastCh)
+		if err != nil {
+			log.Error("s.RatingInfos error(%v)", err)
+		}
+		return
+	})
+	// cal past month data
+	readGroup.Go(func() (err error) {
+		pastScore, err = s.calPastScores(c, pastCh, cw, iw)
+		if err != nil {
+			log.Error("s.calPastScores error(%v)", err)
+		}
+		return
+	})
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	err = s.insertPastRecord(c, times, date.Format(_layout))
+	if err != nil {
+		log.Error("s.upPastRecord error(%v)", err)
+		return
+	}
+
+	err = s.batchInsertPastScore(c, pastScore)
+	if err != nil {
+		log.Error("s.batchInsertPastScore error(%v)", err)
+	}
+	return
+}
+
+// InsertPastRecord insert past record
+func (s *Service) InsertPastRecord(c context.Context, date string) (err error) {
+	return s.insertPastRecord(c, 0, date)
+}
+
+func (s *Service) calPastScores(c context.Context, pastRating chan []*model.Rating, cw float64, iw int64) (pastScore []*model.Past, err error) {
+	pastScore = make([]*model.Past, 0)
+	for rating := range pastRating {
+		p := calPastScore(rating, cw, iw)
+		pastScore = append(pastScore, p...)
+	}
+	return
+}
+
+func calPastScore(rating []*model.Rating, cw float64, iw int64) (pastScore []*model.Past) {
+	pastScore = make([]*model.Past, 0, len(rating))
+	for _, r := range rating {
+		pastScore = append(pastScore, &model.Past{
+			MID:                 r.MID,
+			MetaCreativityScore: int64(float64(r.MetaCreativityScore) * cw),
+			MetaInfluenceScore:  r.MetaInfluenceScore * iw,
+			CreditScore:         r.CreditScore,
+		})
+	}
+	return
+}
+
+// get past calculate record
+func (s *Service) getPastRecord(c context.Context, date string) (times int, err error) {
+	return s.dao.GetPastRecord(c, date)
+}
+
+func (s *Service) insertPastRecord(c context.Context, times int, date string) (err error) {
+	_, err = s.dao.InsertPastRecord(c, times, date)
+	return err
+}
+
+func (s *Service) pastInfos(c context.Context) (past map[int64]*model.Past, err error) {
+	past = make(map[int64]*model.Past)
+	var id int64
+	for {
+		var p []*model.Past
+		p, id, err = s.dao.GetPasts(c, id, int64(_limit))
+		if err != nil {
+			return
+		}
+		for i := 0; i < len(p); i++ {
+			past[p[i].MID] = p[i]
+		}
+		if len(p) < _limit {
+			break
+		}
+	}
+	return
+}
+
+func (s *Service) batchInsertPastScore(c context.Context, past []*model.Past) (err error) {
+	var (
+		buff    = make([]*model.Past, 2000)
+		buffEnd = 0
+	)
+	for _, p := range past {
+		buff[buffEnd] = p
+		buffEnd++
+		if buffEnd >= 2000 {
+			values := assemblePastValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertPastScoreStat(c, values)
+			if err != nil {
+				return
+			}
+		}
+	}
+	if buffEnd > 0 {
+		values := assemblePastValues(buff[:buffEnd])
+		buffEnd = 0
+		_, err = s.dao.InsertPastScoreStat(c, values)
+	}
+	return
+}
+
+func assemblePastValues(past []*model.Past) (values string) {
+	var buf bytes.Buffer
+	for _, p := range past {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(p.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.MetaCreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.MetaInfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(p.CreditScore, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+func (s *Service) delOldPastInfo(c context.Context, limit int64) (err error) {
+	var rows int64
+	for {
+		rows, err = s.dao.DelPastStat(c, limit)
+		if err != nil {
+			return
+		}
+		if rows < limit {
+			break
+		}
+	}
+	return
+}
+
+// DelPastRecord del past record
+func (s *Service) DelPastRecord(c context.Context, date time.Time) (err error) {
+	_, err = s.dao.DelPastRecord(c, date)
+	return
+}

app/job/main/up-rating/service/rating.go

@@ -0,0 +1,127 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// DelRatings del ratings
+func (s *Service) DelRatings(c context.Context, date time.Time) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelRatings(c, date, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return s.DelPastRecord(c, date)
+}
+
+// RatingFast close chan when part finished
+func (s *Service) RatingFast(c context.Context, date time.Time, start, end int, ch chan []*model.Rating) (err error) {
+	defer close(ch)
+	return s.Ratings(c, date, start, end, ch)
+}
+
+// Ratings chan <- ratings, close chan outside
+func (s *Service) Ratings(c context.Context, date time.Time, start, end int, ch chan []*model.Rating) (err error) {
+	for {
+		var rs []*model.Rating
+		rs, start, err = s.dao.GetRatingsFast(c, date, start, end, _limit)
+		if err != nil {
+			return
+		}
+		if len(rs) == 0 {
+			break
+		}
+		ch <- rs
+	}
+	return
+}
+
+// RatingInfos rating infos
+func (s *Service) RatingInfos(c context.Context, date time.Time, ch chan []*model.Rating) (err error) {
+	defer close(ch)
+	var id int
+	for {
+		var rs []*model.Rating
+		rs, id, err = s.dao.GetRatings(c, date, id, _limit)
+		if err != nil {
+			return
+		}
+		ch <- rs
+		if len(rs) == 0 {
+			break
+		}
+	}
+	return
+}
+
+// BatchInsertRatingStat batch insert rating stat
+func (s *Service) BatchInsertRatingStat(c context.Context, wch chan []*model.Rating, date time.Time) (err error) {
+	var (
+		buff    = make([]*model.Rating, _limit)
+		buffEnd = 0
+	)
+	dateStr := date.Format(_layout)
+	for rs := range wch {
+		for _, r := range rs {
+			buff[buffEnd] = r
+			buffEnd++
+			if buffEnd >= _limit {
+				values := ratingStatValues(buff[:buffEnd], dateStr)
+				buffEnd = 0
+				_, err = s.dao.InsertRatingStat(c, date.Month(), values)
+				if err != nil {
+					return
+				}
+			}
+		}
+		if buffEnd > 0 {
+			values := ratingStatValues(buff[:buffEnd], dateStr)
+			buffEnd = 0
+			_, err = s.dao.InsertRatingStat(c, date.Month(), values)
+		}
+	}
+	return
+}
+
+func ratingStatValues(rs []*model.Rating, date string) (values string) {
+	var buf bytes.Buffer
+	for _, r := range rs {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(r.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", date))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MetaCreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MetaInfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MagneticScore, 10))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/up-rating/service/run.go

@@ -0,0 +1,164 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	"golang.org/x/sync/errgroup"
+)
+
+// Run run scores
+func (s *Service) Run(c context.Context, date time.Time) (err error) {
+	date = time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	// get weight parameters
+	params, err := s.getAllParamter(c)
+	if err != nil {
+		log.Error("s.getAllParamter error(%v)", err)
+		return
+	}
+	// get past scores
+	past, err := s.pastInfos(c)
+	if err != nil {
+		log.Error("s.pastInfos error(%v)", err)
+		return
+	}
+	err = s.PrepareData(c, date, past, params)
+	if err != nil {
+		log.Error("s.PrepareDate error(%v)", err)
+		return
+	}
+
+	err = s.CalScores(c, date, past, params)
+	if err != nil {
+		log.Error("s.CalScores error(%v)", err)
+		return
+	}
+
+	err = s.delOldPastInfo(c, int64(_limit))
+	if err != nil {
+		log.Error("s.delOldPastInfo error(%v)", err)
+		return
+	}
+
+	err = s.insertPastRecord(c, 0, date.AddDate(0, 1, 0).Format(_layout))
+	if err != nil {
+		log.Error("s.insertPastRecord error(%v)", err)
+		return
+	}
+
+	log.Info("run data read finished")
+	err = s.InsertTaskStatus(c, 0, 1, date.Format(_layout), "ok")
+	return
+}
+
+// PrepareData prepare old data
+func (s *Service) PrepareData(c context.Context, date time.Time, past map[int64]*model.Past, params *model.RatingParameter) (err error) {
+	var (
+		g         errgroup.Group
+		routines  = s.conf.Con.Concurrent
+		lastMonth = time.Date(date.Year(), date.Month()-1, 1, 0, 0, 0, 0, time.Local)
+	)
+
+	offset, end, _, err := s.RatingOffEnd(c, lastMonth)
+	if err != nil {
+		return
+	}
+	total := end - offset
+	section := (total - total%routines) / routines
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+
+		// read chan: for last ratings
+		rch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			err = s.RatingFast(c, lastMonth, begin, over, rch)
+			if err != nil {
+				log.Error("s.RatingFast error(%v)", err)
+			}
+			return
+		})
+
+		wch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			s.Copy(rch, wch, past, params)
+			return
+		})
+
+		g.Go(func() (err error) {
+			err = s.BatchInsertRatingStat(c, wch, date)
+			if err != nil {
+				log.Error("s.BatchInsertRatingStat error(%v)", err)
+			}
+			return
+		})
+	}
+
+	if err = g.Wait(); err != nil {
+		log.Error("run g.Wait error(%v)", err)
+		return
+	}
+	return
+}
+
+// CalScores cal scores
+func (s *Service) CalScores(c context.Context, date time.Time, past map[int64]*model.Past, params *model.RatingParameter) (err error) {
+	var (
+		routines = s.conf.Con.Concurrent
+		_limit   = s.conf.Con.Limit
+	)
+	t := time.Now().UnixNano()
+	var g errgroup.Group
+	// get id start:end by date
+	offset, end, err := s.BaseInfoOffEnd(c, date)
+	if err != nil {
+		return
+	}
+	total := end - offset
+	section := (total - total%routines) / routines
+	// parallelization and pipeling
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+		// read chan: for origin datas
+		rch := make(chan []*model.BaseInfo, _limit)
+		g.Go(func() (err error) {
+			err = s.BaseInfo(c, date, begin, over, rch)
+			if err != nil {
+				log.Error("s.BaseInfo error(%v)", err)
+			}
+			return
+		})
+
+		// write chan: for calculated results
+		wch := make(chan []*model.Rating, _limit)
+		g.Go(func() (err error) {
+			s.CalScore(rch, wch, params, past, date)
+			return
+		})
+
+		g.Go(func() (err error) {
+			err = s.BatchInsertRatingStat(c, wch, date)
+			if err != nil {
+				log.Error("s.BatchInsertRatingStat error(%v)", err)
+			}
+			return
+		})
+	}
+	if err = g.Wait(); err != nil {
+		log.Error("run g.Wait error(%v)", err)
+		return
+	}
+
+	log.Info("cal time cost:", time.Now().UnixNano()-t)
+	return
+}

app/job/main/up-rating/service/score.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"math"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+	xtime "go-common/library/time"
+)
+
+// Copy copy data from month to month
+func (s *Service) Copy(rch chan []*model.Rating, wch chan []*model.Rating, past map[int64]*model.Past, params *model.RatingParameter) {
+	defer close(wch)
+	for rs := range rch {
+		for _, r := range rs {
+			if _, ok := past[r.MID]; !ok {
+				continue
+			}
+			r.MetaCreativityScore = 0
+			csr := past[r.MID].MetaCreativityScore
+			if csr == 0 {
+				r.CreativityScore = 0
+			} else {
+				r.CreativityScore = int64(math.Min(float64(params.WCS)*math.Log(float64(csr)), float64(params.WCSR)))
+			}
+
+			r.MetaInfluenceScore = 0
+			isr := past[r.MID].MetaInfluenceScore
+			if isr == 0 {
+				r.InfluenceScore = 0
+			} else {
+				r.InfluenceScore = int64(math.Min(float64(params.WIS)*math.Log(float64(isr)), float64(params.WISR)))
+			}
+			r.Date = xtime.Time(time.Date(r.Date.Time().Year(), r.Date.Time().Month()+1, 1, 0, 0, 0, 0, time.Local).Unix())
+			r.MagneticScore = r.CreativityScore + r.InfluenceScore + r.CreditScore
+		}
+		wch <- rs
+	}
+}
+
+// CalScore cal rating score
+func (s *Service) CalScore(rch chan []*model.BaseInfo,
+	wch chan []*model.Rating,
+	params *model.RatingParameter,
+	past map[int64]*model.Past, date time.Time) {
+	defer close(wch)
+	for bs := range rch {
+		m := make([]*model.Rating, 0)
+		for _, b := range bs {
+			if !b.Date.Time().Equal(date) {
+				continue
+			}
+			r := &model.Rating{
+				MID:                 b.MID,
+				TagID:               b.TagID,
+				MetaCreativityScore: calCreativetyMetaScore(b, params),
+				CreativityScore:     calCreativityScore(b, params, past),
+				MetaInfluenceScore:  calInfluenceMetaScore(b, params),
+				InfluenceScore:      calInfluenceScore(b, params, past),
+				CreditScore:         calCreditScore(b, params, past),
+				Date:                b.Date,
+			}
+			r.MagneticScore = r.CreativityScore + r.InfluenceScore + r.CreditScore
+			m = append(m, r)
+		}
+		wch <- m
+	}
+}
+
+func calCreativetyMetaScore(b *model.BaseInfo, params *model.RatingParameter) int64 {
+	// ps: 当月播放分
+	ps := params.WDP*b.PlayIncr + params.WDC*b.CoinIncr
+	// ubs: 当月投稿低保分
+	ubs := params.WDV * int64(math.Min(float64(b.Avs), float64(params.WMDV)))
+	// csm: 当月创作力得分
+	csm := ps + ubs
+	return csm
+}
+
+func calCreativityScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	csm := calCreativetyMetaScore(b, params)
+	// csr: csm + past 创作力原始分
+	var csr int64
+	if _, ok := past[b.MID]; ok {
+		csr = csm + past[b.MID].MetaCreativityScore
+	} else {
+		csr = csm
+	}
+	if csr < 1 {
+		return 0
+	}
+	// cs: 创作力总分
+	cs := math.Min(float64(params.WCS)*math.Log(float64(csr)), float64(params.WCSR))
+	return int64(cs)
+}
+
+func calInfluenceMetaScore(b *model.BaseInfo, params *model.RatingParameter) int64 {
+	// mfans: 当月活跃粉丝数
+	mfans := params.WMAAFans*(b.MAAFans+b.MAHFans) + params.WMAHFans*b.MAHFans
+	return mfans
+}
+
+func calInfluenceScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	mfans := calInfluenceMetaScore(b, params)
+	// isr: 影响力原始分
+	var isr int64
+	if _, ok := past[b.MID]; ok {
+		isr = mfans + past[b.MID].MetaInfluenceScore
+	} else {
+		isr = mfans
+	}
+	if isr < 1 {
+		return 0
+	}
+	// is: up主影响力分
+	is := math.Min(float64(params.WIS)*math.Log(float64(isr)), float64(params.WISR))
+	return int64(is)
+}
+
+func calCreditScore(b *model.BaseInfo, params *model.RatingParameter, past map[int64]*model.Past) int64 {
+	addScore := min(b.OpenAvs*params.HV, params.HVM)
+	minusScore := min(b.LockedAvs*params.HL, params.HLM)
+	var cs int64
+	if _, ok := past[b.MID]; ok {
+		cs = past[b.MID].CreditScore + addScore - minusScore
+	} else {
+		cs = params.HBASE + addScore - minusScore
+	}
+	if cs < 0 {
+		cs = 0
+	}
+	return min(cs, params.HR)
+}
+
+func min(x, y int64) int64 {
+	if x > y {
+		return y
+	}
+	return x
+}

app/job/main/up-rating/service/service.go

@@ -0,0 +1,35 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/up-rating/conf"
+	"go-common/app/job/main/up-rating/dao"
+	"go-common/library/log"
+)
+
+// Service struct
+type Service struct {
+	conf *conf.Config
+	dao  *dao.Dao
+}
+
+// New fn
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		conf: c,
+		dao:  dao.New(c),
+	}
+	log.Info("service start")
+	return s
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close close the service
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/main/up-rating/service/statistics.go

@@ -0,0 +1,292 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+	xtime "go-common/library/time"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	// TotalType total type
+	TotalType int = iota
+	// CreativeType creative type
+	CreativeType
+	// InfluenceType influence type
+	InfluenceType
+	// CreditType Credit type
+	CreditType
+)
+
+var (
+	_offset         int64 = 10
+	_sumTotal       int64
+	_creativeTotal  int64
+	_influenceTotal int64
+	_creditTotal    int64
+)
+
+func initSection(min, max, section int64, ctype int, tagID int64, date xtime.Time) *model.RatingStatis {
+	return &model.RatingStatis{
+		Section: section,
+		Tips:    fmt.Sprintf("\"%d-%d\"", min, max),
+		TagID:   tagID,
+		CDate:   date,
+		CType:   ctype,
+	}
+}
+
+func initSections(totalScore, tagID int64, ctype int, date xtime.Time) (statis []*model.RatingStatis) {
+	var (
+		idx int64
+	)
+	statis = make([]*model.RatingStatis, totalScore/_offset)
+	for idx*_offset < totalScore {
+		statis[idx] = initSection(idx*_offset, (idx+1)*_offset, idx, ctype, tagID, date)
+		idx++
+	}
+	return
+}
+
+// RunStatistics run up rating statistics
+func (s *Service) RunStatistics(c context.Context, date time.Time) (err error) {
+	err = s.initTotalScore(c)
+	if err != nil {
+		log.Error("s.initTotalScore error(%v)", err)
+		return
+	}
+	err = s.delStatistics(c, date)
+	if err != nil {
+		log.Error("s.delStatistics error(%v)", err)
+		return
+	}
+	err = s.statistics(c, date)
+	if err != nil {
+		log.Error("s.scoreStatistics error(%v)", err)
+	}
+	return
+}
+
+func (s *Service) initTotalScore(c context.Context) (err error) {
+	params, err := s.getAllParamter(c)
+	if err != nil {
+		log.Error("s.getAllParamter error(%v)", err)
+		return
+	}
+	_sumTotal = params.WCSR + params.HR + params.WISR
+	_creativeTotal = params.WCSR
+	_influenceTotal = params.WISR
+	_creditTotal = params.HR
+	return
+}
+
+func (s *Service) delStatistics(c context.Context, date time.Time) (err error) {
+	err = s.delRatingCom(c, "up_rating_statistics", date)
+	if err != nil {
+		return
+	}
+	err = s.delRatingCom(c, "up_rating_top", date)
+	return
+}
+
+// delRatingCom del com
+func (s *Service) delRatingCom(c context.Context, table string, date time.Time) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelRatingCom(c, table, date, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return
+}
+
+func (s *Service) statistics(c context.Context, date time.Time) (err error) {
+	var (
+		readGroup errgroup.Group
+		sourceCh  = make(chan []*model.Rating, _limit)
+		statisCh  = make(chan []*model.Rating, _limit)
+		topCh     = make(chan []*model.Rating, _limit)
+
+		sections  map[int]map[int64][]*model.RatingStatis
+		topRating map[int]map[int64]*RatingHeap
+	)
+	baseInfo, err := s.BaseTotal(c, date)
+	if err != nil {
+		log.Error("s.BaseTotal error(%v)", err)
+		return
+	}
+	// get rating info
+	readGroup.Go(func() (err error) {
+		err = s.RatingInfos(c, date, sourceCh)
+		if err != nil {
+			log.Error("s.RatingInfos error(%v)", err)
+		}
+		return
+	})
+
+	// dispatch
+	readGroup.Go(func() (err error) {
+		defer func() {
+			close(topCh)
+			close(statisCh)
+		}()
+		for rating := range sourceCh {
+			statisCh <- rating
+			topCh <- rating
+		}
+		return
+	})
+
+	// top
+	readGroup.Go(func() (err error) {
+		topRating, err = s.ratingTop(c, date, topCh)
+		if err != nil {
+			log.Error("s.RatingTop error(%v)", err)
+		}
+		return
+	})
+	// statis
+	readGroup.Go(func() (err error) {
+		sections, err = s.scoreStatistics(c, date, statisCh, baseInfo)
+		if err != nil {
+			log.Error("s.scoreStatistics error(%v)", err)
+		}
+		return
+	})
+	if err = readGroup.Wait(); err != nil {
+		log.Error("run readGroup.Wait error(%v)", err)
+		return
+	}
+
+	// persistent
+	var writeGroup errgroup.Group
+
+	//up_rating_statistics
+	writeGroup.Go(func() (err error) {
+		err = s.insertSections(c, sections)
+		if err != nil {
+			log.Error("s.insertSections error(%v)", err)
+		}
+		return
+	})
+	// up_rating_top
+	writeGroup.Go(func() (err error) {
+		_, err = s.insertTopRating(c, date, topRating, baseInfo)
+		if err != nil {
+			log.Error("s.insertSections error(%v)", err)
+		}
+		return
+	})
+	if err = writeGroup.Wait(); err != nil {
+		log.Error("run writeGroup.Wait error(%v)", err)
+	}
+	return
+}
+
+func (s *Service) scoreStatistics(c context.Context, date time.Time, source chan []*model.Rating, baseInfo map[int64]*model.BaseInfo) (sections map[int]map[int64][]*model.RatingStatis, err error) {
+	sections = make(map[int]map[int64][]*model.RatingStatis) // map[ctype][tagID][]*model.RatingStatis
+	sections[TotalType] = make(map[int64][]*model.RatingStatis)
+	sections[CreativeType] = make(map[int64][]*model.RatingStatis)
+	sections[InfluenceType] = make(map[int64][]*model.RatingStatis)
+	sections[CreditType] = make(map[int64][]*model.RatingStatis)
+	for rating := range source {
+		for _, r := range rating {
+			statisScoreCtype(TotalType, r.CreativityScore+r.InfluenceScore+r.CreditScore, _sumTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(CreativeType, r.CreativityScore, _creativeTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(InfluenceType, r.InfluenceScore, _influenceTotal, sections, date, r, baseInfo[r.MID])
+			statisScoreCtype(CreditType, r.CreditScore, _creditTotal, sections, date, r, baseInfo[r.MID])
+		}
+	}
+	return
+}
+
+func statisScoreCtype(ctype int, score, totalScore int64, sections map[int]map[int64][]*model.RatingStatis, date time.Time, rate *model.Rating, base *model.BaseInfo) {
+	if _, ok := sections[ctype][rate.TagID]; !ok {
+		sections[ctype][rate.TagID] = initSections(totalScore, rate.TagID, ctype, xtime.Time(date.Unix()))
+	}
+	idx := score / _offset
+	if idx >= int64(len(sections[ctype][rate.TagID])) {
+		idx = int64(len(sections[ctype][rate.TagID]) - 1)
+	}
+	sections[ctype][rate.TagID][idx].Ups++
+	sections[ctype][rate.TagID][idx].TotalScore += rate.CreativityScore + rate.InfluenceScore + rate.CreditScore
+	sections[ctype][rate.TagID][idx].CreativityScore += rate.CreativityScore
+	sections[ctype][rate.TagID][idx].InfluenceScore += rate.InfluenceScore
+	sections[ctype][rate.TagID][idx].CreditScore += rate.CreditScore
+	if base != nil {
+		sections[ctype][rate.TagID][idx].Fans += base.TotalFans
+		sections[ctype][rate.TagID][idx].Avs += base.TotalAvs
+		sections[ctype][rate.TagID][idx].Coin += base.TotalCoin
+		sections[ctype][rate.TagID][idx].Play += base.TotalPlay
+	}
+}
+
+func (s *Service) insertSections(c context.Context, sections map[int]map[int64][]*model.RatingStatis) (err error) {
+	for ctype, tags := range sections {
+		for tagID, statis := range tags {
+			_, err = s.insertRatingStatis(c, ctype, tagID, statis)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) insertRatingStatis(c context.Context, ctype int, tagID int64, statis []*model.RatingStatis) (rows int64, err error) {
+	return s.dao.InsertRatingStatis(c, assembleRatingStatis(c, ctype, tagID, statis))
+}
+
+func assembleRatingStatis(c context.Context, ctype int, tagID int64, statis []*model.RatingStatis) (vals string) {
+	var buf bytes.Buffer
+	for _, s := range statis {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(s.Ups, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Section, 10))
+		buf.WriteByte(',')
+		buf.WriteString(s.Tips)
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.TotalScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Fans, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Avs, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Coin, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.Play, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(s.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(s.CType))
+		buf.WriteByte(',')
+		buf.WriteString("'" + s.CDate.Time().Format(_layout) + "'")
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}

app/job/main/up-rating/service/task_status.go

@@ -0,0 +1,12 @@
+package service
+
+import (
+	"context"
+	"fmt"
+)
+
+// InsertTaskStatus insert task status
+func (s *Service) InsertTaskStatus(c context.Context, ctype, status int, date, message string) (err error) {
+	_, err = s.dao.InsertTaskStatus(c, fmt.Sprintf("(%d, %d, '%s', '%s')", ctype, status, date, message))
+	return
+}

app/job/main/up-rating/service/top.go

@@ -0,0 +1,116 @@
+package service
+
+import (
+	"bytes"
+	"container/heap"
+	"context"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// ratingTop get top ups
+func (s *Service) ratingTop(c context.Context, date time.Time, source chan []*model.Rating) (topRating map[int]map[int64]*RatingHeap, err error) {
+	topRating = make(map[int]map[int64]*RatingHeap) // map[ctype][tagID]
+	topRating[CreativeType] = make(map[int64]*RatingHeap)
+	topRating[InfluenceType] = make(map[int64]*RatingHeap)
+	for rating := range source {
+		for _, r := range rating {
+			if _, ok := topRating[CreativeType][r.TagID]; !ok {
+				topRating[CreativeType][r.TagID] = &RatingHeap{}
+			}
+			pushTopRating(topRating[CreativeType][r.TagID], CreativeType, r)
+			if _, ok := topRating[InfluenceType][r.TagID]; !ok {
+				topRating[InfluenceType][r.TagID] = &RatingHeap{}
+			}
+			pushTopRating(topRating[InfluenceType][r.TagID], InfluenceType, r)
+		}
+	}
+	return
+}
+
+func pushTopRating(h *RatingHeap, ctype int, r *model.Rating) {
+	tr := &model.TopRating{
+		MID:   r.MID,
+		CType: ctype,
+		TagID: r.TagID,
+	}
+	switch ctype {
+	case CreativeType:
+		tr.Score = r.CreativityScore
+	case InfluenceType:
+		tr.Score = r.InfluenceScore
+	}
+	heap.Push(h, tr)
+	if h.Len() > 10 {
+		heap.Pop(h)
+	}
+}
+
+func (s *Service) insertTopRating(c context.Context, date time.Time, topRating map[int]map[int64]*RatingHeap, baseInfo map[int64]*model.BaseInfo) (rows int64, err error) {
+	return s.dao.InsertTopRating(c, assemberTopRating(date, topRating, baseInfo))
+}
+
+func assemberTopRating(date time.Time, topRating map[int]map[int64]*RatingHeap, baseInfo map[int64]*model.BaseInfo) (vals string) {
+	var buf bytes.Buffer
+	for _, tagTop := range topRating {
+		for _, h := range tagTop {
+			for h.Len() > 0 {
+				tr := heap.Pop(h).(*model.TopRating)
+				info := baseInfo[tr.MID]
+				if info == nil {
+					info = &model.BaseInfo{}
+				}
+				buf.WriteString("(")
+				buf.WriteString(strconv.FormatInt(tr.MID, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.Itoa(tr.CType))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(tr.TagID, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(tr.Score, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(info.TotalFans, 10))
+				buf.WriteByte(',')
+				buf.WriteString(strconv.FormatInt(info.TotalPlay, 10))
+				buf.WriteByte(',')
+				buf.WriteString("'" + date.Format(_layout) + "'")
+				buf.WriteString(")")
+				buf.WriteByte(',')
+			}
+		}
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	vals = buf.String()
+	buf.Reset()
+	return
+}
+
+// RatingHeap rating heap for topK
+type RatingHeap []*model.TopRating
+
+// Len len
+func (r RatingHeap) Len() int { return len(r) }
+
+// Less less
+func (r RatingHeap) Less(i, j int) bool { return r[i].Score < r[j].Score }
+
+// Swap swap
+func (r RatingHeap) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+
+// Push push to heap
+func (r *RatingHeap) Push(x interface{}) {
+	*r = append(*r, x.(*model.TopRating))
+}
+
+// Pop pop from heap
+func (r *RatingHeap) Pop() interface{} {
+	old := *r
+	n := len(old)
+	x := old[n-1]
+	*r = old[0 : n-1]
+	return x
+}

app/job/main/up-rating/service/trend.go

@@ -0,0 +1,444 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/up-rating/model"
+
+	"go-common/library/log"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// CalTrend cal trend
+func (s *Service) CalTrend(c context.Context, date time.Time) (err error) {
+	return s.calTrend(c, date)
+}
+
+func (s *Service) calTrend(c context.Context, date time.Time) (err error) {
+	ds, err := s.getDiffs(c, date)
+	if err != nil {
+		return
+	}
+
+	var (
+		magneticSec  = sections(TotalType)
+		creativeSec  = sections(CreativeType)
+		influenceSec = sections(InfluenceType)
+		creditSec    = sections(CreditType)
+		g            errgroup.Group
+
+		// for concurrent write
+		_ascMagneticCh  = make(chan []*model.Diff, 5)
+		_ascCreativeCh  = make(chan []*model.Diff, 5)
+		_ascInfluenceCh = make(chan []*model.Diff, 5)
+		_ascCreditCh    = make(chan []*model.Diff, 5)
+
+		_descMagneticCh  = make(chan []*model.Diff, 5)
+		_descCreativeCh  = make(chan []*model.Diff, 5)
+		_descInfluenceCh = make(chan []*model.Diff, 5)
+		_descCreditCh    = make(chan []*model.Diff, 5)
+	)
+
+	// magnetic ascend
+	g.Go(func() (err error) {
+		defer close(_ascMagneticCh)
+		mr := classify(ds, TotalType, "asc", magneticSec)
+		push(_ascMagneticCh, mr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascMagneticCh)
+		return
+	})
+
+	// creativity ascend
+	g.Go(func() (err error) {
+		defer close(_ascCreativeCh)
+		cr := classify(ds, CreativeType, "asc", creativeSec)
+		push(_ascCreativeCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascCreativeCh)
+		return
+	})
+
+	// influence ascend
+	g.Go(func() (err error) {
+		defer close(_ascInfluenceCh)
+		ir := classify(ds, InfluenceType, "asc", influenceSec)
+		push(_ascInfluenceCh, ir)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascInfluenceCh)
+		return
+	})
+
+	// credit ascend
+	g.Go(func() (err error) {
+		defer close(_ascCreditCh)
+		cr := classify(ds, CreditType, "asc", creditSec)
+		push(_ascCreditCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "asc", _ascCreditCh)
+		return
+	})
+
+	// magnetic descend
+	g.Go(func() (err error) {
+		defer close(_descMagneticCh)
+		mr := classify(ds, TotalType, "desc", magneticSec)
+		push(_descMagneticCh, mr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descMagneticCh)
+		return
+	})
+
+	// creativity descend
+	g.Go(func() (err error) {
+		defer close(_descCreativeCh)
+		cr := classify(ds, CreativeType, "desc", creativeSec)
+		push(_descCreativeCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descCreativeCh)
+		return
+	})
+
+	// influence descend
+	g.Go(func() (err error) {
+		defer close(_descInfluenceCh)
+		ir := classify(ds, InfluenceType, "desc", influenceSec)
+		push(_descInfluenceCh, ir)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descInfluenceCh)
+		return
+	})
+
+	// credit descend
+	g.Go(func() (err error) {
+		defer close(_descCreditCh)
+		cr := classify(ds, CreditType, "desc", creditSec)
+		push(_descCreditCh, cr)
+		return
+	})
+
+	g.Go(func() (err error) {
+		err = s.batchInsertDiffs(c, "desc", _descCreditCh)
+		return
+	})
+
+	if err = g.Wait(); err != nil {
+		log.Error("g.Wait error(%v)", err)
+	}
+	return
+}
+
+func push(ch chan []*model.Diff, m map[int64]map[int]Heap) {
+	for _, sm := range m {
+		for _, h := range sm {
+			ch <- h.Result()
+		}
+	}
+}
+
+func (s *Service) getDiffs(c context.Context, date time.Time) (ds map[int64][]*model.Diff, err error) {
+	lastMonth := time.Date(date.Year(), date.Month()-1, 1, 0, 0, 0, 0, time.Local)
+	last, err := s.getR(c, lastMonth)
+	if err != nil {
+		return
+	}
+
+	curMonth := time.Date(date.Year(), date.Month(), 1, 0, 0, 0, 0, time.Local)
+	cur, err := s.getR(c, curMonth)
+	if err != nil {
+		return
+	}
+	ds = diff(last, cur)
+	return
+}
+
+func sections(ctype int) (ss []*section) {
+	switch ctype {
+	case TotalType:
+		ss = initSec(10, 600)
+	case CreativeType, InfluenceType, CreditType:
+		ss = initSec(10, 200)
+	}
+	return
+}
+
+type section struct {
+	start int64
+	end   int64
+	tips  string
+}
+
+func initSec(c, total int64) (ss []*section) {
+	size := total / c
+	for index := int64(0); index < c; index++ {
+		start := index * size
+		end := start + size - 1
+		if index == c-1 {
+			end = total
+		}
+		sec := &section{
+			start: start,
+			end:   end,
+			tips:  fmt.Sprintf("\"%d-%d\"", start, end),
+		}
+		ss = append(ss, sec)
+	}
+	return
+}
+
+func getSection(score int64, ss []*section) (index int, tips string) {
+	for index, section := range ss {
+		if score >= section.start && score <= section.end {
+			return index, section.tips
+		}
+	}
+	return
+}
+
+func getHeap(order string, ctype int) Heap {
+	if order == "asc" {
+		return &AscHeap{heap: make([]*model.Diff, 0), ctype: ctype}
+	}
+	return &DescHeap{heap: make([]*model.Diff, 0), ctype: ctype}
+}
+
+// map[tag_id]map[section][]diffs
+func classify(ds map[int64][]*model.Diff, ctype int, order string, ss []*section) (m map[int64]map[int]Heap) {
+	m = make(map[int64]map[int]Heap)
+	for tagID, diffs := range ds {
+		for _, diff := range diffs {
+			sec, tips := getSection(diff.GetScore(ctype), ss)
+			// need clone
+			_diff := clone(diff)
+			_diff.Section = sec
+			_diff.CType = ctype
+			_diff.Tips = tips
+			if sm, ok := m[tagID]; ok {
+				if _, ok = sm[sec]; ok {
+					sm[sec].Put(_diff)
+				} else {
+					h := getHeap(order, ctype)
+					h.Put(_diff)
+					sm[sec] = h
+				}
+			} else {
+				h := getHeap(order, ctype)
+				h.Put(_diff)
+				sm := make(map[int]Heap)
+				sm[sec] = h
+				m[tagID] = sm
+			}
+		}
+	}
+	return
+}
+
+func clone(a *model.Diff) *model.Diff {
+	return &model.Diff{
+		MID:             a.MID,
+		MagneticScore:   a.MagneticScore,
+		CreativityScore: a.CreativityScore,
+		InfluenceScore:  a.InfluenceScore,
+		CreditScore:     a.CreditScore,
+		MagneticDiff:    a.MagneticDiff,
+		CreativityDiff:  a.CreativityDiff,
+		InfluenceDiff:   a.InfluenceDiff,
+		CreditDiff:      a.CreditDiff,
+		TotalAvs:        a.TotalAvs,
+		Fans:            a.Fans,
+		TagID:           a.TagID,
+		Date:            a.Date,
+	}
+}
+
+// ds map[tag_id][]*model.Diff
+func diff(last map[int64]*model.Rating, cur map[int64]*model.Rating) (ds map[int64][]*model.Diff) {
+	ds = make(map[int64][]*model.Diff)
+	for mid, r := range cur {
+		if _, ok := last[mid]; !ok {
+			continue
+		}
+		lr := last[mid]
+		diff := &model.Diff{
+			MID:             mid,
+			TagID:           r.TagID,
+			MagneticScore:   r.MagneticScore,
+			CreativityScore: r.CreativityScore,
+			InfluenceScore:  r.InfluenceScore,
+			CreditScore:     r.CreditScore,
+			MagneticDiff:    int(r.MagneticScore - lr.MagneticScore),
+			CreativityDiff:  int(r.CreativityScore - lr.CreativityScore),
+			InfluenceDiff:   int(r.InfluenceScore - lr.InfluenceScore),
+			CreditDiff:      int(r.CreditScore - lr.CreditScore),
+			Date:            r.Date,
+		}
+		if _, ok := ds[diff.TagID]; ok {
+			ds[diff.TagID] = append(ds[diff.TagID], diff)
+		} else {
+			ds[diff.TagID] = []*model.Diff{diff}
+		}
+	}
+	return
+}
+
+// GetR m[mid]*model.Rating
+func (s *Service) getR(c context.Context, date time.Time) (m map[int64]*model.Rating, err error) {
+	var (
+		g        errgroup.Group
+		routines = 5 // 4 core + 1
+		ch       = make(chan []*model.Rating, routines)
+	)
+
+	m = make(map[int64]*model.Rating)
+	offset, end, total, err := s.RatingOffEnd(c, date)
+	if err != nil {
+		return
+	}
+	if total == 0 {
+		return
+	}
+	t := end - offset
+	section := (t - t%routines) / routines
+	for i := 0; i < routines; i++ {
+		begin := section*i + offset
+		over := begin + section
+		if i == routines-1 {
+			over = end
+		}
+		g.Go(func() (err error) {
+			err = s.Ratings(c, date, begin, over, ch)
+			if err != nil {
+				log.Error("get rating infos error(%v)", err)
+			}
+			return
+		})
+	}
+
+	g.Go(func() (err error) {
+	Loop:
+		for rs := range ch {
+			for _, r := range rs {
+				m[r.MID] = r
+			}
+			if len(m) == total {
+				break Loop
+			}
+		}
+		return
+	})
+	if err = g.Wait(); err != nil {
+		log.Error("get rating wait error(%v)", err)
+	}
+	return
+}
+
+// BatchInsertDiffs batch insert diffs
+func (s *Service) batchInsertDiffs(c context.Context, table string, wch chan []*model.Diff) (err error) {
+	var (
+		buff    = make([]*model.Diff, _limit)
+		buffEnd = 0
+	)
+	for ds := range wch {
+		for _, d := range ds {
+			buff[buffEnd] = d
+			buffEnd++
+			if buffEnd >= _limit {
+				values := diffValues(buff[:buffEnd])
+				buffEnd = 0
+				_, err = s.dao.InsertTrend(c, table, values)
+				if err != nil {
+					return
+				}
+			}
+		}
+		if buffEnd > 0 {
+			values := diffValues(buff[:buffEnd])
+			buffEnd = 0
+			_, err = s.dao.InsertTrend(c, table, values)
+		}
+	}
+	return
+}
+
+func diffValues(ds []*model.Diff) (values string) {
+	var buf bytes.Buffer
+	for _, r := range ds {
+		buf.WriteString("(")
+		buf.WriteString(strconv.FormatInt(r.MID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.TagID, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreativityScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CreativityDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.InfluenceScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.InfluenceDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.CreditScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CreditDiff))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.FormatInt(r.MagneticScore, 10))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.MagneticDiff))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", r.Date.Time().Format(_layout)))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.CType))
+		buf.WriteByte(',')
+		buf.WriteString(strconv.Itoa(r.Section))
+		buf.WriteByte(',')
+		buf.WriteString(fmt.Sprintf("'%s'", r.Tips))
+		buf.WriteString(")")
+		buf.WriteByte(',')
+	}
+	if buf.Len() > 0 {
+		buf.Truncate(buf.Len() - 1)
+	}
+	values = buf.String()
+	buf.Reset()
+	return
+}
+
+// DelTrends del trends
+func (s *Service) DelTrends(c context.Context, table string) (err error) {
+	for {
+		var rows int64
+		rows, err = s.dao.DelTrend(c, table, _limit)
+		if err != nil {
+			return
+		}
+		if rows == 0 {
+			break
+		}
+	}
+	return
+}

app/job/main/up-rating/service/trend_heap.go

@@ -0,0 +1,101 @@
+package service
+
+import (
+	"container/heap"
+
+	"go-common/app/job/main/up-rating/model"
+)
+
+// Heap for diff topK
+type Heap interface {
+	Put(*model.Diff)
+	Result() []*model.Diff
+}
+
+// AscHeap for asc
+type AscHeap struct {
+	heap  []*model.Diff
+	ctype int
+}
+
+// Put put diff to heap
+func (a *AscHeap) Put(diff *model.Diff) {
+	heap.Push(a, diff)
+	if a.Len() > 200 {
+		heap.Pop(a)
+	}
+}
+
+// Result get result
+func (a *AscHeap) Result() []*model.Diff {
+	return a.heap
+}
+
+// Len len
+func (a *AscHeap) Len() int { return len(a.heap) }
+
+// Less less
+func (a *AscHeap) Less(i, j int) bool {
+	return a.heap[i].GetScore(a.ctype) < a.heap[j].GetScore(a.ctype)
+}
+
+// Swap swap
+func (a *AscHeap) Swap(i, j int) { a.heap[i], a.heap[j] = a.heap[j], a.heap[i] }
+
+// Push push to heap
+func (a *AscHeap) Push(x interface{}) {
+	a.heap = append(a.heap, x.(*model.Diff))
+}
+
+// Pop pop from heap
+func (a *AscHeap) Pop() interface{} {
+	old := a.heap
+	n := len(old)
+	x := old[n-1]
+	a.heap = old[0 : n-1]
+	return x
+}
+
+// DescHeap for desc
+type DescHeap struct {
+	heap  []*model.Diff
+	ctype int
+}
+
+// Put to descHeap
+func (d *DescHeap) Put(diff *model.Diff) {
+	heap.Push(d, diff)
+	if d.Len() > 200 {
+		heap.Pop(d)
+	}
+}
+
+// Result desc heap result
+func (d *DescHeap) Result() []*model.Diff {
+	return d.heap
+}
+
+// Len len
+func (d *DescHeap) Len() int { return len(d.heap) }
+
+// Less less
+func (d *DescHeap) Less(i, j int) bool {
+	return d.heap[i].GetScore(d.ctype) > d.heap[j].GetScore(d.ctype)
+}
+
+// Swap swap
+func (d *DescHeap) Swap(i, j int) { d.heap[i], d.heap[j] = d.heap[j], d.heap[i] }
+
+// Push push to desc heap
+func (d *DescHeap) Push(x interface{}) {
+	d.heap = append(d.heap, x.(*model.Diff))
+}
+
+// Pop pop from desc heap
+func (d *DescHeap) Pop() interface{} {
+	old := d.heap
+	n := len(old)
+	x := old[n-1]
+	d.heap = old[0 : n-1]
+	return x
+}