Create & Init Project...

app/job/main/playlist/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/playlist/cmd:all-srcs",
+        "//app/job/main/playlist/conf:all-srcs",
+        "//app/job/main/playlist/dao:all-srcs",
+        "//app/job/main/playlist/http:all-srcs",
+        "//app/job/main/playlist/model:all-srcs",
+        "//app/job/main/playlist/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/CHANGELOG.md

@@ -0,0 +1,58 @@
+
+# playlist-job
+
+
+### v1.4.2
+##### Features
+1. waitgroup
+
+### v1.4.1
+##### Features
+> 1.修改bm
+
+### v1.4.0
+##### Features
+> 1.use bm
+
+### v1.3.0
+##### Features
+> 1.迁移main目录
+
+### v1.2.1
+##### Bug Fixes
+> 1.fix close of closed channel
+
+### v1.2.0
+##### Features
+> 1.Stat-T拆分消费
+
+### v1.1.1
+##### Features
+> 1.add log
+
+### v1.1.0
+##### Features
+> 1.添加分享消费databus
+> 2.播放数与分享数改为绝对值更新
+
+### v1.0.4
+##### Bug Fixes
+> 1.fix send on closed channel
+> 2.更新计数不更新播单缓存mtime
+
+### v1.0.3
+##### Features
+> 1.更新计数不更新播单修改时间
+> 2.log 修改
+
+### v1.0.2
+##### Features
+> 1.上报过滤修改
+
+### v1.0.1
+##### Features
+> 1.加普罗米修期上报
+
+### v1.0.0
+##### Features
+> 1.项目初始化

app/job/main/playlist/CONTRIBUTORS.md

@@ -0,0 +1,10 @@
+# Owner
+liweijia
+
+# Author 
+guanyanliang
+liweijia
+wuhao02
+
+# Reviewer
+zhapuyu

app/job/main/playlist/OWNERS

@@ -0,0 +1,17 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- guanyanliang
+- liweijia
+- wuhao02
+labels:
+- job
+- job/main/playlist
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- guanyanliang
+- liweijia
+- wuhao02
+- zhapuyu

app/job/main/playlist/README.md

@@ -0,0 +1,10 @@
+#### playlist-job
+
+##### 项目简介
+> 1.播单job
+
+##### 编译环境
+> 请只用golang v1.7.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  

app/job/main/playlist/cmd/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["playlist-job-test.toml"],
+    importpath = "go-common/app/job/main/playlist/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/http:go_default_library",
+        "//app/job/main/playlist/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/cmd/goconvey.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+command -v goconvey >/dev/null 2>&1 || { echo >&2 "required goconvey but it's not installed."; echo "Aborting."; echo "Please run commond: go get github.com/smartystreets/goconvey"; exit 1; }
+
+cd ../
+goconvey -excludedDirs "vendor" -packages 1

app/job/main/playlist/cmd/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/http"
+	"go-common/app/job/main/playlist/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	initConf()
+	initLog()
+	defer log.Close()
+	log.Info("playlist-job start")
+	srv := service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	initSignal(srv)
+}
+
+func initConf() {
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+}
+
+func initLog() {
+	log.Init(conf.Conf.Log)
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+}
+
+func initSignal(srv *service.Service) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("playlist-job get a signal: %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			if err := srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+			// TODO: reload
+		default:
+			return
+		}
+	}
+}

app/job/main/playlist/cmd/playlist-job-test.toml

@@ -0,0 +1,140 @@
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/playlist-job.pid"
+env="dev" # dev or pro
+dir = "./"
+family = "playlist-job"
+
+[log]
+    family = "playlist-job"
+    dir = "/data/log/playlist-job/"
+
+[app]
+    key = "b1014d7c339a5649"
+    secret = "75b74b612aa792b112e6504cae44c319"
+
+
+[HTTPServer]
+    addr = "0.0.0.0:7201"
+    timeout = "1s"
+
+[HTTPClient]
+    key = "b1014d7c339a5649"
+    secret = "75b74b612aa792b112e6504cae44c319"
+    dial      = "50ms"
+    timeout   = "1s"
+    keepAlive = "60s"
+
+
+[playlistViewSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "StatView-MainWebSvr-S"
+    topic = "StatView-T"
+    action = "sub"
+    name = "playlist-job/playlist-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[playlistFavSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "StatFav-MainWebSvr-S"
+    topic = "StatFav-T"
+    action = "sub"
+    name = "playlist-job/playlist-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[playlistReplySub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "StatReply-MainWebSvr-S"
+    topic = "StatReply-T"
+    action = "sub"
+    name = "playlist-job/playlist-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[playlistShareSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "StatShare-MainWebSvr-S"
+    topic = "StatShare-T"
+    action = "sub"
+    name = "playlist-job/playlist-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[playlistRPC]
+    pullInterval = "10s"
+    [playlistRPC.client]
+    proto = "tcp"
+    timeout = "1s"
+    timer = 1000
+    [playlistRPC.client.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [playlistRPC.zookeeper]
+    root = "/microservice/playlist-service/"
+    addrs =  ["172.18.33.172:2181"]
+    timeout = "30s"
+
+[mysql]
+    addr = "172.16.33.205"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_playlist?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout = "4h"
+    queryTimeout = "100ms"
+    execTimeout = "100ms"
+    tranTimeout = "200ms"
+    [mysql.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+    name = "playlist-job"
+    proto = "tcp"
+    addr = "172.16.33.54:6379"
+    idle = 10
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[job]
+    updateDbInterval = "0m"
+    interceptOn = false
+    viewCacheTTL = "30m"

app/job/main/playlist/conf/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/playlist/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/conf/conf.go

@@ -0,0 +1,111 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	xlog "go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// Config .
+type Config struct {
+	// Env .
+	Env string
+	// App .
+	App *bm.App
+	// Log is go-common log.
+	Log *xlog.Config
+	// Tracer .
+	Tracer *trace.Config
+	// PlaylistStatSub databus.
+	PlaylistViewSub  *databus.Config
+	PlaylistFavSub   *databus.Config
+	PlaylistReplySub *databus.Config
+	PlaylistShareSub *databus.Config
+	// HTTPServer .
+	HTTPServer *bm.ServerConfig
+	// HTTPClient .
+	HTTPClient *bm.ClientConfig
+	// RPC .
+	PlaylistRPC *rpc.ClientConfig
+	// Mysql .
+	Mysql *sql.Config
+	// Redis .
+	Redis *redis.Config
+	// Job params .
+	Job *job
+}
+
+type job struct {
+	InterceptOn      bool
+	ViewCacheTTL     xtime.Duration
+	UpdateDbInterval xtime.Duration
+}
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config.
+	Conf = &Config{}
+)
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init .
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			xlog.Info("config reload")
+			if load() != nil {
+				xlog.Error("config reload err")
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/playlist/dao/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/playlist/model:go_default_library",
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/model:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/playlist/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/playlist/model:go_default_library",
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/dao/dao.go

@@ -0,0 +1,61 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/main/playlist/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/stat/prom"
+)
+
+// Dao .
+type Dao struct {
+	c            *conf.Config
+	db           *xsql.DB
+	redis        *redis.Pool
+	httpClient   *bm.Client
+	viewCacheTTL int64
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:            c,
+		db:           xsql.NewMySQL(c.Mysql),
+		redis:        redis.NewPool(c.Redis),
+		httpClient:   bm.NewClient(c.HTTPClient),
+		viewCacheTTL: int64(time.Duration(c.Job.ViewCacheTTL) / time.Second),
+	}
+	return
+}
+
+// PromError stat and log.
+func PromError(name string, format string, args ...interface{}) {
+	prom.BusinessErrCount.Incr(name)
+	log.Error(format, args...)
+}
+
+// PromInfo prometheus info count.
+func PromInfo(name string) {
+	prom.BusinessInfoCount.Incr(name)
+}
+
+// Ping reports the health of the db/cache etc.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.db.Ping(c); err != nil {
+		return
+	}
+	err = d.pingRedis(c)
+	return
+}
+
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}

app/job/main/playlist/dao/dao_test.go

@@ -0,0 +1,22 @@
+package dao
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/playlist/conf"
+)
+
+var d *Dao
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../cmd/playlist-job-test.toml")
+		flag.Set("conf", dir)
+		conf.Init()
+		if d == nil {
+			d = New(conf.Conf)
+		}
+		f(d)
+	}
+}

app/job/main/playlist/dao/mysql.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+
+	plmdl "go-common/app/interface/main/playlist/model"
+	"go-common/app/job/main/playlist/model"
+)
+
+const (
+	_statSQL    = "SELECT id,mid,fid,view,reply,fav,share,mtime FROM `playlist_stat` WHERE id = ?"
+	_upViewSQL  = "UPDATE playlist_stat SET  view= ?, mtime=mtime  WHERE id = ?"
+	_upFavSQL   = "UPDATE playlist_stat SET  fav= ?, mtime=mtime  WHERE id = ?"
+	_upReplySQL = "UPDATE playlist_stat SET  reply= ?, mtime=mtime  WHERE id = ?"
+	_upShareSQL = "UPDATE playlist_stat SET  share= ?, mtime=mtime  WHERE id = ?"
+)
+
+// Update updates stat  in db.
+func (d *Dao) Update(c context.Context, stat *model.StatM, tp string) (rows int64, err error) {
+	var tmpSQL string
+	switch tp {
+	case model.ViewCountType:
+		tmpSQL = _upViewSQL
+	case model.FavCountType:
+		tmpSQL = _upFavSQL
+	case model.ReplyCountType:
+		tmpSQL = _upReplySQL
+	case model.ShareCountType:
+		tmpSQL = _upShareSQL
+	}
+	res, err := d.db.Exec(c, tmpSQL, *stat.Count, stat.ID)
+	if err != nil {
+		PromError("db:更新计数", tp+" Update(%d,%+v) error(%v)", stat.ID, stat, err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// Stat returns stat info.
+func (d *Dao) Stat(c context.Context, pid int64) (stat *plmdl.PlStat, err error) {
+	row := d.db.QueryRow(c, _statSQL, pid)
+	stat = &plmdl.PlStat{}
+	if err = row.Scan(&stat.ID, &stat.Mid, &stat.Fid, &stat.View, &stat.Reply, &stat.Fav, &stat.Share, &stat.MTime); err != nil {
+		PromError("db:读取计数", "Stat(%v) error(%v)", pid, err)
+	}
+	return
+}

app/job/main/playlist/dao/mysql_test.go

@@ -0,0 +1,36 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	plmdl "go-common/app/interface/main/playlist/model"
+	"go-common/app/job/main/playlist/model"
+	xtime "go-common/library/time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_Stat(t *testing.T) {
+	Convey("test stat", t, WithDao(func(d *Dao) {
+		pid := int64(41)
+		res, err := d.Stat(context.TODO(), pid)
+		So(err, ShouldBeNil)
+		Printf("%+v", res)
+	}))
+}
+
+func TestDao_Update(t *testing.T) {
+	Convey("test update", t, WithDao(func(d *Dao) {
+		var (
+			pid  int64 = 1
+			aid  int64 = 1
+			view int64 = 2
+		)
+		arg := &model.StatM{Type: plmdl.PlDBusType, ID: pid, Aid: aid, Count: &view, Timestamp: xtime.Time(time.Now().Unix()), IP: ""}
+		res, err := d.Update(context.TODO(), arg, model.ViewCountType)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+}

app/job/main/playlist/dao/redis.go

@@ -0,0 +1,62 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+)
+
+const (
+	// view
+	_viewPrefix = "v_%d_%d_%s"
+)
+
+func viewKey(pid, aid int64, ip string) (key string) {
+	if ip == "" {
+		// let it pass if ip is empty.
+		return
+	}
+	return fmt.Sprintf(_viewPrefix, pid, aid, ip)
+}
+
+// Intercept intercepts illegal views.
+func (d *Dao) Intercept(c context.Context, pid, aid int64, ip string) (ban bool) {
+	var (
+		err   error
+		exist bool
+		key   = viewKey(pid, aid, ip)
+		conn  = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if key == "" {
+		return
+	}
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		PromError("redis:EXISTS播放数", "conn.Do(EXISTS, %s) error(%v)", key, err)
+		return
+	}
+	if exist {
+		ban = true
+		return
+	}
+	if err = conn.Send("SET", key, ""); err != nil {
+		PromError("redis:SET播放数", "conn.Send(EXPIRE, %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.viewCacheTTL); err != nil {
+		PromError("redis:EXPIRE播放数", "conn.Send(EXPIRE, %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		PromError("redis:播放数缓存Flush", "conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			PromError("redis:播放数缓存Receive", "conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	return
+}

app/job/main/playlist/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/playlist/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/http/http.go

@@ -0,0 +1,35 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var pjSrv *service.Service
+
+// Init .
+func Init(c *conf.Config, s *service.Service) {
+	pjSrv = s
+	engineOut := bm.DefaultServer(c.HTTPServer)
+	outerRouter(engineOut)
+	// init Outer server
+	if err := engineOut.Start(); err != nil {
+		log.Error("engineOut.Start error(%v)", err)
+		panic(err)
+	}
+}
+
+func outerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+func ping(c *bm.Context) {
+	if err := pjSrv.Ping(c); err != nil {
+		log.Error("playlist job ping error")
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/job/main/playlist/model/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["playlist.go"],
+    importpath = "go-common/app/job/main/playlist/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/model/playlist.go

@@ -0,0 +1,76 @@
+package model
+
+import (
+	"fmt"
+
+	"go-common/library/time"
+)
+
+const (
+	// ViewCountType view count type.
+	ViewCountType = "view"
+	// FavCountType fav count type.
+	FavCountType = "favorite"
+	// ReplyCountType reply count type.
+	ReplyCountType = "reply"
+	// ShareCountType share count type.
+	ShareCountType = "share"
+)
+
+// StatM  playlist's topic stat message in databus.
+type StatM struct {
+	Type      string    `json:"type"`
+	ID        int64     `json:"id"`
+	Aid       int64     `json:"aid"`
+	Count     *int64    `json:"count"`
+	Timestamp time.Time `json:"timestamp"`
+	IP        string    `json:"ip"`
+}
+
+// StatMsg means playlist's stat message in databus.
+type StatMsg struct {
+	Type     string    `json:"type"`
+	Pid      int64     `json:"pid"`
+	Mid      int64     `json:"mid"`
+	Fid      int64     `json:"fid"`
+	Aid      int64     `json:"aid"`
+	View     *int64    `json:"view"`
+	Favorite *int64    `json:"fav"`
+	Reply    *int64    `json:"reply"`
+	Share    *int64    `json:"share"`
+	MTime    time.Time `json:"mtime"`
+	IP       string    `json:"ip"`
+}
+
+// String format sm
+func (sm *StatM) String(tp string) (res string) {
+	if sm == nil {
+		res = "<nil>"
+		return
+	}
+	res = fmt.Sprintf("pid: %v, aid: %v, ip: %v, "+tp+"(%s) count(%d)", sm.ID, sm.Aid, sm.IP, formatPInt(sm.Count))
+	return
+}
+
+func formatPInt(s *int64) (res string) {
+	if s == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("%d", *s)
+}
+
+// Merge merges stat.
+func Merge(last, m *StatMsg) {
+	if m.View != nil && *m.View >= 0 {
+		*last.View = *m.View
+	}
+	if m.Share != nil && *m.Share >= 0 {
+		*last.Share = *m.Share
+	}
+	if m.Favorite != nil && *m.Favorite >= 0 {
+		*last.Favorite = *m.Favorite
+	}
+	if m.Reply != nil && *m.Reply >= 0 {
+		*last.Reply = *m.Reply
+	}
+}

app/job/main/playlist/service/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/playlist/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/playlist/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/playlist/model:go_default_library",
+        "//app/interface/main/playlist/rpc/client:go_default_library",
+        "//app/job/main/playlist/conf:go_default_library",
+        "//app/job/main/playlist/dao:go_default_library",
+        "//app/job/main/playlist/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/playlist/service/service.go

@@ -0,0 +1,189 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	plmdl "go-common/app/interface/main/playlist/model"
+	plarpc "go-common/app/interface/main/playlist/rpc/client"
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/dao"
+	"go-common/app/job/main/playlist/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+)
+
+const (
+	_sharding = 10 // goroutines for dealing the stat
+	_chanSize = 10240
+)
+
+// Service .
+type Service struct {
+	c                *conf.Config
+	dao              *dao.Dao
+	waiter           *sync.WaitGroup
+	closed           bool
+	playlistViewSub  *databus.Databus
+	playlistFavSub   *databus.Databus
+	playlistReplySub *databus.Databus
+	playlistShareSub *databus.Databus
+	playlistRPC      *plarpc.Service
+	updateDbInterval int64
+	statCh           [_sharding]chan *model.StatM
+}
+
+// New creates a Service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:                c,
+		dao:              dao.New(c),
+		waiter:           new(sync.WaitGroup),
+		playlistViewSub:  databus.New(c.PlaylistViewSub),
+		playlistFavSub:   databus.New(c.PlaylistFavSub),
+		playlistReplySub: databus.New(c.PlaylistReplySub),
+		playlistShareSub: databus.New(c.PlaylistShareSub),
+		playlistRPC:      plarpc.New(c.PlaylistRPC),
+		updateDbInterval: int64(time.Duration(c.Job.UpdateDbInterval) / time.Second),
+	}
+	for i := int64(0); i < _sharding; i++ {
+		// for stat
+		s.statCh[i] = make(chan *model.StatM, _chanSize)
+		s.waiter.Add(1)
+		go s.viewproc(i)
+	}
+	s.waiter.Add(1)
+	go s.consumeView()
+	s.waiter.Add(1)
+	go s.consumeFav()
+	s.waiter.Add(1)
+	go s.consumeReply()
+	s.waiter.Add(1)
+	go s.consumeShare()
+	return
+}
+
+// consumeView consumes playlist's view.
+func (s *Service) consumeView() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			for i := 0; i < _sharding; i++ {
+				close(s.statCh[i])
+			}
+			return
+		}
+		msg, ok := <-s.playlistViewSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job view consumer exit!")
+			time.Sleep(10 * time.Millisecond)
+			continue
+		}
+		msg.Commit()
+		viewSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, viewSM); err != nil {
+			dao.PromError("service:解析计数databus消息", "json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if viewSM.Type != plmdl.PlDBusType || viewSM.ID <= 0 {
+			continue
+		}
+		key := viewSM.ID % _sharding
+		s.statCh[key] <- viewSM
+		prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+		log.Info("consumeView key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, viewSM.String(model.ViewCountType))
+	}
+}
+
+// consumeFav  consumes playlist's favorite.
+func (s *Service) consumeFav() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistFavSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job favorite consumer exit!")
+			return
+		}
+		msg.Commit()
+		favSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, favSM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if favSM.Type != plmdl.PlDBusType || favSM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, favSM, model.FavCountType)
+		log.Info("consumeFav key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, favSM.String(model.FavCountType))
+	}
+}
+
+// consumeReply  consumes playlist's reply.
+func (s *Service) consumeReply() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistReplySub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job reply consumer exit!")
+			return
+		}
+		msg.Commit()
+		replySM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, replySM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if replySM.Type != plmdl.PlDBusType || replySM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, replySM, model.ReplyCountType)
+		log.Info("consumeReply key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, replySM.String(model.ReplyCountType))
+	}
+}
+
+// consumeShare  consumes playlist's share.
+func (s *Service) consumeShare() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.playlistShareSub.Messages()
+		if !ok {
+			log.Info("databus: playlist-job share consumer exit!")
+			return
+		}
+		msg.Commit()
+		shareSM := &model.StatM{}
+		if err := json.Unmarshal(msg.Value, shareSM); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		if shareSM.Type != plmdl.PlDBusType || shareSM.ID <= 0 {
+			continue
+		}
+		s.upStat(c, shareSM, model.ShareCountType)
+		log.Info("consumeShare key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, shareSM.String(model.ShareCountType))
+	}
+}
+
+// Ping reports the heath of services.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close releases resources which owned by the Service instance.
+func (s *Service) Close() (err error) {
+	defer s.waiter.Wait()
+	s.closed = true
+	s.playlistViewSub.Close()
+	s.playlistFavSub.Close()
+	s.playlistReplySub.Close()
+	s.playlistShareSub.Close()
+	log.Info("playlist-job has been closed.")
+	return
+}

app/job/main/playlist/service/service_test.go

@@ -0,0 +1,39 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/playlist/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	svr *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/playlist-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	svr = New(conf.Conf)
+	time.Sleep(time.Second)
+
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		f(svr)
+	}
+}
+
+func TestService_Ping(t *testing.T) {
+	Convey("test ping", t, WithService(func(s *Service) {
+		err := s.Ping(context.TODO())
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/playlist/service/stat.go

@@ -0,0 +1,126 @@
+package service
+
+import (
+	"context"
+
+	plamdl "go-common/app/interface/main/playlist/model"
+	"go-common/app/job/main/playlist/conf"
+	"go-common/app/job/main/playlist/dao"
+	"go-common/app/job/main/playlist/model"
+	"go-common/library/log"
+)
+
+func (s *Service) viewproc(i int64) {
+	defer s.waiter.Done()
+	var (
+		c  = context.TODO()
+		ch = s.statCh[i]
+	)
+	for {
+		sm, ok := <-ch
+		if !ok {
+			log.Warn("statproc(%d) quit", i)
+			return
+		}
+		// filter view count
+		if conf.Conf.Job.InterceptOn && sm.Count != nil && *sm.Count > 0 {
+			if s.intercept(sm) {
+				log.Info("intercept view count (pid:%d, aid:%d, ip:%s)", sm.ID, sm.Aid, sm.IP)
+				dao.PromInfo("stat:访问计数拦截")
+				continue
+			}
+		}
+		s.upStat(c, sm, model.ViewCountType)
+	}
+}
+
+func (s *Service) upStat(c context.Context, sm *model.StatM, tp string) {
+	// update cache
+	s.updateCache(c, sm, tp)
+	// update db
+	s.updateDB(c, sm, tp)
+}
+
+// updateDB update stat in db.
+func (s *Service) updateDB(c context.Context, stat *model.StatM, tp string) (err error) {
+	if _, err = s.dao.Update(context.TODO(), stat, tp); err != nil {
+		return
+	}
+	log.Info("update db success "+tp+" pid(%d) count(%d) ", stat.ID, *stat.Count)
+	dao.PromInfo("stat:更新计数DB")
+	return
+}
+
+// updateCache update stat in cache
+func (s *Service) updateCache(c context.Context, sm *model.StatM, tp string) (err error) {
+	var (
+		mid  int64
+		fid  int64
+		st   *plamdl.PlStat
+		stat *plamdl.ArgStats
+	)
+	if st, err = s.dao.Stat(c, sm.ID); err != nil {
+		log.Error("s.dao.Stat(%d) error(%v)", sm.ID, err)
+		return
+	}
+	switch tp {
+	case model.ViewCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  *sm.Count,
+			Reply: st.Reply,
+			Fav:   st.Fav,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.FavCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: st.Reply,
+			Fav:   *sm.Count,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.ReplyCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: *sm.Count,
+			Fav:   st.Fav,
+			Share: st.Share,
+			MTime: st.MTime,
+		}}
+	case model.ShareCountType:
+		stat = &plamdl.ArgStats{PlStat: &plamdl.PlStat{
+			ID:    sm.ID,
+			Mid:   st.Mid,
+			Fid:   st.Fid,
+			View:  st.View,
+			Reply: st.Reply,
+			Fav:   st.Fav,
+			Share: *sm.Count,
+			MTime: st.MTime,
+		}}
+	}
+	if err = s.playlistRPC.SetStat(c, stat); err != nil {
+		dao.PromError("stat:更新计数缓存",
+			"s.playlistRPC.SetStat "+tp+" pid(%d) fid(%d) mid(%d) view(%d) favorite(%d) reply(%d) share(%d) error(%v)",
+			sm.ID, fid, mid, *sm.Count, st.Fav, st.Reply, st.Share, err)
+	}
+	log.Info("update cache success "+tp+"  pid(%d)  aid(%d)  fid(%d) mid(%d) view(%d) favorite(%d) reply(%d) share(%d)",
+		sm.ID, sm.Aid, fid, mid, *sm.Count, st.Fav, st.Reply, st.Share)
+	dao.PromInfo("stat:更新计数缓存")
+	return
+}
+
+// intercept intercepts illegal views.
+func (s *Service) intercept(stat *model.StatM) bool {
+	return s.dao.Intercept(context.TODO(), stat.ID, stat.Aid, stat.IP)
+}