Create & Init Project...

app/job/main/archive-shjd/BUILD

@@ -0,0 +1,20 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/archive-shjd/cmd:all-srcs",
+        "//app/job/main/archive-shjd/conf:all-srcs",
+        "//app/job/main/archive-shjd/http:all-srcs",
+        "//app/job/main/archive-shjd/model:all-srcs",
+        "//app/job/main/archive-shjd/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/CHANGELOG.md

@@ -0,0 +1,27 @@
+# 稿件字段同步
+
+## Version 1.2.2
+> 1.修改消费canal时的数据处理
+
+## Version 1.2.1
+> 1.nw-->new
+
+## Version 1.2.1
+> 1.消费嘉定机房的accountNotify更新用户缓存
+
+## Version 1.2.0
+> 1.增加notify的databus pub，闭环在一个机房
+
+## Version 1.1.0
+> 1.增加stat消费
+
+## Version 1.0.2
+> 1.databus bugfix
+
+## Version 1.0.1
+
+> 1.修改配置文件名
+
+## Version 1.0.0
+
+> 1.初始化项目

app/job/main/archive-shjd/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+peiyifei
+
+# Author 
+peiyifei
+haoguanwei
+yujia
+
+# Reviewer
+peiyifei
+haoguanwei

app/job/main/archive-shjd/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- peiyifei
+- yujia
+labels:
+- job
+- job/main/archive-shjd
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- haoguanwei
+- peiyifei
+- yujia

app/job/main/archive-shjd/README.md

@@ -0,0 +1,13 @@
+#### archive-job
+
+##### 项目简介
+> 1.提供稿件相关的自动化逻辑  
+
+##### 编译环境
+> 请只用golang v1.8.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  
+
+##### 特别说明
+> 1.model目录可能会被其他项目引用，请谨慎请改并通知各方。  

app/job/main/archive-shjd/cmd/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "archive-job-example.toml",
+        "archive-job-kisjd-test.toml",
+    ],
+    importpath = "go-common/app/job/main/archive-shjd/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/archive-shjd/conf:go_default_library",
+        "//app/job/main/archive-shjd/http:go_default_library",
+        "//app/job/main/archive-shjd/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/cmd/archive-job-example.toml

@@ -0,0 +1,77 @@
+version = "1.0.0"
+user    = "nobody"
+pid     = "/tmp/archive-job.pid"
+dir     = "./"
+perf    = "127.0.0.1:6161"
+trace   = false
+debug   = false
+env     = "test"
+
+tick = "50ms"
+
+[xlog]
+dir = "/data/log/archive-job/"
+
+[app]
+key    = "e7482d29be4a95b8"
+secret = "9e803791cdef756e75faee68e12b7442"
+
+[host]
+apiCo = "http://api.bilibili.co"
+
+[httpClient]
+dial      = "100ms"
+timeout   = "1s"
+keepAlive = "60s"
+timer     = 128
+
+[databus]
+key = "0Pub71WwEMKXu63qtztu"
+secret = "0Pub71WwEMKXu63qtztv"
+group = "Stat-UGC-S"
+topic = "Stat-T"
+action = "sub"
+    [databus.redis]
+    name = "archive-job/databus"
+    proto = "tcp"
+    addr = "172.16.33.56:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[kafka]
+group = "archive-job-consumer"
+topics = ["archive-push", "stat"]
+offset = false
+    [kafka.zookeeper]
+    root = "/kafka"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[archiveRPC]
+    pullInterval = "10s"
+    [archiveRPC.client]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [[archiveRPC.backup]]
+    proto = "tcp"
+    addr = "172.16.0.26:6089"
+    timeout = "1s"
+    timer = 1000
+    [archiveRPC.zookeeper]
+    root = "/microservice/archive-service/"
+    addrs = ["172.16.0.148:2181"]
+    timeout = "30s"
+
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+username = "2333@bilibili.com"
+password = "2333"
+bangumi = ["233@bilibili.com"]
+movie = ["233@bilibili.com"]

app/job/main/archive-shjd/cmd/archive-job-kisjd-test.toml

@@ -0,0 +1,161 @@
+[Redis]
+name = "archive-job/track"
+proto = "unix"
+addr = "/tmp/shjd-archive-service-group1-redis.sock"
+active = 1000
+idle = 20
+dialTimeout = "80ms"
+readTimeout = "200ms"
+writeTimeout = "200ms"
+idleTimeout = "80s"
+
+[Databus]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "ArcResult-SHJD-MainAppSvr-S"
+topic = "ArcResult-SHJD-T"
+action = "sub"
+name = "stat-job/databus"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[ViewSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatView-MainAppSvr-SHJD-S"
+topic = "StatView-T"
+action = "sub"
+name = "stat-job/view"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[DmSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatDM-MainAppSvr-SHJD-S"
+topic = "StatDM-T"
+action = "sub"
+name = "stat-job/dm"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[ReplySub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatReply-MainAppSvr-SHJD-S"
+topic = "StatReply-T"
+action = "sub"
+name = "stat-job/reply"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[FavSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatFav-MainAppSvr-SHJD-S"
+topic = "StatFav-T"
+action = "sub"
+name = "stat-job/fav"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[CoinSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatCoin-MainAppSvr-SHJD-S"
+topic = "StatCoin-T"
+action = "sub"
+name = "stat-job/coin"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[ShareSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatShare-MainAppSvr-SHJD-S"
+topic = "StatShare-T"
+action = "sub"
+name = "stat-job/share"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[RankSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatRank-MainAppSvr-SHJD-S"
+topic = "StatRank-T"
+action = "sub"
+name = "stat-job/rank"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+	
+[likeSub]
+key = "8e27ab7e39270b59"
+secret = "4b8c2e03c48f0b21eacd7a11e2a3b003"
+group = "StatLike-MainAppSvr-SHJD-S"
+topic = "StatLike-T"
+action = "sub"
+name = "stat-job/like"
+proto = "tcp"
+addr = "172.18.21.41:6205"
+idle = 1
+active = 1
+dialTimeout = "1s"
+readTimeout = "60s"
+writeTimeout = "1s"
+idleTimeout = "10s"
+
+[[ArchiveRPCs]]
+    cluster = "group1"
+    zone = "sh004"
+[[ArchiveRPCs]]
+    cluster = "group2"
+    zone = "sh004"

app/job/main/archive-shjd/cmd/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/archive-shjd/conf"
+	"go-common/app/job/main/archive-shjd/http"
+	"go-common/app/job/main/archive-shjd/service"
+	"go-common/library/log"
+)
+
+var (
+	srv *service.Service
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("archive-kisjd-job start")
+	srv = service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	signalHandler()
+}
+
+func signalHandler() {
+	var (
+		err error
+		ch  = make(chan os.Signal, 1)
+	)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		si := <-ch
+		switch si {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("get a signal %s, stop the consume process", si.String())
+			if err = srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/main/archive-shjd/conf/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/archive-shjd/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/conf/conf.go

@@ -0,0 +1,94 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/queue/databus"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	Conf     = &Config{}
+	client   *conf.Client
+)
+
+type Config struct {
+	// Env
+	Env string
+	// interface XLog
+	Log *log.Config
+	// databus
+	Databus       *databus.Config
+	ViewSub       *databus.Config
+	DmSub         *databus.Config
+	ReplySub      *databus.Config
+	FavSub        *databus.Config
+	CoinSub       *databus.Config
+	ShareSub      *databus.Config
+	RankSub       *databus.Config
+	LikeSub       *databus.Config
+	NotifyPub     *databus.Config
+	AccountNotify *databus.Config
+	// rpc
+	ArchiveRPCs []*rpc.ClientConfig
+	// http
+	BM *bm.ServerConfig
+	// redis
+	Redis *redis.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init config.
+func Init() (err error) {
+	if confPath != "" {
+		_, err = toml.DecodeFile(confPath, &Conf)
+		return
+	}
+	err = remote()
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf = &Config{}
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/archive-shjd/http/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/archive-shjd/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/archive-shjd/conf:go_default_library",
+        "//app/job/main/archive-shjd/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/http/http.go

@@ -0,0 +1,38 @@
+package http
+
+import (
+	"go-common/app/job/main/archive-shjd/conf"
+	"go-common/app/job/main/archive-shjd/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"net/http"
+)
+
+var (
+	arcSvr *service.Service
+)
+
+// Init init http router.
+func Init(c *conf.Config, s *service.Service) {
+	arcSvr = s
+	e := bm.DefaultServer(c.BM)
+	innerRouter(e)
+	// init internal server
+	if err := e.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := arcSvr.Ping(); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/job/main/archive-shjd/model/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "message.go",
+        "result.go",
+        "retry.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/archive-shjd/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/service/main/archive/api:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/model/message.go

@@ -0,0 +1,25 @@
+package model
+
+import "encoding/json"
+
+// Message databus
+type Message struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// Notify is
+type Notify struct {
+	Table  string   `json:"table"`
+	Action string   `json:"action"`
+	Nw     *Archive `json:"new"`
+	Old    *Archive `json:"old"`
+}
+
+// AccountNotify is
+type AccountNotify struct {
+	Mid    int64  `json:"mid"`
+	Action string `json:"action"`
+}

app/job/main/archive-shjd/model/result.go

@@ -0,0 +1,55 @@
+package model
+
+import (
+	"database/sql/driver"
+	"time"
+)
+
+// Archive archive result
+type Archive struct {
+	ID          int64     `json:"id"`
+	AID         int64     `json:"aid"`
+	Mid         int64     `json:"mid"`
+	TypeID      int16     `json:"typeid"`
+	Videos      int       `json:"videos"`
+	Title       string    `json:"title"`
+	Cover       string    `json:"cover"`
+	Content     string    `json:"content"`
+	Duration    int       `json:"duration"`
+	Attribute   int32     `json:"attribute"`
+	Copyright   int8      `json:"copyright"`
+	Access      int       `json:"access"`
+	PubTime     wocaoTime `json:"pubtime"`
+	CTime       wocaoTime `json:"ctime"`
+	MTime       wocaoTime `json:"mtime"`
+	State       int       `json:"state"`
+	MissionID   int64     `json:"mission_id"`
+	OrderID     int64     `json:"order_id"`
+	RedirectURL string    `json:"redirect_url"`
+	Forward     int64     `json:"forward"`
+	Dynamic     string    `json:"dynamic"`
+}
+
+// Video is
+type Video struct {
+	AID int64 `json:"aid"`
+	CID int64 `json:"cid"`
+}
+
+type wocaoTime string
+
+// Scan scan time.
+func (jt *wocaoTime) Scan(src interface{}) (err error) {
+	switch sc := src.(type) {
+	case time.Time:
+		*jt = wocaoTime(sc.Format("2006-01-02 15:04:05"))
+	case string:
+		*jt = wocaoTime(sc)
+	}
+	return
+}
+
+// Value get time value.
+func (jt wocaoTime) Value() (driver.Value, error) {
+	return time.Parse("2006-01-02 15:04:05", string(jt))
+}

app/job/main/archive-shjd/model/retry.go

@@ -0,0 +1,18 @@
+package model
+
+// is
+const (
+	TypeForUpdateVideo   = int(0)
+	TypeForDelVideo      = int(1)
+	TypeForUpdateArchive = int(2)
+)
+
+// RetryItem struct
+type RetryItem struct {
+	Tp     int      `json:"type"`
+	AID    int64    `json:"aid"`
+	CID    int64    `json:"cid"`
+	Old    *Archive `json:"new_archive"`
+	Nw     *Archive `json:"old_archive"`
+	Action string   `json:"action"`
+}

app/job/main/archive-shjd/model/stat.go

@@ -0,0 +1,70 @@
+package model
+
+import "go-common/app/service/main/archive/api"
+
+// is
+const (
+	TypeForView  = "view"
+	TypeForDm    = "dm"
+	TypeForReply = "reply"
+	TypeForFav   = "fav"
+	TypeForCoin  = "coin"
+	TypeForShare = "share"
+	TypeForRank  = "rank"
+	TypeForLike  = "like"
+)
+
+// StatMsg stat info.
+type StatMsg struct {
+	Aid     int64  `json:"aid"`
+	Click   int32  `json:"click"`
+	DM      int32  `json:"dm"`
+	Reply   int32  `json:"reply"`
+	Fav     int32  `json:"fav"`
+	Coin    int32  `json:"coin"`
+	Share   int32  `json:"share"`
+	NowRank int32  `json:"now_rank"`
+	HisRank int32  `json:"his_rank"`
+	Like    int32  `json:"like"`
+	Type    string `json:"-"`
+	Ts      int64  `json:"-"`
+}
+
+// StatCount is
+type StatCount struct {
+	Type      string `json:"type"`
+	Aid       int64  `json:"id"`
+	Count     int32  `json:"count"`
+	TimeStamp int64  `json:"timestamp"`
+}
+
+// Merge merge message and stat from db.
+func Merge(m *StatMsg, s *api.Stat) {
+	if m.Click >= 0 && m.Type == TypeForView {
+		s.View = m.Click
+	}
+	if m.Coin >= 0 && m.Type == TypeForCoin {
+		s.Coin = m.Coin
+	}
+	if m.DM >= 0 && m.Type == TypeForDm {
+		s.Danmaku = m.DM
+	}
+	if m.Fav >= 0 && m.Type == TypeForFav {
+		s.Fav = m.Fav
+	}
+	if m.Reply >= 0 && m.Type == TypeForReply {
+		s.Reply = m.Reply
+	}
+	if m.Share >= 0 && m.Type == TypeForShare && m.Share > s.Share {
+		s.Share = m.Share
+	}
+	if m.NowRank >= 0 && m.Type == TypeForRank {
+		s.NowRank = m.NowRank
+	}
+	if m.HisRank >= 0 && m.Type == TypeForRank {
+		s.HisRank = m.HisRank
+	}
+	if m.Like >= 0 && m.Type == TypeForLike {
+		s.Like = m.Like
+	}
+}

app/job/main/archive-shjd/service/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/archive-shjd/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "account.go",
+        "cache.go",
+        "retry.go",
+        "service.go",
+        "stat.go",
+    ],
+    importpath = "go-common/app/job/main/archive-shjd/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/archive-shjd/conf:go_default_library",
+        "//app/job/main/archive-shjd/model:go_default_library",
+        "//app/service/main/account/model:go_default_library",
+        "//app/service/main/account/rpc/client:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/archive-shjd/service/account.go

@@ -0,0 +1,138 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/archive-shjd/model"
+	accmdl "go-common/app/service/main/account/model"
+	"go-common/app/service/main/archive/api"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+const (
+	_actForUname = "updateUname"
+	_actForFace  = "updateFace"
+	_actForAdmin = "updateByAdmin"
+)
+
+func (s *Service) accountNotifyproc() {
+	defer s.waiter.Done()
+	var msgs = s.accountNotify.Messages()
+	for {
+		var (
+			msg *databus.Message
+			ok  bool
+			err error
+			c   = context.TODO()
+		)
+		if msg, ok = <-msgs; !ok {
+			log.Error("s.cachesub.messages closed")
+			return
+		}
+		msg.Commit()
+		m := &model.AccountNotify{}
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		log.Info("accountNotify got key(%s) value(%s)", msg.Key, msg.Value)
+		if m.Action != _actForAdmin && m.Action != _actForFace && m.Action != _actForUname {
+			log.Warn("accountNotify skip action(%s) values(%s)", m.Action, msg.Value)
+			continue
+		}
+		var count int
+		if count, err = s.arcRPCs["group1"].UpCount2(c, &arcmdl.ArgUpCount2{Mid: m.Mid}); err != nil {
+			log.Error("s.arcRPC.UpCount2(%d) error(%v)", m.Mid, err)
+			continue
+		}
+		if count == 0 {
+			log.Info("accountNotify mid(%d) passed(%d)", m.Mid, count)
+			continue
+		}
+		if m.Action == _actForAdmin {
+			// check uname or face is updated
+			var am []*api.Arc
+			if am, err = s.arcRPCs["group1"].UpArcs3(c, &arcmdl.ArgUpArcs2{Mid: m.Mid, Ps: 2, Pn: 1}); err != nil {
+				if ecode.Cause(err).Equal(ecode.NothingFound) {
+					err = nil
+					log.Info("accountNotify mid(%d) no passed archive", m.Mid)
+					continue
+				}
+				log.Error("accountNotify mid(%d) error(%v)", m.Mid, err)
+				continue
+			}
+			if len(am) == 0 {
+				log.Info("accountNotify mid(%d) no passed archive", m.Mid)
+				continue
+			}
+			var info *accmdl.Info
+			if info, err = s.accRPC.Info3(c, &accmdl.ArgMid{Mid: m.Mid}); err != nil {
+				log.Error("accountNotify accRPC.info3(%d) error(%v)", m.Mid, err)
+				continue
+			}
+			if info.Name == am[0].Author.Name && info.Face == am[0].Author.Face {
+				log.Info("accountNotify face(%s) name(%s) not change", info.Face, info.Name)
+				continue
+			}
+		}
+		s.notifyMu.Lock()
+		s.notifyMid[m.Mid] = struct{}{}
+		s.notifyMu.Unlock()
+	}
+}
+
+func (s *Service) clearMidCache() {
+	defer s.waiter.Done()
+	for {
+		time.Sleep(5 * time.Second)
+		s.notifyMu.Lock()
+		mids := s.notifyMid
+		s.notifyMid = make(map[int64]struct{})
+		s.notifyMu.Unlock()
+		log.Info("start clearMidCache mids(%d)", len(mids))
+		for mid := range mids {
+			s.updateUpperCache(context.TODO(), mid)
+		}
+		log.Info("finish clearMidCache mids(%d)", len(mids))
+		if s.close && len(s.notifyMid) == 0 {
+			return
+		}
+	}
+}
+
+func (s *Service) updateUpperCache(c context.Context, mid int64) (err error) {
+	failedCnt := 0
+	for k, rpc := range s.arcRPCs {
+		pn := 1
+		for {
+			var arcs []*api.Arc
+			if arcs, err = rpc.UpArcs3(c, &arcmdl.ArgUpArcs2{Mid: mid, Pn: pn}); err != nil {
+				log.Error("rpc(%s) UpArcs3(%d) error(%v)", k, mid, err)
+				break
+			}
+			pn++
+			if len(arcs) == 0 {
+				break
+			}
+			for _, arc := range arcs {
+				if err = rpc.ArcCache2(c, &arcmdl.ArgCache2{Aid: arc.Aid, Tp: arcmdl.CacheUpdate}); err != nil {
+					log.Error("s.arcRPC(%d).ArcCache2(%d, %s) mid(%d) error(%v)", k, arc.Aid, arcmdl.CacheUpdate, mid, err)
+					failedCnt++
+					continue
+				}
+			}
+
+		}
+	}
+	if failedCnt > 0 {
+		log.Error("accountNotify updateUpperCache mid(%d) failed(%d)", mid, failedCnt)
+		return
+	}
+	log.Info("accountNofity updateUpperCache mid(%d)", mid)
+	return
+}

app/job/main/archive-shjd/service/cache.go

@@ -0,0 +1,126 @@
+package service
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/job/main/archive-shjd/model"
+	arcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// UpdateCache is
+func (s *Service) UpdateCache(old *model.Archive, nw *model.Archive, action string) (err error) {
+	defer func() {
+		if err == nil {
+			s.notifyPub.Send(context.Background(), strconv.FormatInt(nw.AID, 10), &model.Notify{Table: _tableArchive, Nw: nw, Old: old, Action: action})
+			return
+		}
+		// retry
+		item := &model.RetryItem{
+			Old:    old,
+			Nw:     nw,
+			Tp:     model.TypeForUpdateArchive,
+			Action: action,
+		}
+		if err1 := s.PushItem(context.TODO(), item); err1 != nil {
+			log.Error("s.PushItem(%+v) error(%+v)", item, err1)
+			return
+		}
+	}()
+	args := &arcmdl.ArgCache2{}
+	args.Aid = nw.AID
+	args.Tp = arcmdl.CacheUpdate
+	if old == nil {
+		// insert
+		if nw.State >= 0 {
+			args.Tp = arcmdl.CacheAdd
+		}
+	} else {
+		if nw.State >= 0 {
+			args.Tp = arcmdl.CacheAdd
+		} else {
+			args.Tp = arcmdl.CacheDelete
+		}
+		if nw.Mid != old.Mid {
+			args.OldMid = old.Mid
+		}
+		if old.TypeID != nw.TypeID {
+			fieldAgs := &arcmdl.ArgFieldCache2{Aid: nw.AID, TypeID: nw.TypeID, OldTypeID: old.TypeID}
+			for cluster, arc := range s.arcRPCs {
+				if err = arc.ArcFieldCache2(context.TODO(), fieldAgs); err != nil {
+					log.Error("s.arcRPC.ArcFieldCache2(%s, %+v) error(%+v)", cluster, fieldAgs, err)
+					return
+				}
+			}
+		}
+	}
+	for cluster, arc := range s.arcRPCs {
+		if err = arc.ArcCache2(context.TODO(), args); err != nil {
+			log.Error("s.arcRPC.ArcCache2(%s,%+v) error(%v)", cluster, args, err)
+			return
+		}
+	}
+	return
+}
+
+// UpdateVideoCache is
+func (s *Service) UpdateVideoCache(aid, cid int64) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		// retry
+		item := &model.RetryItem{
+			AID: aid,
+			CID: cid,
+			Tp:  model.TypeForUpdateVideo,
+		}
+		if err1 := s.PushItem(context.TODO(), item); err1 != nil {
+			log.Error("s.PushItem(%+v) error(%+v)", item, err1)
+			return
+		}
+	}()
+	for cluster, arc := range s.arcRPCs {
+		if err = arc.UpVideo2(context.TODO(), &arcmdl.ArgVideo2{Aid: aid, Cid: cid}); err != nil {
+			if ecode.Cause(err).Equal(ecode.NothingFound) {
+				err = nil
+				return
+			}
+			err = errors.Wrapf(err, "s.arcRPC.UpVideo2 cluster(%s)", cluster)
+		}
+	}
+	return
+}
+
+// DelteVideoCache del video cache
+func (s *Service) DelteVideoCache(aid, cid int64) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		// retry
+		item := &model.RetryItem{
+			AID: aid,
+			CID: cid,
+			Tp:  model.TypeForDelVideo,
+		}
+		if err1 := s.PushItem(context.TODO(), item); err1 != nil {
+			log.Error("s.PushItem(%+v) error(%+v)", item, err1)
+			return
+		}
+	}()
+	for cluster, arc := range s.arcRPCs {
+		if err = arc.DelVideo2(context.TODO(), &arcmdl.ArgVideo2{Aid: aid, Cid: cid}); err != nil {
+			if ecode.Cause(err).Equal(ecode.NothingFound) {
+				err = nil
+				return
+			}
+			err = errors.Wrapf(err, "s.arcRPC.VdelVideo2 cluster(%s)", cluster)
+		}
+	}
+	return
+}

app/job/main/archive-shjd/service/retry.go

@@ -0,0 +1,88 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/archive-shjd/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_retryList = "retry_list"
+)
+
+func (s *Service) retryconsumer() {
+	defer s.waiter.Done()
+	for {
+		if s.close {
+			log.Info("retryconsumer closed")
+			return
+		}
+		var (
+			c    = context.TODO()
+			err  error
+			item *model.RetryItem
+		)
+		if item, err = s.PopItem(c); err != nil {
+			log.Error("%+v")
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		if item == nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		log.Info("get retry item(%+v)", item)
+		switch item.Tp {
+		case model.TypeForDelVideo:
+			s.DelteVideoCache(item.AID, item.CID)
+		case model.TypeForUpdateVideo:
+			s.UpdateVideoCache(item.AID, item.CID)
+		case model.TypeForUpdateArchive:
+			s.UpdateCache(item.Old, item.Nw, item.Action)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+// PushItem is
+func (s *Service) PushItem(c context.Context, item *model.RetryItem) (err error) {
+	conn := s.rds.Get(c)
+	defer conn.Close()
+	bs, err := json.Marshal(item)
+	if err != nil {
+		err = errors.Wrap(err, "json.Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryList, bs); err != nil {
+		err = errors.Wrap(err, "conn.Send(RPUSH)")
+		return
+	}
+	return
+}
+
+// PopItem is
+func (s *Service) PopItem(c context.Context) (item *model.RetryItem, err error) {
+	conn := s.rds.Get(c)
+	defer conn.Close()
+	bs, err := redis.Bytes(conn.Do("LPOP", _retryList))
+	if err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		err = errors.WithStack(err)
+		return
+	}
+	item = &model.RetryItem{}
+	if err = json.Unmarshal(bs, item); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	return
+}

app/job/main/archive-shjd/service/service.go

@@ -0,0 +1,222 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/archive-shjd/conf"
+	"go-common/app/job/main/archive-shjd/model"
+	accrpc "go-common/app/service/main/account/rpc/client"
+	"go-common/app/service/main/archive/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_tableArchive = "archive"
+	_tableVideo   = "archive_video"
+	_actionInsert = "insert"
+	_actionUpdate = "update"
+	_actionDelete = "delete"
+	_sharding     = 10
+)
+
+type lastTmStat struct {
+	last int64
+	stat *api.Stat
+}
+
+// Service service
+type Service struct {
+	c             *conf.Config
+	waiter        sync.WaitGroup
+	canal         *databus.Databus
+	canalChan     chan *model.Message
+	subMap        map[string]*databus.Databus
+	subView       *databus.Databus
+	subDm         *databus.Databus
+	subReply      *databus.Databus
+	subFav        *databus.Databus
+	subCoin       *databus.Databus
+	subShare      *databus.Databus
+	subRank       *databus.Databus
+	subLike       *databus.Databus
+	notifyPub     *databus.Databus
+	accountNotify *databus.Databus
+	subStatCh     []chan *model.StatMsg
+	arcRPCs       map[string]*arcrpc.Service2
+	accRPC        *accrpc.Service3
+	notifyMid     map[int64]struct{}
+	notifyMu      sync.Mutex
+	rds           *redis.Pool
+	statSM        []map[int64]*lastTmStat
+	close         bool
+}
+
+// New is archive service implementation.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:         c,
+		canal:     databus.New(c.Databus),
+		canalChan: make(chan *model.Message, 10240),
+		rds:       redis.NewPool(c.Redis),
+		subMap:    make(map[string]*databus.Databus),
+		// databus
+		subView:       databus.New(c.ViewSub),
+		subDm:         databus.New(c.DmSub),
+		subReply:      databus.New(c.ReplySub),
+		subFav:        databus.New(c.FavSub),
+		subCoin:       databus.New(c.CoinSub),
+		subShare:      databus.New(c.ShareSub),
+		subRank:       databus.New(c.RankSub),
+		subLike:       databus.New(c.LikeSub),
+		notifyPub:     databus.New(c.NotifyPub),
+		accountNotify: databus.New(c.AccountNotify),
+		notifyMid:     make(map[int64]struct{}, 10240),
+		accRPC:        accrpc.New3(nil),
+	}
+	s.arcRPCs = make(map[string]*arcrpc.Service2)
+	for _, cc := range c.ArchiveRPCs {
+		s.arcRPCs[cc.Cluster] = arcrpc.New2(cc)
+	}
+	s.subMap[model.TypeForView] = s.subView
+	s.subMap[model.TypeForDm] = s.subDm
+	s.subMap[model.TypeForReply] = s.subReply
+	s.subMap[model.TypeForFav] = s.subFav
+	s.subMap[model.TypeForCoin] = s.subCoin
+	s.subMap[model.TypeForShare] = s.subShare
+	s.subMap[model.TypeForRank] = s.subRank
+	s.subMap[model.TypeForLike] = s.subLike
+	for i := 0; i < _sharding; i++ {
+		s.waiter.Add(1)
+		go s.canalChanproc()
+		s.subStatCh = append(s.subStatCh, make(chan *model.StatMsg, 10240))
+		s.statSM = append(s.statSM, map[int64]*lastTmStat{})
+		s.waiter.Add(1)
+		go s.statDealproc(i)
+	}
+	for k, d := range s.subMap {
+		s.waiter.Add(1)
+		go s.consumerproc(k, d)
+	}
+	s.waiter.Add(1)
+	go s.canalproc()
+	s.waiter.Add(1)
+	go s.retryconsumer()
+	s.waiter.Add(1)
+	go s.accountNotifyproc()
+	s.waiter.Add(1)
+	go s.clearMidCache()
+	return s
+}
+
+func (s *Service) canalChanproc() {
+	defer s.waiter.Done()
+	for {
+		m, ok := <-s.canalChan
+		if !ok {
+			log.Info("canalChanproc closed")
+			return
+		}
+		log.Info("got canal message table(%s) action(%s) old(%s) new(%s)", m.Table, m.Action, m.Old, m.New)
+		var err error
+		switch m.Table {
+		case _tableArchive:
+			var (
+				old *model.Archive
+				nw  *model.Archive
+			)
+			switch m.Action {
+			case _actionInsert:
+				if err = json.Unmarshal(m.New, &nw); err != nil {
+					log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+					continue
+				}
+			case _actionUpdate:
+				if err = json.Unmarshal(m.Old, &old); err != nil {
+					log.Error("json.Unmarshal(%s) error(%v)", m.Old, err)
+					continue
+				}
+				if err = json.Unmarshal(m.New, &nw); err != nil {
+					log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+					continue
+				}
+			default:
+				log.Warn("got unknow action(%s)", m.Action)
+				continue
+			}
+			s.UpdateCache(old, nw, m.Action)
+		case _tableVideo:
+			var video *model.Video
+			if err = json.Unmarshal(m.New, &video); err != nil {
+				log.Error("json.Unmarshal(%s) error(%v)", m.New, err)
+				continue
+			}
+			switch m.Action {
+			case _actionInsert, _actionUpdate:
+				err = s.UpdateVideoCache(video.AID, video.CID)
+			case _actionDelete:
+				err = s.DelteVideoCache(video.AID, video.CID)
+			default:
+				bs, _ := json.Marshal(m)
+				log.Error("unknow action(%s) message(%s)", m.Action, bs)
+			}
+			if err != nil {
+				log.Error("%+v", err)
+				continue
+			}
+		default:
+			log.Warn("table(%s) skiped", m.Table)
+		}
+	}
+}
+
+func (s *Service) canalproc() {
+	defer s.waiter.Done()
+	msgs := s.canal.Messages()
+	for {
+		msg, ok := <-msgs
+		if !ok || s.close {
+			close(s.canalChan)
+			log.Info("s.closed databus canal")
+			return
+		}
+		var (
+			m   = &model.Message{}
+			err error
+		)
+		msg.Commit()
+		log.Info("got message(%s)", msg.Value)
+		if err = json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			continue
+		}
+		s.canalChan <- m
+	}
+}
+
+// Ping check status
+func (s *Service) Ping() (err error) {
+	conn := s.rds.Get(context.TODO())
+	defer conn.Close()
+	if _, err = conn.Do("SET", "PING", "PONG"); err != nil {
+		err = errors.Wrap(err, "redis ping")
+		return
+	}
+	return
+}
+
+// Close is
+func (s *Service) Close() (err error) {
+	s.close = true
+	time.Sleep(5 * time.Second)
+	s.canal.Close()
+	s.waiter.Wait()
+	return
+}

app/job/main/archive-shjd/service/service_test.go

@@ -0,0 +1,46 @@
+package service
+
+import (
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/archive-shjd/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/archive-job-kisjd-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func Test_Ping(t *testing.T) {
+	Convey("Ping", t, func() {
+		s.Ping()
+	})
+}
+
+func Test_DelteVideoCache(t *testing.T) {
+	Convey("DelteVideoCache", t, func() {
+		s.DelteVideoCache(1, 1)
+	})
+}
+
+func Test_UpdateVideoCache(t *testing.T) {
+	Convey("UpdateVideoCache", t, func() {
+		s.UpdateVideoCache(1, 1)
+	})
+}
+
+func Test_Close(t *testing.T) {
+	Convey("Close", t, func() {
+		s.Close()
+	})
+}

app/job/main/archive-shjd/service/stat.go

@@ -0,0 +1,123 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/archive-shjd/model"
+	"go-common/app/service/main/archive/api"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+)
+
+// consumerproc consumer all topic
+func (s *Service) consumerproc(k string, d *databus.Databus) {
+	defer s.waiter.Done()
+	var msgs = d.Messages()
+	for {
+		var (
+			err error
+			ok  bool
+			msg *databus.Message
+			now = time.Now().Unix()
+		)
+		msg, ok = <-msgs
+		if !ok || s.close {
+			log.Info("databus(%s) consumer exit", k)
+			return
+		}
+		msg.Commit()
+		var ms = &model.StatCount{}
+		if err = json.Unmarshal(msg.Value, ms); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(msg.Value), err)
+			continue
+		}
+		if ms.Aid <= 0 || (ms.Type != "archive" && ms.Type != "archive_his") {
+			log.Warn("message(%s) error", msg.Value)
+			continue
+		}
+		if now-ms.TimeStamp > 1800 {
+			log.Warn("topic(%s) message(%s) too early", msg.Topic, msg.Value)
+			continue
+		}
+		stat := &model.StatMsg{Aid: ms.Aid, Type: k, Ts: ms.TimeStamp}
+		switch k {
+		case model.TypeForView:
+			stat.Click = ms.Count
+		case model.TypeForDm:
+			stat.DM = ms.Count
+		case model.TypeForReply:
+			stat.Reply = ms.Count
+		case model.TypeForFav:
+			stat.Fav = ms.Count
+		case model.TypeForCoin:
+			stat.Coin = ms.Count
+		case model.TypeForShare:
+			stat.Share = ms.Count
+		case model.TypeForRank:
+			stat.HisRank = ms.Count
+		case model.TypeForLike:
+			stat.Like = ms.Count
+		default:
+			log.Error("unknow type(%s) message(%s)", k, msg.Value)
+			continue
+		}
+		s.subStatCh[stat.Aid%_sharding] <- stat
+		log.Info("got message(%+v)", stat)
+	}
+}
+
+func (s *Service) statDealproc(i int) {
+	defer s.waiter.Done()
+	var (
+		ch  = s.subStatCh[i]
+		sm  = s.statSM[i]
+		c   = context.TODO()
+		ls  *lastTmStat
+		err error
+	)
+	for {
+		ms, ok := <-ch
+		if !ok {
+			log.Info("statDealproc(%d) quit", i)
+			return
+		}
+		// get stat
+		if ls, ok = sm[ms.Aid]; !ok {
+			var stat *api.Stat
+			for _, arc := range s.arcRPCs {
+				if stat, err = arc.Stat3(c, &archive.ArgAid2{Aid: ms.Aid}); err == nil {
+					break
+				}
+			}
+			if stat == nil {
+				log.Error("stat(%d) is nill", ms.Aid)
+				continue
+			}
+			ls = &lastTmStat{}
+			ls.stat = stat
+			sm[ms.Aid] = ls
+		}
+		model.Merge(ms, ls.stat)
+		// update cache
+		st := &api.Stat{
+			Aid:     ls.stat.Aid,
+			View:    ls.stat.View,
+			Danmaku: ls.stat.Danmaku,
+			Reply:   ls.stat.Reply,
+			Fav:     ls.stat.Fav,
+			Coin:    ls.stat.Coin,
+			Share:   ls.stat.Share,
+			NowRank: ls.stat.NowRank,
+			HisRank: ls.stat.HisRank,
+			Like:    ls.stat.Like,
+		}
+		for cluster, arc := range s.arcRPCs {
+			if err = arc.SetStat2(c, st); err != nil {
+				log.Error("s.arcRPC.SetStat2(%s) (%+v) error(%v)", cluster, st, err)
+			}
+		}
+	}
+}