Create & Init Project...

app/job/openplatform/article/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/openplatform/article/cmd:all-srcs",
+        "//app/job/openplatform/article/conf:all-srcs",
+        "//app/job/openplatform/article/dao:all-srcs",
+        "//app/job/openplatform/article/http:all-srcs",
+        "//app/job/openplatform/article/model:all-srcs",
+        "//app/job/openplatform/article/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/CHANGELOG.md

@@ -0,0 +1,183 @@
+# article-job
+
+##### Version 1.20.6
+> 1. 删除脏数据
+
+##### Version 1.20.5
+> 1. fix read split
+
+##### Version 1.20.4
+> 1. 添加神马MIP推送
+
+##### Version 1.20.3
+> 1. recheck without act
+
+##### Version 1.20.2
+> 1. update waitGroup
+
+##### Version 1.20.1
+> 1. 长评接入
+
+##### Version 1.19.6
+> 1. 文章删除后关闭评论
+
+##### Version 1.19.5
+> 1. fix redis
+
+##### Version 1.19.4
+> 1. 消费redis中的用户阅读心跳数据，通过infoc上报阅读时长到数据平台
+
+##### Version 1.19.3
+> 1. add err log
+> 2. fix nil pointer
+
+##### Version 1.19.2
+> 1. 优化sql语句
+
+##### Version 1.19.1
+> 1. 增加分区作者推荐计算
+
+##### Version 1.18.1
+> 1. 浏览数回查 day 或 view 任意为0 认为不生效
+
+##### Version 1.18.0
+> 1. 修复 set to setex
+
+##### Version1.17.0
+> 1. update infoc sdk
+
+##### Version 1.16.1
+> 1. 使用env
+
+##### Version 1.16.0
+> 1. 使用bm
+
+##### Version 1.15.1
+> 1. 修复生成最新投稿的问题
+
+##### Version 1.15.0
+
+> 1. 生成分区数据重构: 生成分区数据(最新文章 排序列表)移动到article-job中 
+> 2. 作者权限缓存重构: 由redis移动到mc中
+> 3. 搜索表增加attributes字段
+
+##### Version 1.14.0
+> 1. 评论/硬币/收藏 使用通用计数流
+> 2. 增加动态消息重试
+
+##### Version 1.13.1
+> 1. 重构游戏缓存 remove redis pie
+##### Version 1.13.0
+> 1. 专栏草稿写库方式重构
+
+### v1.12.0
+> 1. 专栏阅读数防刷需求
+
+### v1.11.0
+> 定时更新热点标签
+
+### v1.10.0
+> 定时更新文集的总阅读数
+
+### v1.9.0
+> 专栏过审/打回/锁定向b+发送消息
+
+### v1.8.0
+> 专栏过审调用流量管理接口
+
+### v1.7.14
+> 修复活动接口
+
+### v1.7.13
+> 定期刷新作者列表
+
+### v1.7.12
+> 点赞时调用bili的活动排序接口
+
+### v1.7.11
+> 接入点赞计数databus
+
+### v1.7.10
+> fixed game sync bug
+
+### v1.7.9
+> 增加image_urls字段
+
+### v1.7.8
+> 接入点赞databus
+
+### v1.7.7
+> 修复草稿panicbug
+
+### v1.7.6
+> 1.游戏白名单迁移到数据库
+
+### v1.7.5
+> 1.接入点赞服务
+
+### v1.7.4
+> 1.fixed bug of update stats ctime
+
+### v1.7.3
+> 1.fixed close channel and game
+
+### v1.7.2
+> 1.游戏的文章变动时候进行通知  
+
+### v1.7.1
+> 1.更新http client   
+
+### v1.7.0
+> 1.增加阅读数排序   
+
+### v1.6.0
+> 1.接入搜索
+
+### v1.5.0
+> 1.动态更新排序列表缓存
+
+### v1.4.1
+> 1.修改刷新最新文章的时间
+
+### v1.4.0
+> 1.监听作者通过/拒绝事件  
+
+### v1.3.4
+> 1.草稿重试改成同步，异步重试有时序问题  
+
+### v1.3.3
+> 1.草稿添加无限重试  
+
+### v1.3.2
+> 1.调整添加、更新、删除文章缓存
+
+### v1.3.1
+> 1.刷新CDN添加APP端URL
+
+### v1.3.0
+> 1.binlog监听改为filtered_articles
+
+### v1.2.3
+> 1.去除filter相关逻辑
+
+### v1.2.2
+> 1.databus中的草稿消息去掉content字段
+
+### v1.2.1
+> 1.过滤操作放到文章里面做
+
+### v1.2.0
+> 1.使用新版过滤rpc  
+
+### v1.1.0
+> 1.草稿异步落库  
+> 2.去掉conf.go中已经不用的刷新推荐数据的配置
+
+### v1.0.2
+> 1.当编辑直接在后台修改文章时更新缓存
+
+### v1.0.1
+> 1.加 prometheus 监控
+
+### v1.0.0
+> 1.项目初始化

app/job/openplatform/article/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+qiuliang
+changxuanran
+lijiadong
+
+# Author 
+changxuanran
+lijiadong
+
+# Reviewer
+qiuliang

app/job/openplatform/article/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- changxuanran
+- lijiadong
+- qiuliang
+labels:
+- job
+- job/openplatform/article
+- openplatform
+options:
+  no_parent_owners: true
+reviewers:
+- changxuanran
+- lijiadong
+- qiuliang

app/job/openplatform/article/README.md

@@ -0,0 +1,10 @@
+#### article-job
+
+##### 项目简介
+> 1.专栏job
+
+##### 编译环境
+> 请只用golang v1.7.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  

app/job/openplatform/article/cmd/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "article-job-test.toml",
+        "goconvey.toml",
+    ],
+    importpath = "go-common/app/job/openplatform/article/cmd",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//app/job/openplatform/article/http:go_default_library",
+        "//app/job/openplatform/article/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/cmd/article-job-test.toml

@@ -0,0 +1,264 @@
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/article-job.pid"
+env="dev" # dev or pro
+dir = "./"
+perf = "0.0.0.0:6740"
+family = "article-job"
+
+[xlog]
+    dir = "/data/log/article-job/"
+
+[tracer]
+    proto = "udp"
+    addr = "172.18.21.30:5140"
+    tag = "platform/article-job"
+
+[app]
+    key = "9cfc54570033cd61"
+    secret = "9d63835fa38fe58a62d9f49ef5da296f"
+
+[SMS]
+    phone = "11111111111,11111111111"
+    token = "token"
+
+[bm]
+     addr = "0.0.0.0:6741"
+     timeout = "1s"
+
+[HTTPClient]
+    key = "9cfc54570033cd61"
+    secret = "9d63835fa38fe58a62d9f49ef5da296f"
+    dial      = "50ms"
+    timeout   = "1s"
+    keepAlive = "60s"
+    [HTTPClient.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[GameHTTPClient]
+    key = "9cfc54570033cd61"
+    secret = "9d63835fa38fe58a62d9f49ef5da296f"
+    dial      = "50ms"
+    timeout   = "1s"
+    keepAlive = "60s"
+    [GameHTTPClient.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+
+[articleSub]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret = "0QEO9F8JuuIxZzNDvklI"
+    group = "Article-Binlog-S"
+    topic = "Article-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[articleStatSub]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret = "0QEO9F8JuuIxZzNDvklI"
+    group = "ArticleStat-Article-S"
+    topic = "ArticleStat-T"
+    action = "sub"
+    name = "article-job/article-stat-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[LikeStatSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "LikesBinlog-MainWebSvr-S"
+    topic = "LikesBinlog-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+[ReplyStatSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "LikesBinlog-MainWebSvr-S"
+    topic = "LikesBinlog-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+[FavoriteStatSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "LikesBinlog-MainWebSvr-S"
+    topic = "LikesBinlog-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+[CoinStatSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "LikesBinlog-MainWebSvr-S"
+    topic = "LikesBinlog-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+[DynamicDbus]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group= "CommTemplate-MainWebSvr-P"
+    topic= "CommTemplate-T"
+    action="pub"
+    name = "article/article-pub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[articleRPC]
+    pullInterval = "10s"
+    [articleRPC.client]
+    token = "123456"
+    proto = "tcp"
+    timeout = "1s"
+    timer = 1000
+    [articleRPC.client.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [articleRPC.zookeeper]
+    root = "/microservice/article-service/"
+    addrs =  ["172.16.33.54:2181"]
+    timeout = "30s"
+
+[tagRPC]
+    pullInterval = "10s"
+    [tagRPC.client]
+    proto = "tcp"
+    addr = "172.16.33.56:6099"
+    timeout = "1s"
+    timer = 1000
+    [tagRPC.client.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [tagRPC.zookeeper]
+    root = "/microservice/tag-service/"
+    addrs = ["172.16.33.54:2181"]
+    timeout = "30s"
+[db]
+    name = "172.16.0.148:3306"
+    dsn = "test:test@tcp(172.16.33.54:3306)/bilibili_article?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "100ms"
+    execTimeout = "100ms"
+    tranTimeout = "200ms"
+ [db.breaker]
+  window = "3s"
+  sleep = "100ms"
+  bucket = 10
+  ratio = 0.5
+  request = 100
+
+[redis]
+    name = "article-job/views"
+    proto = "tcp"
+    addr = "172.16.33.54:6381"
+    active = 100
+    idle = 100
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "80s"
+
+ [artRedis]
+    name = "article-job/views"
+    proto = "tcp"
+    addr = "172.16.33.54:6381"
+    active = 100
+    idle = 100
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "80s"
+
+[cheatInfoc]
+taskID = "1"
+proto = "tcp"
+addr = "127.0.0.1:80"
+chanSize = 10240
+
+[ReadInfoc]
+taskID = "1"
+proto = "tcp"
+addr = "127.0.0.1:80"
+chanSize = 10240
+
+[job]
+    viewCacheTTL = "30m"
+    dupViewCacheTTL = "30m"
+    updateDbInterval = "2m"
+    UpdateSortInterval = "1s"
+    GameCacheExpire = "5m"
+    ListReadCountInterval = "1h"
+    HotspotInterval = "1m"
+    HotspotForceInterval = "30m"
+    actLikeURL = "http://matsuri.bilibili.co/matsuri/api/article/like"
+    FlowURL = "http://uat-archive.api.bilibili.co/videoup/flow/entry/mid"
+    maxNewArtsNum = 1000
+    MaxSortArtsNum = 1000
+    ExpireSortArts = "24h"
+    TTLSortArts = "72h"
+    SortLimitTime = "504h"

app/job/openplatform/article/cmd/goconvey.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+command -v goconvey >/dev/null 2>&1 || { echo >&2 "required goconvey but it's not installed."; echo "Aborting."; echo "Please run commond: go get github.com/smartystreets/goconvey"; exit 1; }
+
+goconvey -excludedDirs "vendor" -packages 1

app/job/openplatform/article/cmd/goconvey.toml

@@ -0,0 +1,197 @@
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/article-job.pid"
+env="dev" # dev or pro
+dir = "./"
+perf = "0.0.0.0:6413"
+family = "article-job"
+
+[xlog]
+    dir = "/data/log/article-job/"
+
+[tracer]
+    proto = "udp"
+    addr = "172.18.21.30:5140"
+    tag = "platform/article-job"
+
+[app]
+    key = "9cfc54570033cd61"
+    secret = "9d63835fa38fe58a62d9f49ef5da296f"
+
+[SMS]
+    phone = "11111111111,11111111111"
+    token = "token"
+
+[bm]
+     addr = "0.0.0.0:6741"
+     timeout = "1s"
+
+[HTTPClient]
+    dial = "1s"
+    timeout = "3s"
+    keepAlive = "60s"
+    key = "9cfc54570033cd61"
+    secret = "9d63835fa38fe58a62d9f49ef5da296f"
+    [HTTPClient.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[GameHTTPClient]
+    key = "b336ebf10107c212"
+    secret = "ZadQSt7mCo9nwVu03SwKDEpgSlRBVtFc"
+    dial      = "50ms"
+    timeout   = "1s"
+    keepAlive = "60s"
+    [GameHTTPClient.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[articleSub]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret = "0QEO9F8JuuIxZzNDvklI"
+    group = "Article-Binlog-S"
+    topic = "Article-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[articleStatSub]
+    key = "0QEO9F8JuuIxZzNDvklH"
+    secret = "0QEO9F8JuuIxZzNDvklI"
+    group = "ArticleStat-Article-S"
+    topic = "ArticleStat-T"
+    action = "sub"
+    name = "article-job/article-stat-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[LikeStatSub]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group = "LikesBinlog-MainWebSvr-S"
+    topic = "LikesBinlog-T"
+    action = "sub"
+    name = "article-job/article-sub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[DynamicDbus]
+    key = "9765cdac5894f2ba"
+    secret = "1448f5f2cd6029f6af6c5d438cd31edd"
+    group= "CommTemplate-MainWebSvr-P"
+    topic= "CommTemplate-T"
+    action="pub"
+    name = "article/article-pub"
+    proto = "tcp"
+    addr = "172.16.33.158:6205"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "60s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[articleRPC]
+    pullInterval = "10s"
+    [articleRPC.client]
+    token = "123456"
+    proto = "tcp"
+    timeout = "1s"
+    timer = 1000
+    [articleRPC.client.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [articleRPC.zookeeper]
+    root = "/microservice/article-service/"
+    addrs =  ["172.16.33.54:2181"]
+    timeout = "30s"
+
+[tagRPC]
+    pullInterval = "10s"
+    [tagRPC.client]
+    proto = "tcp"
+    addr = "172.16.33.56:6099"
+    timeout = "1s"
+    timer = 1000
+    [tagRPC.client.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [tagRPC.zookeeper]
+    root = "/microservice/tag-service/"
+    addrs = ["172.16.33.54:2181"]
+    timeout = "30s"
+
+[db]
+    name = "172.16.0.148:3306"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_article?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "100ms"
+    execTimeout = "100ms"
+    tranTimeout = "200ms"
+    [db.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+    name = "article-job/views"
+    proto = "tcp"
+    addr = "127.0.0.1:6379"
+    active = 10
+    idle = 2
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "80s"
+
+[cheatInfoc]
+taskID = "1"
+proto = "tcp"
+addr = "127.0.0.1:80"
+chanSize = 10240
+
+[job]
+    viewCacheTTL = "30m"
+    dupViewCacheTTL = "30m"
+    updateDbInterval = "2m"
+    GameCacheExpire = "5m"
+    UpdateSortInterval = "1s"
+    HotspotInterval = "1m"
+    HotspotForceInterval = "30m"
+    actLikeURL = "http://matsuri.bilibili.co/matsuri/api/article/like"
+    FlowURL = "http://uat-archive.api.bilibili.co/videoup/flow/entry/mid"

app/job/openplatform/article/cmd/main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/openplatform/article/conf"
+	"go-common/app/job/openplatform/article/http"
+	"go-common/app/job/openplatform/article/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	initConf()
+	initLog()
+	defer log.Close()
+	log.Info("article-job start")
+	srv := service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	initSignal(srv)
+}
+
+func initConf() {
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+}
+
+func initLog() {
+	log.Init(conf.Conf.Xlog)
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+}
+
+func initSignal(srv *service.Service) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("article-job get a signal: %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			if err := srv.Close(); err != nil {
+				log.Error("srv close consumer error(%+v)", err)
+			}
+			return
+		case syscall.SIGHUP:
+			// TODO: reload
+		default:
+			return
+		}
+	}
+}

app/job/openplatform/article/conf/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/openplatform/article/conf",
+    tags = ["automanaged"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/conf/conf.go

@@ -0,0 +1,156 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	xlog "go-common/library/log"
+	"go-common/library/log/infoc"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// Config .
+type Config struct {
+	// Env
+	Env string
+	// App
+	App *bm.App
+	// Xlog is go-common log.
+	Xlog *xlog.Config
+	// Tracer .
+	Tracer *trace.Config
+	// ArchiveSub databus
+	ArticleSub *databus.Config
+	// ArticleStatSub databus
+	ArticleStatSub *databus.Config
+	// LikeStatSub databus
+	LikeStatSub     *databus.Config
+	ReplyStatSub    *databus.Config
+	FavoriteStatSub *databus.Config
+	CoinStatSub     *databus.Config
+	// DynamicDbus pub databus
+	DynamicDbus *databus.Config
+	// BM
+	BM *bm.ServerConfig
+	// HTTPClient .
+	HTTPClient     *bm.ClientConfig
+	GameHTTPClient *bm.ClientConfig
+	// RPC .
+	ArticleRPC *rpc.ClientConfig
+	TagRPC     *rpc.ClientConfig
+	// DB
+	DB *sql.Config
+	// Redis
+	Redis *redis.Config
+	// SMS text message.
+	SMS *sms
+	// CheatInfoc
+	CheatInfoc *infoc.Config
+	// ReadInfoc
+	ReadInfoc *infoc.Config
+	// article interface redis
+	ArtRedis *redis.Config
+	// Job params
+	Job *job
+	// Sitemap
+	Sitemap Sitemap
+}
+
+// Sitemap .
+type Sitemap struct {
+	Interval int64
+	Size     int
+}
+
+type job struct {
+	ViewCacheTTL          xtime.Duration
+	DupViewCacheTTL       xtime.Duration
+	UpdateDbInterval      xtime.Duration
+	UpdateSortInterval    xtime.Duration
+	GameCacheExpire       xtime.Duration
+	ListReadCountInterval xtime.Duration
+	HotspotInterval       xtime.Duration
+	HotspotForceInterval  xtime.Duration
+	ExpireSortArts        xtime.Duration
+	TTLSortArts           xtime.Duration
+	SortLimitTime         xtime.Duration
+	RecommendExpire       xtime.Duration
+	Words                 int64
+	StatDays              int64
+	ActLikeURL            string
+	FlowURL               string
+	MaxNewArtsNum         int64
+	MaxSortArtsNum        int64
+}
+
+type sms struct {
+	Phone string
+	Token string
+}
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init .
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			xlog.Info("config reload")
+			if load() != nil {
+				xlog.Error("config reload err")
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/openplatform/article/dao/BUILD

@@ -0,0 +1,78 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "activity_test.go",
+        "dao_test.go",
+        "game_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+        "reply_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/openplatform/article/model:go_default_library",
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "activity.go",
+        "art_redis.go",
+        "dao.cache.go",
+        "dao.go",
+        "dynamic.go",
+        "flow.go",
+        "game.go",
+        "media.go",
+        "mysql.go",
+        "purge.go",
+        "recommend.go",
+        "redis.go",
+        "reply.go",
+    ],
+    importpath = "go-common/app/job/openplatform/article/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/openplatform/article/model:go_default_library",
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//app/job/openplatform/article/model:go_default_library",
+        "//library/cache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/dao/activity.go

@@ -0,0 +1,43 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_typeArticle = "12"
+)
+
+// LikeSync like sync
+func (d *Dao) LikeSync(c context.Context, aid, likes int64) (err error) {
+	params := url.Values{}
+	params.Set("oid", strconv.FormatInt(aid, 10))
+	params.Set("likes", strconv.FormatInt(likes, 10))
+	params.Set("type", _typeArticle)
+	resp := struct {
+		Code int
+		Data interface{}
+	}{}
+	if err = d.httpClient.Get(c, d.c.Job.ActLikeURL, "", params, &resp); err != nil {
+		log.Error("activity: d.LikeSync.Get(%s) error(%+v)", d.c.Job.ActLikeURL+params.Encode(), err)
+		PromError("activity:同步点赞数")
+		return
+	}
+	if resp.Code != 0 {
+		// 未参与活动
+		if resp.Code == -403 {
+			return
+		}
+		err = ecode.Int(resp.Code)
+		log.Error("activity: d.LikeSync.Get(%s) error(%+v)", d.c.Job.ActLikeURL+"?"+params.Encode(), resp)
+		PromError("activity:同步点赞数")
+		return
+	}
+	log.Info("activity: dao.LikeSync success aid: %v count: %v ", aid, likes)
+	return
+}

app/job/openplatform/article/dao/activity_test.go

@@ -0,0 +1,15 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_LikeSync(t *testing.T) {
+	Convey("work", t, WithDao(func(d *Dao) {
+		err := d.LikeSync(context.Background(), 1, 20)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/openplatform/article/dao/art_redis.go

@@ -0,0 +1,78 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+func sortedKey(categoryID int64, field int) string {
+	return fmt.Sprintf("art_sort_%d_%d", categoryID, field)
+}
+
+// ExpireSortCache expire sort cache
+func (d *Dao) ExpireSortCache(c context.Context, categoryID int64, field int) (ok bool, err error) {
+	key := sortedKey(categoryID, field)
+	conn := d.artRedis.Get(c)
+	defer conn.Close()
+	var ttl int64
+	if ttl, err = redis.Int64(conn.Do("TTL", key)); err != nil {
+		PromError("redis:排序缓存ttl")
+		log.Error("conn.Do(TTL, %s) error(%+v)", key, err)
+	}
+	if ttl > (d.redisSortTTL - d.redisSortExpire) {
+		ok = true
+	}
+	return
+}
+
+// AddSortCaches add sort articles cache
+func (d *Dao) AddSortCaches(c context.Context, categoryID int64, field int, arts [][2]int64, maxLength int64) (err error) {
+	var (
+		id, score int64
+		key       = sortedKey(categoryID, field)
+		conn      = d.artRedis.Get(c)
+		count     int
+	)
+	defer conn.Close()
+	if len(arts) == 0 {
+		return
+	}
+	if err = conn.Send("DEL", key); err != nil {
+		PromError("redis:删除排序缓存")
+		log.Error("conn.Send(DEL, %s) error(%+v)", key, err)
+		return
+	}
+	count++
+	for _, art := range arts {
+		id = art[0]
+		score = art[1]
+		if err = conn.Send("ZADD", key, "CH", score, id); err != nil {
+			PromError("redis:增加排序缓存")
+			log.Error("conn.Send(ZADD, %s, %d, %v) error(%+v)", key, score, id, err)
+			return
+		}
+		count++
+	}
+	if err = conn.Send("EXPIRE", key, d.redisSortTTL); err != nil {
+		PromError("redis:排序缓存设定过期")
+		log.Error("conn.Send(EXPIRE, %s, %d) error(%+v)", key, d.redisSortTTL, err)
+		return
+	}
+	count++
+	if err = conn.Flush(); err != nil {
+		PromError("redis:增加排序缓存flush")
+		log.Error("conn.Flush error(%+v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			PromError("redis:增加排序缓存receive")
+			log.Error("conn.Receive error(%+v)", err)
+			return
+		}
+	}
+	return
+}

app/job/openplatform/article/dao/dao.cache.go

@@ -0,0 +1,47 @@
+// Code generated by $GOPATH/src/go-common/app/tool/cache/gen. DO NOT EDIT.
+
+/*
+  Package dao is a generated cache proxy package.
+  It is generated from:
+  type _cache interface {
+		GameList(c context.Context) (res []int64, err error)
+	}
+*/
+
+package dao
+
+import (
+	"context"
+
+	"go-common/library/net/metadata"
+	"go-common/library/stat/prom"
+)
+
+var _ _cache
+
+// GameList get data from cache if miss will call source method, then add to cache.
+func (d *Dao) GameList(c context.Context) (res []int64, err error) {
+	addCache := true
+	res, err = d.CacheGameList(c)
+	if err != nil {
+		addCache = false
+		err = nil
+	}
+	if len(res) != 0 {
+		prom.CacheHit.Incr("GameList")
+		return
+	}
+	prom.CacheMiss.Incr("GameList")
+	res, err = d.RawGameList(c)
+	if err != nil {
+		return
+	}
+	var miss = res
+	if !addCache {
+		return
+	}
+	d.cache.Save(func() {
+		d.AddCacheGameList(metadata.WithContext(c), miss)
+	})
+	return
+}

app/job/openplatform/article/dao/dao.go

@@ -0,0 +1,106 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/article/conf"
+	"go-common/library/cache"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+)
+
+// Dao .
+type Dao struct {
+	c                             *conf.Config
+	db                            *xsql.DB
+	redis                         *redis.Pool
+	artRedis                      *redis.Pool
+	httpClient                    *bm.Client
+	gameHTTPClient                *bm.Client
+	viewCacheTTL, gameCacheExpire int64
+	dupViewCacheTTL               int64
+	redisSortExpire               int64
+	redisSortTTL                  int64
+	// stmt
+	updateSearchStmt      *xsql.Stmt
+	delSearchStmt         *xsql.Stmt
+	updateSearchStatsStmt *xsql.Stmt
+	gameStmt              *xsql.Stmt
+	cheatStmt             *xsql.Stmt
+	newestArtsMetaStmt    *xsql.Stmt
+	searchArtsStmt        *xsql.Stmt
+	updateRecheckStmt     *xsql.Stmt
+	getRecheckStmt        *xsql.Stmt
+	settingsStmt          *xsql.Stmt
+	midByPubtimeStmt      *xsql.Stmt
+	statByMidStmt         *xsql.Stmt
+	dynamicDbus           *databus.Databus
+	cache                 *cache.Cache
+}
+
+var (
+	errorsCount = prom.BusinessErrCount
+	cacheLen    = prom.BusinessInfoCount
+	infosCount  = prom.BusinessInfoCount
+)
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:               c,
+		db:              xsql.NewMySQL(c.DB),
+		redis:           redis.NewPool(c.Redis),
+		artRedis:        redis.NewPool(c.ArtRedis),
+		httpClient:      bm.NewClient(c.HTTPClient),
+		gameHTTPClient:  bm.NewClient(c.GameHTTPClient),
+		viewCacheTTL:    int64(time.Duration(c.Job.ViewCacheTTL) / time.Second),
+		dupViewCacheTTL: int64(time.Duration(c.Job.DupViewCacheTTL) / time.Second),
+		gameCacheExpire: int64(time.Duration(c.Job.GameCacheExpire) / time.Second),
+		redisSortExpire: int64(time.Duration(c.Job.ExpireSortArts) / time.Second),
+		redisSortTTL:    int64(time.Duration(c.Job.TTLSortArts) / time.Second),
+		dynamicDbus:     databus.New(c.DynamicDbus),
+		cache:           cache.New(1, 1024),
+	}
+	d.updateSearchStmt = d.db.Prepared(_updateSearch)
+	d.delSearchStmt = d.db.Prepared(_delSearch)
+	d.updateSearchStatsStmt = d.db.Prepared(_updateSearchStats)
+	d.newestArtsMetaStmt = d.db.Prepared(_newestArtsMetaSQL)
+	d.gameStmt = d.db.Prepared(_gameList)
+	d.cheatStmt = d.db.Prepared(_allCheat)
+	d.searchArtsStmt = d.db.Prepared(_searchArticles)
+	d.updateRecheckStmt = d.db.Prepared(_updateCheckState)
+	d.getRecheckStmt = d.db.Prepared(_checkStateSQL)
+	d.settingsStmt = d.db.Prepared(_settingsSQL)
+	d.midByPubtimeStmt = d.db.Prepared(_midsByPublishTimeSQL)
+	d.statByMidStmt = d.db.Prepared(_statByMidSQL)
+	return
+}
+
+//go:generate $GOPATH/src/go-common/app/tool/cache/gen
+type _cache interface {
+	GameList(c context.Context) (res []int64, err error)
+}
+
+// PromError prometheus error count.
+func PromError(name string) {
+	errorsCount.Incr(name)
+}
+
+// PromInfo prometheus info count.
+func PromInfo(name string) {
+	infosCount.Incr(name)
+}
+
+// Ping reports the health of the db/cache etc.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.db.Ping(c); err != nil {
+		PromError("db:Ping")
+		return
+	}
+	err = d.pingRedis(c)
+	return
+}

app/job/openplatform/article/dao/dao_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/openplatform/article/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../cmd/goconvey.toml")
+		flag.Set("conf", dir)
+		flag.Parse()
+		conf.Init()
+		d := New(conf.Conf)
+		f(d)
+	}
+}
+
+func Test_Reply(t *testing.T) {
+	Convey("open reply", t, WithDao(func(d *Dao) {
+		var (
+			err error
+			c   = context.TODO()
+		)
+		err = d.OpenReply(c, 88, 88)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/openplatform/article/dao/dynamic.go

@@ -0,0 +1,33 @@
+package dao
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+)
+
+const _dynamicArt = 64
+
+// PubDynamic pub dynamic
+func (d *Dao) PubDynamic(c context.Context, mid int64, aid int64, show bool, comment string, ts int64, dynamicIntro string) (err error) {
+	msg := &model.DynamicMsg{}
+	msg.Card.Type = _dynamicArt
+	msg.Card.Rid = aid
+	msg.Card.OwnerID = mid
+	if show {
+		msg.Card.Show = 1
+	}
+	msg.Card.Comment = comment
+	msg.Card.Ts = ts
+	msg.Card.Dynamic = dynamicIntro
+	if err = d.dynamicDbus.Send(c, strconv.FormatInt(aid, 10), msg); err != nil {
+		PromError("dynamic:发送动态消息")
+		log.Error("dynamic: d.SendPubDynamic(%+v) error(%+v)", msg, err)
+		return
+	}
+	PromInfo("databus:发送动态消息")
+	log.Info("dynamic: dao.PubDynamic(%+v)", msg)
+	return
+}

app/job/openplatform/article/dao/flow.go

@@ -0,0 +1,39 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_typeArticleFlow = "3"
+)
+
+// FlowSync 流量管理同步过审文章
+func (d *Dao) FlowSync(c context.Context, mid, aid int64) (err error) {
+	params := url.Values{}
+	params.Set("oid", strconv.FormatInt(aid, 10))
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	params.Set("business", _typeArticleFlow)
+	resp := struct {
+		Code int
+		Data interface{}
+	}{}
+	if err = d.httpClient.Post(c, d.c.Job.FlowURL, "", params, &resp); err != nil {
+		log.Error("flow: d.FlowSync.Post(%s) error(%+v)", d.c.Job.FlowURL+params.Encode(), err)
+		PromError("flow:文章过审")
+		return
+	}
+	if resp.Code != 0 {
+		err = ecode.Int(resp.Code)
+		log.Error("flow: d.FlowSync.Post(%s) error(%+v)", d.c.Job.FlowURL+"?"+params.Encode(), resp)
+		PromError("flow:文章过审")
+		return
+	}
+	log.Info("flow: dao.FlowSync success aid: %v mid: %v ", aid, mid)
+	return
+}

app/job/openplatform/article/dao/game.go

@@ -0,0 +1,111 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+	"time"
+
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/cache/redis"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_gameSyncURL = "http://line3-h5-mobile-api.biligame.com/h5/internal/article/sync"
+	_typeAdd     = "1"
+	_typeUpdate  = "2"
+	_typeDel     = "3"
+	_gameKey     = "artjob_game_mids"
+)
+
+// GameSync game sync
+func (d *Dao) GameSync(c context.Context, action string, cvid int64) (err error) {
+	params := url.Values{}
+	params.Set("timestamp", strconv.FormatInt(time.Now().Unix()*1000, 10))
+	var tp string
+	if action == model.ActInsert {
+		tp = _typeAdd
+	} else if action == model.ActUpdate {
+		tp = _typeUpdate
+	} else {
+		tp = _typeDel
+	}
+	params.Set("type", tp)
+	params.Set("article_id", strconv.FormatInt(cvid, 10))
+	resp := struct {
+		Code int
+	}{}
+	if err = d.gameHTTPClient.Post(c, _gameSyncURL, "", params, &resp); err != nil {
+		log.Error("game: d.gameHTTPClient.Post(%s) error(%+v)", _gameSyncURL+params.Encode(), err)
+		PromError("game:同步数据")
+		return
+	}
+	if resp.Code != 0 {
+		err = ecode.Int(resp.Code)
+		log.Error("game: d.gameHTTPClient.Get(%s) code: %v error(%+v)", _gameSyncURL, resp.Code, err)
+		PromError("game:同步数据")
+		return
+	}
+	log.Info("game: dao.GameSync success action: %v cvid: %v", action, cvid)
+	return
+}
+
+// CacheGameList .
+func (d *Dao) CacheGameList(c context.Context) (mids []int64, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if mids, err = redis.Int64s(conn.Do("ZRANGE", _gameKey, 0, -1)); err != nil {
+		PromError("redis:游戏列表缓存")
+		log.Error("conn.Zrange(%s) error(%+v)", _gameKey, err)
+	}
+	return
+}
+
+// AddCacheGameList .
+func (d *Dao) AddCacheGameList(c context.Context, mids []int64) (err error) {
+	var (
+		key   = _gameKey
+		conn  = d.redis.Get(c)
+		count int
+	)
+	defer conn.Close()
+	if len(mids) == 0 {
+		return
+	}
+	if err = conn.Send("DEL", key); err != nil {
+		PromError("redis:删除游戏列表缓存")
+		log.Error("conn.Send(DEL, %s) error(%+v)", key, err)
+		return
+	}
+	count++
+	for i, mid := range mids {
+		score := i
+		if err = conn.Send("ZADD", key, "CH", score, mid); err != nil {
+			PromError("redis:增加游戏列表缓存")
+			log.Error("conn.Send(ZADD, %s, %d, %v) error(%+v)", key, score, mid, err)
+			return
+		}
+		count++
+	}
+	if err = conn.Send("EXPIRE", key, d.gameCacheExpire); err != nil {
+		PromError("redis:游戏列表缓存设定过期")
+		log.Error("conn.Send(EXPIRE, %s, %d) error(%+v)", key, d.gameCacheExpire, err)
+		return
+	}
+	count++
+	if err = conn.Flush(); err != nil {
+		PromError("redis:增加游戏列表缓存flush")
+		log.Error("conn.Flush error(%+v)", err)
+		return
+	}
+	for i := 0; i < count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			PromError("redis:增加游戏列表缓存receive")
+			log.Error("conn.Receive error(%+v)", err)
+			return
+		}
+	}
+	return
+}

app/job/openplatform/article/dao/game_test.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_GameSync(t *testing.T) {
+	Convey("work", t, WithDao(func(d *Dao) {
+		err := d.GameSync(context.Background(), "add", 1)
+		So(err, ShouldBeNil)
+	}))
+}
+
+func Test_NewGameCache(t *testing.T) {
+	Convey("work", t, WithDao(func(d *Dao) {
+		res, err := d.GameList(context.TODO())
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+}

app/job/openplatform/article/dao/media.go

@@ -0,0 +1,41 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const _delScoreURL = "http://api.bilibili.co/pgc/internal/review/score/delete"
+
+// DelScore .
+func (d *Dao) DelScore(c context.Context, aid, mediaID, mid int64) (err error) {
+	if mediaID == 0 || mid == 0 || aid == 0 {
+		return
+	}
+	params := url.Values{}
+	params.Set("media_id", strconv.FormatInt(mediaID, 10))
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	params.Set("oid", strconv.FormatInt(aid, 10))
+	params.Set("from", "1")
+	var resp struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	}
+	err = d.httpClient.Post(c, _delScoreURL, "", params, &resp)
+	if err != nil {
+		PromError("media:番剧删除评分接口")
+		log.Error("media: d.client.Post(%s, data:%s) error(%+v)", _delScoreURL, params.Encode(), err)
+		return
+	}
+	if resp.Code != 0 {
+		PromError("media:番剧删除评分接口")
+		log.Error("media: d.client.Post(%s, data:%s) res: %+v", _delScoreURL, params.Encode(), resp)
+		err = ecode.Int(resp.Code)
+	}
+	log.Info("media: del score success(media_id: %d, mid: %d, oid: %d)", mediaID, mid, aid)
+	return
+}

app/job/openplatform/article/dao/mysql.go

@@ -0,0 +1,411 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+const (
+	_sharding = 100
+
+	// stat
+	_statSQL   = "SELECT article_id,favorite,reply,share,likes,dislike,view,coin FROM article_stats_%s WHERE article_id=%d and deleted_time =0"
+	_upStatSQL = `INSERT INTO article_stats_%s (article_id,favorite,reply,share,likes,dislike,view,coin) VALUES (?,?,?,?,?,?,?,?) 
+					ON DUPLICATE KEY UPDATE favorite=?,reply=?,share=?,likes=?,dislike=?,view=?,coin=?`
+	_updateSearch = `INSERT INTO search_articles(ctime, article_id, category_id, title, summary, template_id, mid, image_urls, publish_time, content, tags, stats_view, stats_favorite, stats_likes, stats_dislike, stats_reply, stats_share, stats_coin, origin_image_urls, attributes, keywords)
+	     VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE ctime = ?, category_id =?, title=?, summary=?, template_id=?, mid=?, image_urls=?, publish_time=?, content=?, tags=?, stats_view=?, stats_favorite=?, stats_likes=?, stats_dislike=?, stats_reply=?, stats_share=?, stats_coin = ?, origin_image_urls = ?, attributes = ?, keywords = ?`
+	_delSearch                 = "DELETE FROM search_articles where article_id = ?"
+	_articleContentSQL         = "SELECT content FROM filtered_article_contents_%s WHERE article_id = ?"
+	_updateSearchStats         = "UPDATE search_articles SET stats_view=?, stats_favorite=?, stats_likes=?, stats_dislike=?, stats_reply=?, stats_share=?, stats_coin =? where article_id = ?"
+	_gameList                  = "SELECT mid FROM white_list_users WHERE group_id = 4 AND deleted = 0"
+	_allCheat                  = "SELECT article_id, lv FROM stats_filters WHERE deleted_time = 0"
+	_newestArtsMetaSQL         = "SELECT article_id, publish_time, attributes FROM filtered_articles ORDER BY publish_time DESC LIMIT ?"
+	_newestArtCategorysMetaSQL = "SELECT article_id, publish_time, attributes FROM filtered_articles where category_id in (%s) ORDER BY publish_time DESC LIMIT %d"
+	_searchArticles            = "select article_id, category_id, attributes, stats_view, stats_reply, stats_favorite, stats_likes, stats_coin from search_articles where publish_time >= ? and publish_time < ?"
+	_checkStateSQL             = "SELECT publish_time,check_state FROM articles WHERE id = ? and deleted_time = 0"
+	_updateCheckState          = "UPDATE articles SET check_state = 3 WHERE id = ?"
+	_settingsSQL               = "SELECT name,value FROM article_settings WHERE deleted_time=0"
+	_midsByPublishTimeSQL      = "SELECT mid FROM articles WHERE publish_time > ? and state = 0 and deleted_time = 0 group by mid"
+	_statByMidSQL              = "select count(*), sum(words), category_id from articles where mid = ? and state = 0 and deleted_time = 0 group by category_id"
+	_keywordsSQL               = "select tags from article_contents_%s where article_id = ?"
+	_actIDSQL                  = "select act_id from articles where id = ?"
+	_lastModsArtsSQL           = "select id from articles where state in (0,5,6,7) and deleted_time = 0 order by mtime desc limit ?"
+)
+
+var _searchInterval = int64(3 * 24 * 3600)
+
+func (d *Dao) hit(id int64) string {
+	return fmt.Sprintf("%02d", id%_sharding)
+}
+
+// Stat returns stat info.
+func (d *Dao) Stat(c context.Context, aid int64) (stat *artmdl.StatMsg, err error) {
+	stat = &artmdl.StatMsg{}
+	err = d.db.QueryRow(c, fmt.Sprintf(_statSQL, d.hit(aid), aid)).Scan(&stat.Aid, &stat.Favorite, &stat.Reply, &stat.Share, &stat.Like, &stat.Dislike, &stat.View, &stat.Coin)
+	if err == sql.ErrNoRows {
+		err = nil
+		stat = nil
+	} else if err != nil {
+		log.Error("Stat(%v) error(%+v)", aid, err)
+		PromError("db:读取计数")
+	}
+	return
+}
+
+// Update updates stat in db.
+func (d *Dao) Update(c context.Context, stat *artmdl.StatMsg) (rows int64, err error) {
+	res, err := d.db.Exec(c, fmt.Sprintf(_upStatSQL, d.hit(stat.Aid)), stat.Aid, *stat.Favorite, *stat.Reply, *stat.Share, *stat.Like, *stat.Dislike, *stat.View, *stat.Coin,
+		*stat.Favorite, *stat.Reply, *stat.Share, *stat.Like, *stat.Dislike, *stat.View, *stat.Coin)
+	if err != nil {
+		log.Error("Update(%d,%+v) error(%+v)", stat.Aid, stat, err)
+		PromError("db:更新计数")
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// UpdateSearch update search article table
+func (d *Dao) UpdateSearch(c context.Context, a *model.SearchArticle) (err error) {
+	_, err = d.updateSearchStmt.Exec(c, a.CTime, a.ID, a.CategoryID, a.Title, a.Summary, a.TemplateID, a.Mid, a.ImageURLs, a.PublishTime, a.Content, a.Tags, a.StatsView, a.StatsFavorite, a.StatsLikes, a.StatsDisLike, a.StatsReply, a.StatsShare, a.StatsCoin, a.OriginImageURLs, a.Attributes, a.Keywords,
+		a.CTime, a.CategoryID, a.Title, a.Summary, a.TemplateID, a.Mid, a.ImageURLs, a.PublishTime, a.Content, a.Tags, a.StatsView, a.StatsFavorite, a.StatsLikes, a.StatsDisLike, a.StatsReply, a.StatsShare, a.StatsCoin, a.OriginImageURLs, a.Attributes, a.Keywords)
+	if err != nil {
+		PromError("db:更新搜索表")
+		log.Error("UpdateSearch(%+v) error(%+v)", a, err)
+	}
+	return
+}
+
+// DelSearch del search article table
+func (d *Dao) DelSearch(c context.Context, aid int64) (err error) {
+	_, err = d.delSearchStmt.Exec(c, aid)
+	if err != nil {
+		PromError("db:删除搜索表")
+		log.Error("DelSearch(%v) error(%+v)", aid, err)
+		return
+	}
+	return
+}
+
+// UpdateRecheck update recheck  table
+func (d *Dao) UpdateRecheck(c context.Context, aid int64) (err error) {
+	_, err = d.updateRecheckStmt.Exec(c, aid)
+	if err != nil {
+		PromError("db:修改回查状态")
+		log.Error("UpdateRecheck(%v) error(%+v)", aid, err)
+	}
+	return
+}
+
+// ArticleContent get article content
+func (d *Dao) ArticleContent(c context.Context, id int64) (res string, err error) {
+	contentSQL := fmt.Sprintf(_articleContentSQL, d.hit(id))
+	if err = d.db.QueryRow(c, contentSQL, id).Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		PromError("db:ArticleContent")
+		log.Error("dao.ArticleContent(%s) error(%+v)", contentSQL, err)
+	}
+	return
+}
+
+// GetRecheckInfo get article recheck
+func (d *Dao) GetRecheckInfo(c context.Context, id int64) (publishTime int64, checkState int, err error) {
+	if err = d.getRecheckStmt.QueryRow(c, id).Scan(&publishTime, &checkState); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		PromError("db:ReCheckQuery")
+		log.Error("dao.GetRecheckInfo(%d) error(%+v)", id, err)
+	}
+	return
+}
+
+// UpdateSearchStats update search stats
+func (d *Dao) UpdateSearchStats(c context.Context, stat *artmdl.StatMsg) (err error) {
+	_, err = d.updateSearchStatsStmt.Exec(c, *stat.View, *stat.Favorite, *stat.Like, *stat.Dislike, *stat.Reply, *stat.Share, *stat.Coin, stat.Aid)
+	if err != nil {
+		log.Error("updateSearchStatsStmt(%d,%+v) error(%+v)", stat.Aid, stat, err)
+		PromError("db:更新搜索计数")
+	}
+	return
+}
+
+// RawGameList game list
+func (d *Dao) RawGameList(c context.Context) (mids []int64, err error) {
+	rows, err := d.gameStmt.Query(c)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var id int64
+		if rows.Scan(&id); err != nil {
+			PromError("db:GameList")
+			log.Error("dao.GameList() error(%+v)", err)
+			return
+		}
+		mids = append(mids, id)
+	}
+	return
+}
+
+// CheatArts cheat list
+func (d *Dao) CheatArts(c context.Context) (res map[int64]int, err error) {
+	rows, err := d.cheatStmt.Query(c)
+	if err != nil {
+		return
+	}
+	res = make(map[int64]int)
+	defer rows.Close()
+	defer func() {
+		if err != nil {
+			PromError("db:CheatArts")
+			log.Error("dao.CheatArts() error(%+v)", err)
+		}
+	}()
+	for rows.Next() {
+		var id int64
+		var lv int
+		if rows.Scan(&id, &lv); err != nil {
+			return
+		}
+		res[id] = lv
+	}
+	err = rows.Err()
+	return
+}
+
+// NewestArtIDs find newest article's id
+func (d *Dao) NewestArtIDs(c context.Context, limit int64) (res [][2]int64, err error) {
+	rows, err := d.newestArtsMetaStmt.Query(c, limit)
+	if err != nil {
+		PromError("db:最新文章")
+		log.Error("dao.newestArtsMetaStmt.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			id        int64
+			ptime     int64
+			attribute int32
+		)
+		if err = rows.Scan(&id, &ptime, &attribute); err != nil {
+			PromError("db:最新文章scan")
+			log.Error("dao.NewestArtIDs.rows.Scan error(%+v)", err)
+			return
+		}
+		if artmdl.NoDistributeAttr(attribute) || artmdl.NoRegionAttr(attribute) {
+			continue
+		}
+		res = append(res, [2]int64{id, ptime})
+	}
+	if err = rows.Err(); err != nil {
+		PromError("db:最新文章")
+		log.Error("dao.NewestArtIDs.rows error(%+v)", err)
+	}
+	return
+}
+
+// NewestArtIDByCategory find newest article's id
+func (d *Dao) NewestArtIDByCategory(c context.Context, cids []int64, limit int64) (res [][2]int64, err error) {
+	sql := fmt.Sprintf(_newestArtCategorysMetaSQL, xstr.JoinInts(cids), limit)
+	rows, err := d.db.Query(c, sql)
+	if err != nil {
+		PromError("db:最新文章")
+		log.Error("dao.NewestArtIDByCategorys.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			id        int64
+			ptime     int64
+			attribute int32
+		)
+		if err = rows.Scan(&id, &ptime, &attribute); err != nil {
+			PromError("db:最新文章scan")
+			log.Error("dao.NewestArtIDByCategory.rows.Scan error(%+v)", err)
+			return
+		}
+		if artmdl.NoDistributeAttr(attribute) || artmdl.NoRegionAttr(attribute) {
+			continue
+		}
+		res = append(res, [2]int64{id, ptime})
+	}
+	if err = rows.Err(); err != nil {
+		PromError("db:最新文章")
+		log.Error("dao.NewestArtIDByCategory.Scan error(%+v)", err)
+	}
+	return
+}
+
+// SearchArts get articles publish time after ptime
+func (d *Dao) SearchArts(c context.Context, ptime int64) (res []*model.SearchArticle, err error) {
+	var rows *sql.Rows
+	now := time.Now().Unix()
+	for ; ptime < now; ptime += _searchInterval {
+		if rows, err = d.searchArtsStmt.Query(c, ptime, ptime+_searchInterval); err != nil {
+			PromError("db:searchArts")
+			log.Error("mysql: search arts Query() error(%+v)", err)
+			return
+		}
+		defer rows.Close()
+		for rows.Next() {
+			ba := &model.SearchArticle{}
+			if err = rows.Scan(&ba.ID, &ba.CategoryID, &ba.Attributes, &ba.StatsView, &ba.StatsReply, &ba.StatsFavorite, &ba.StatsLikes, &ba.StatsCoin); err != nil {
+				PromError("db:searchArts")
+				log.Error("mysql: search arts Scan() error(%+v)", err)
+				return
+			}
+			res = append(res, ba)
+		}
+		if err = rows.Err(); err != nil {
+			PromError("db:searchArts")
+			log.Error("mysql: search arts Query() error(%+v)", err)
+			return
+		}
+	}
+	return
+}
+
+// Settings gets article settings.
+func (d *Dao) Settings(c context.Context) (res map[string]string, err error) {
+	var rows *sql.Rows
+	if rows, err = d.settingsStmt.Query(c); err != nil {
+		PromError("db:文章配置查询")
+		log.Error("mysql: db.settingsStmt.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[string]string)
+	for rows.Next() {
+		var name, value string
+		if err = rows.Scan(&name, &value); err != nil {
+			PromError("文章配置scan")
+			log.Error("mysql: rows.Scan error(%+v)", err)
+			return
+		}
+		res[name] = value
+	}
+	err = rows.Err()
+	if err = rows.Err(); err != nil {
+		PromError("db:loadSettings")
+		log.Error("mysql: load settings Query() error(%+v)", err)
+	}
+	return
+}
+
+//MidsByPublishTime get mids by publish time
+func (d *Dao) MidsByPublishTime(c context.Context, pubTime int64) (mids []int64, err error) {
+	var rows *sql.Rows
+	if rows, err = d.midByPubtimeStmt.Query(c, pubTime); err != nil {
+		PromError("db:查询近7天mid")
+		log.Error("mysql: db.MidsByPublishTime.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mid int64
+		if err = rows.Scan(&mid); err != nil {
+			PromError("查询近7天mid scan")
+			log.Error("mysql: rows.Scan error(%+v)", err)
+			return
+		}
+		mids = append(mids, mid)
+	}
+	err = rows.Err()
+	if err = rows.Err(); err != nil {
+		PromError("db:MidsByPublishTime")
+		log.Error("mysql: get mids by publish time Query() error(%+v)", err)
+	}
+	return
+}
+
+//StatByMid get author info by mid
+func (d *Dao) StatByMid(c context.Context, mid int64) (res map[int64][2]int64, err error) {
+	res = make(map[int64][2]int64)
+	var rows *sql.Rows
+	if rows, err = d.statByMidStmt.Query(c, mid); err != nil {
+		PromError("db:作者分区数据")
+		log.Error("mysql: db.statByMidStmt.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var (
+			data [2]int64
+			cate int64
+		)
+		if err = rows.Scan(&data[0], &data[1], &cate); err != nil {
+			PromError("作者分区数据 scan")
+			log.Error("mysql: rows.Scan error(%+v)", err)
+			return
+		}
+		res[cate] = data
+	}
+	err = rows.Err()
+	if err = rows.Err(); err != nil {
+		PromError("db:StatByMid")
+		log.Error("mysql: get stat infos by mid Query() error(%+v)", err)
+	}
+	return
+}
+
+// Keywords .
+func (d *Dao) Keywords(c context.Context, id int64) (keywords string, err error) {
+	var sqlStr = fmt.Sprintf(_keywordsSQL, d.hit(id))
+	row := d.db.QueryRow(c, sqlStr, id)
+	if err = row.Scan(&keywords); err != nil {
+		PromError("db:Keywords")
+		log.Error("d.keywords scan error(%+v)", err)
+	}
+	return
+}
+
+// IsAct .
+func (d *Dao) IsAct(c context.Context, id int64) (res bool) {
+	var actID int64
+	if err := d.db.QueryRow(c, _actIDSQL, id).Scan(&actID); err != nil {
+		PromError("db:IsAct")
+		log.Error("d.IsAct scan error(%+v)", err)
+		return
+	}
+	if actID > 0 {
+		res = true
+	}
+	return
+}
+
+// LastModIDs .
+func (d *Dao) LastModIDs(c context.Context, size int) (aids []int64, err error) {
+	var (
+		id   int64
+		rows *sql.Rows
+	)
+	if rows, err = d.db.Query(c, _lastModsArtsSQL, size); err != nil {
+		PromError("db:LastModIDs")
+		log.Error("d.LastModIDs query error(%+v) size(%d)", err, size)
+		return
+	}
+	for rows.Next() {
+		if err = rows.Scan(&id); err != nil {
+			PromError("db:LastModIDs")
+			log.Error("d.LastModIDs scan error(%+v) size(%d)", err, size)
+			return
+		}
+		aids = append(aids, id)
+	}
+	return
+}

app/job/openplatform/article/dao/mysql_test.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Update(t *testing.T) {
+	var (
+		c    = context.TODO()
+		cnt1 = int64(1)
+		cnt2 = int64(2)
+		st1  = &artmdl.StatMsg{
+			Aid:      888,
+			View:     &cnt1,
+			Favorite: &cnt1,
+			Like:     &cnt1,
+			Dislike:  &cnt1,
+			Reply:    &cnt1,
+			Share:    &cnt1,
+		}
+		st2 = &artmdl.StatMsg{
+			Aid:      888,
+			View:     &cnt2,
+			Favorite: &cnt2,
+			Like:     &cnt2,
+			Dislike:  &cnt2,
+			Reply:    &cnt2,
+			Share:    &cnt2,
+		}
+	)
+	Convey("update stats", t, WithDao(func(d *Dao) {
+		rows, err := d.Update(c, st1)
+		So(err, ShouldBeNil)
+		So(rows, ShouldBeGreaterThan, 0)
+
+		Convey("get st1", func() {
+			stat, err1 := d.Stat(c, 888)
+			So(err1, ShouldBeNil)
+			So(stat, ShouldResemble, st1)
+		})
+
+		rows, err = d.Update(c, st2)
+		So(err, ShouldBeNil)
+		So(rows, ShouldBeGreaterThan, 0)
+
+		Convey("get st2", func() {
+			stat, err1 := d.Stat(c, 888)
+			So(err1, ShouldBeNil)
+			So(stat, ShouldResemble, st2)
+		})
+	}))
+}
+
+func Test_GameList(t *testing.T) {
+	Convey("work", t, WithDao(func(d *Dao) {
+		mids, err := d.GameList(context.Background())
+		So(err, ShouldBeNil)
+		So(mids, ShouldNotBeEmpty)
+	}))
+}
+
+func Test_NewestArtIDByCategory(t *testing.T) {
+	var _dataCategory = int64(6)
+	Convey("get data", t, WithDao(func(d *Dao) {
+		res, err := d.NewestArtIDByCategory(context.TODO(), []int64{_dataCategory}, 100)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+	Convey("no data", t, WithDao(func(d *Dao) {
+		res, err := d.NewestArtIDByCategory(context.TODO(), []int64{1000}, 100)
+		So(err, ShouldBeNil)
+		So(res, ShouldBeEmpty)
+	}))
+}
+
+func Test_NewestArtIDs(t *testing.T) {
+	Convey("get data", t, WithDao(func(d *Dao) {
+		res, err := d.NewestArtIDs(context.TODO(), 100)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+}
+
+func Test_SearchArts(t *testing.T) {
+	Convey("should get data", t, WithDao(func(d *Dao) {
+		_searchInterval = 24 * 3600 * 365
+		res, err := d.SearchArts(context.TODO(), 0)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+}

app/job/openplatform/article/dao/purge.go

@@ -0,0 +1,32 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"go-common/library/log"
+)
+
+const _purgeURL = "http://cp.bilibili.co/api_purge.php"
+
+// PurgeCDN purges cdn.
+func (d *Dao) PurgeCDN(c context.Context, file string) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Second)
+		if e := d.PushCDN(c, file); e != nil {
+			log.Error("d.PushCDN(%s) error(%+v)", file, e)
+		}
+	}()
+	params := url.Values{}
+	params.Set("sid", "2d0586d2c63fb82a69b20c8992811055")
+	params.Set("file", file)
+	if err = d.httpClient.Get(c, _purgeURL, "", params, nil); err != nil {
+		log.Error("d.httpClient.Get() error(%+v)", err)
+		PromError("purge:刷新CDN")
+	}
+	return
+}

app/job/openplatform/article/dao/recommend.go

@@ -0,0 +1,95 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+// AddcategoriesAuthors .
+func (d *Dao) AddcategoriesAuthors(c context.Context, data map[int64][]int64) (err error) {
+	if len(data) == 0 {
+		return
+	}
+	conn := d.artRedis.Get(c)
+	defer conn.Close()
+	for k, x := range data {
+		key := fmt.Sprintf("recommends:authors:%d", k)
+		args := redis.Args{}.Add(key)
+		for _, v := range x {
+			args = args.Add(fmt.Sprintf("%d", v))
+		}
+		if err = conn.Send("DEL", key); err != nil {
+			log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
+			PromError("redis:分区作者数据")
+			return
+		}
+		if err = conn.Send("SADD", args...); err != nil {
+			log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
+			PromError("redis:分区作者数据")
+			return
+		}
+		if err = conn.Send("EXPIRE", key, d.c.Job.RecommendExpire); err != nil {
+			log.Error("conn.Send(EXPIRE, %s) error(%v)", d.c.Job.RecommendExpire, err)
+			PromError("redis:分区作者数据")
+			return
+		}
+		if err = conn.Flush(); err != nil {
+			log.Error("conn.Flush error(%v)", err)
+			PromError("redis:分区作者数据")
+			return
+		}
+		for i := 0; i < 3; i++ {
+			if _, err = conn.Receive(); err != nil {
+				log.Error("conn.Receive error(%v)", err)
+				PromError("redis:分区作者数据")
+				return
+			}
+		}
+	}
+	return
+}
+
+// AddAuthorMostCategories .
+func (d *Dao) AddAuthorMostCategories(c context.Context, mid int64, categories []int64) (err error) {
+	if len(categories) == 0 {
+		return
+	}
+	conn := d.artRedis.Get(c)
+	defer conn.Close()
+	key := fmt.Sprintf("author:categories:%d", mid)
+	args := redis.Args{}.Add(key)
+	for _, v := range categories {
+		args = args.Add(fmt.Sprintf("%d", v))
+	}
+	if err = conn.Send("DEL", key); err != nil {
+		log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
+		PromError("redis:作者分区数据")
+		return
+	}
+	if err = conn.Send("SADD", args...); err != nil {
+		log.Error("conn.Send(SADD, %s, %+v) error(%v)", key, args, err)
+		PromError("redis:作者分区数据")
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.c.Job.RecommendExpire); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%v)", d.c.Job.RecommendExpire, err)
+		PromError("redis:作者分区数据")
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		PromError("redis:作者分区数据")
+		return
+	}
+	for i := 0; i < 3; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive error(%v)", err)
+			PromError("redis:作者分区数据")
+			return
+		}
+	}
+	return
+}

app/job/openplatform/article/dao/redis.go

@@ -0,0 +1,631 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	// view
+	_viewPrefix = "v_"
+
+	// retry stat
+	_retryStatKey = "retry_stat"
+	// RetryUpdateStatCache .
+	RetryUpdateStatCache = 1
+	// RetryUpdateStatDB .
+	RetryUpdateStatDB = 2
+	// RetryStatCount is retry upper limit.
+	RetryStatCount = 10
+
+	// retry cache
+	_retryArtCacheKey     = "retry_art_cache"
+	_retryGameCacheKey    = "artj_retry_game_cache"
+	_retryFlowCacheKey    = "artj_retry_flow_cache"
+	_retryDynamicCacheKey = "artj_retry_dynamic_cache"
+	// RetryAddArtCache .
+	RetryAddArtCache = 1
+	// RetryUpdateArtCache .
+	RetryUpdateArtCache = 2
+	// RetryDeleteArtCache .
+	RetryDeleteArtCache = 3
+	// RetryDeleteArtRecCache .
+	RetryDeleteArtRecCache = 4
+
+	// retry reply
+	_retryReplyKey = "retry_reply"
+	// retry purge cdn
+	_retryCDNKey   = "retry_cdn"
+	_recheckArtKey = "recheck_lock_%d"
+
+	// reading start set
+	_readPingSet = "art:readping"
+	// reading during on some device for some article
+	_prefixReadPing = "art:readping:%s:%d"
+)
+
+// StatRetry .
+type StatRetry struct {
+	Action int             `json:"action"`
+	Count  int             `json:"count"`
+	Data   *artmdl.StatMsg `json:"data"`
+}
+
+func viewKey(aid, mid int64, ip string) (key string) {
+	if ip == "" {
+		// let it pass if ip is empty.
+		return
+	}
+	key = _viewPrefix + strconv.FormatInt(aid, 10) + ip
+	if mid != 0 {
+		key += strconv.FormatInt(mid, 10)
+	}
+	return
+}
+
+func dupViewKey(aid, mid int64) (key string) {
+	return fmt.Sprintf("dv_%v_%v", aid, mid)
+}
+
+func recheckKey(aid int64) (key string) {
+	return fmt.Sprintf(_recheckArtKey, aid)
+}
+
+// pingRedis checks redis healthy.
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("SET", "PING", "PONG"); err != nil {
+		PromError("redis:Ping")
+		log.Error("redis: conn.Do(SET,PING,PONG) error(%+v)", err)
+	}
+	return
+}
+
+// Intercept intercepts illegal views.
+func (d *Dao) Intercept(c context.Context, aid, mid int64, ip string) (ban bool) {
+	var (
+		err   error
+		exist bool
+		key   = viewKey(aid, mid, ip)
+		conn  = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if key == "" {
+		return
+	}
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
+		PromError("redis:EXISTS阅读数")
+		return
+	}
+	if exist {
+		ban = true
+		return
+	}
+	if err = conn.Send("SET", key, "1"); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		PromError("redis:SET阅读数")
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.viewCacheTTL); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		PromError("redis:EXPIRE阅读数")
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%+v)", err)
+		PromError("redis:阅读数缓存Flush")
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%+v)", err)
+			PromError("redis:阅读数缓存Receive")
+			return
+		}
+	}
+	return
+}
+
+// DupViewIntercept intercepts illegal views.
+func (d *Dao) DupViewIntercept(c context.Context, aid, mid int64) (ban bool) {
+	if mid == 0 {
+		return
+	}
+	var (
+		err   error
+		exist bool
+		key   = dupViewKey(aid, mid)
+		conn  = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if exist, err = redis.Bool(conn.Do("EXISTS", key)); err != nil {
+		log.Error("conn.Do(EXISTS, %s) error(%+v)", key, err)
+		PromError("redis:EXISTS连续阅读数")
+		return
+	}
+	if exist {
+		ban = true
+		return
+	}
+	if err = conn.Send("SET", key, "1"); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		PromError("redis:SET连续阅读数")
+		return
+	}
+	if err = conn.Send("EXPIRE", key, d.dupViewCacheTTL); err != nil {
+		log.Error("conn.Send(EXPIRE, %s) error(%+v)", key, err)
+		PromError("redis:EXPIRE连续阅读数")
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%+v)", err)
+		PromError("redis:连续阅读数缓存Flush")
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%+v)", err)
+			PromError("redis:连续阅读数缓存Receive")
+			return
+		}
+	}
+	return
+}
+
+// PushStat pushs failed item to redis.
+func (d *Dao) PushStat(c context.Context, retry *StatRetry) (err error) {
+	var (
+		length int64
+		bs     []byte
+		conn   = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryStatKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryStatKey, err)
+		PromError("redis:计数重试LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_stat_length", length)
+	if bs, err = json.Marshal(retry); err != nil {
+		log.Error("json.Marshal(%v) error(%+v)", retry, err)
+		PromError("redis:计数重试消息Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryStatKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:计数重试RPUSH")
+	}
+	return
+}
+
+// PopStat pops failed item from redis.
+func (d *Dao) PopStat(c context.Context) (bs []byte, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryStatKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryStatKey, err)
+		PromError("redis:计数重试LPOP")
+	}
+	return
+}
+
+// PushReply opens article's reply.
+func (d *Dao) PushReply(c context.Context, aid, mid int64) (err error) {
+	var (
+		length int64
+		conn   = d.redis.Get(c)
+		bs     = []byte(strconv.FormatInt(aid, 10) + "_" + strconv.FormatInt(mid, 10))
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryReplyKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryReplyKey, err)
+		PromError("redis:打开评论重试LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_reply_length", length)
+	if _, err = conn.Do("RPUSH", _retryReplyKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:打开评论重试RPUSH")
+	}
+	return
+}
+
+// PopReply consume reply's job.
+func (d *Dao) PopReply(c context.Context) (aid, mid int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		bs   []byte
+		v    string
+		arr  []string
+	)
+	defer conn.Close()
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryReplyKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryReplyKey, err)
+		PromError("redis:打开评论重试LPOP")
+		return
+	}
+	if v = string(bs); v == "" {
+		return
+	}
+	if arr = strings.Split(v, "_"); len(arr) < 2 {
+		log.Error("reply retry param error (%s)", v)
+		PromError("redis:打开评论重试消息内容错误")
+		return
+	}
+	aid, _ = strconv.ParseInt(arr[0], 10, 64)
+	mid, _ = strconv.ParseInt(arr[1], 10, 64)
+	return
+}
+
+// PushCDN .
+func (d *Dao) PushCDN(c context.Context, file string) (err error) {
+	var (
+		length int64
+		conn   = d.redis.Get(c)
+		bs     = []byte(file)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryCDNKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryCDNKey, err)
+		PromError("redis:重试刷新CDN LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_cdn_length", length)
+	if _, err = conn.Do("RPUSH", _retryCDNKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:重试刷新CDN RPUSH")
+	}
+	return
+}
+
+// PopCDN .
+func (d *Dao) PopCDN(c context.Context) (file string, err error) {
+	var (
+		conn = d.redis.Get(c)
+		bs   []byte
+	)
+	defer conn.Close()
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryCDNKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryCDNKey, err)
+		PromError("redis:重试刷新CDN LPOP")
+		return
+	}
+	file = string(bs)
+	return
+}
+
+// CacheRetry struct of retry cache info.
+type CacheRetry struct {
+	Action int   `json:"action"`
+	Aid    int64 `json:"aid"`
+	Mid    int64 `json:"mid"`
+	Cid    int64 `json:"cid"`
+}
+
+// PushArtCache .
+func (d *Dao) PushArtCache(c context.Context, info *CacheRetry) (err error) {
+	var (
+		length int64
+		bs     []byte
+		conn   = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryArtCacheKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryArtCacheKey, err)
+		PromError("redis:重试文章缓存LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_art_cache_length", length)
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%v) error(%+v)", info, err)
+		PromError("redis:重试文章缓存Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryArtCacheKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:重试文章缓存RPUSH")
+	}
+	return
+}
+
+// PopArtCache .
+func (d *Dao) PopArtCache(c context.Context) (bs []byte, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryArtCacheKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryArtCacheKey, err)
+		PromError("redis:重试文章缓存LPOP")
+	}
+	return
+}
+
+// PushGameCache .
+func (d *Dao) PushGameCache(c context.Context, info *model.GameCacheRetry) (err error) {
+	var (
+		length int64
+		bs     []byte
+		conn   = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryGameCacheKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryGameCacheKey, err)
+		PromError("redis:重试游戏缓存LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_game_cache_length", length)
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%v) error(%+v)", info, err)
+		PromError("redis:重试游戏缓存Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryGameCacheKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:重试游戏缓存RPUSH")
+	}
+	return
+}
+
+// PopGameCache .
+func (d *Dao) PopGameCache(c context.Context) (res *model.GameCacheRetry, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	var bs []byte
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryGameCacheKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryGameCacheKey, err)
+		PromError("redis:重试游戏缓存LPOP")
+		return
+	}
+	res = new(model.GameCacheRetry)
+	if err = json.Unmarshal(bs, res); err != nil {
+		log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
+		PromError("redis:解析游戏缓存")
+	}
+	return
+}
+
+// PushFlowCache .
+func (d *Dao) PushFlowCache(c context.Context, info *model.FlowCacheRetry) (err error) {
+	var (
+		length int64
+		bs     []byte
+		conn   = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryFlowCacheKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryFlowCacheKey, err)
+		PromError("redis:重试flow缓存LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_flow_cache_length", length)
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%v) error(%+v)", info, err)
+		PromError("redis:重试flow缓存Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryFlowCacheKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:重试flow缓存RPUSH")
+	}
+	return
+}
+
+// PopFlowCache .
+func (d *Dao) PopFlowCache(c context.Context) (res *model.FlowCacheRetry, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	var bs []byte
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryFlowCacheKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryFlowCacheKey, err)
+		PromError("redis:重试flow缓存LPOP")
+		return
+	}
+	res = new(model.FlowCacheRetry)
+	if err = json.Unmarshal(bs, res); err != nil {
+		log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
+		PromError("redis:解析flow缓存")
+	}
+	return
+}
+
+// PushDynamicCache put dynamic to redis
+func (d *Dao) PushDynamicCache(c context.Context, info *model.DynamicCacheRetry) (err error) {
+	var (
+		length int64
+		bs     []byte
+		conn   = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if length, err = redis.Int64(conn.Do("LLEN", _retryDynamicCacheKey)); err != nil {
+		log.Error("conn.Do(%s) error(%+v)", _retryDynamicCacheKey, err)
+		PromError("redis:重试dynamic缓存LLEN")
+		return
+	}
+	cacheLen.State("redis:retry_dynamic_cache_length", length)
+	if bs, err = json.Marshal(info); err != nil {
+		log.Error("json.Marshal(%v) error(%+v)", info, err)
+		PromError("redis:重试dynamic缓存Marshal")
+		return
+	}
+	if _, err = conn.Do("RPUSH", _retryDynamicCacheKey, bs); err != nil {
+		log.Error("conn.Do(RPUSH, %s) error(%+v)", bs, err)
+		PromError("redis:重试dynamic缓存RPUSH")
+	}
+	return
+}
+
+// PopDynamicCache .
+func (d *Dao) PopDynamicCache(c context.Context) (res *model.DynamicCacheRetry, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	var bs []byte
+	if bs, err = redis.Bytes(conn.Do("LPOP", _retryDynamicCacheKey)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.Bytes(conn.Do(LPOP, %s)) error(%+v)", _retryDynamicCacheKey, err)
+		PromError("redis:重试dynamic缓存LPOP")
+		return
+	}
+	res = new(model.DynamicCacheRetry)
+	if err = json.Unmarshal(bs, res); err != nil {
+		log.Error("redis.Unmarshal(%s) error(%+v)", bs, err)
+		PromError("redis:解析dynamic缓存")
+	}
+	return
+}
+
+// GetRecheckCache get recheck info from redis
+func (d *Dao) GetRecheckCache(c context.Context, aid int64) (isRecheck bool, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	key := recheckKey(aid)
+	if isRecheck, err = redis.Bool(conn.Do("GET", key)); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+			return
+		}
+		log.Error("redis.BOOL(conn.Do(GET, %s)) error(%+v)", key, err)
+		PromError("redis:获取回查缓存")
+		return
+	}
+	return
+}
+
+// SetRecheckCache set recheck info to redis
+func (d *Dao) SetRecheckCache(c context.Context, aid int64) (isRecheck bool, err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	key := recheckKey(aid)
+	if _, err = conn.Do("SETEX", key, 604800, true); err != nil {
+		log.Error("redis.BOOL(conn.Do(SETEX, %s)) error(%+v)", key, err)
+		PromError("redis:设置回查缓存")
+		return
+	}
+	return
+}
+
+func readPingSetKey() string {
+	return _readPingSet
+}
+
+func readPingKey(buvid string, aid int64) string {
+	return fmt.Sprintf(_prefixReadPing, buvid, aid)
+}
+
+// ReadPingSet 获取所有阅读记录（不删除）
+func (d *Dao) ReadPingSet(c context.Context) (res []*model.Read, err error) {
+	var (
+		key    = readPingSetKey()
+		conn   = d.artRedis.Get(c)
+		tmpRes []string
+		tmpArr []string
+	)
+	defer conn.Close()
+
+	if err = conn.Send("SMEMBERS", key); err != nil {
+		log.Error("conn.Send(SMEMBERS, %s) error(%+v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%+v)", err)
+		return
+	}
+	if tmpRes, err = redis.Strings(conn.Receive()); err != nil {
+		log.Error("conn.Receive error(%+v)", err)
+		return
+	}
+	for _, tmp := range tmpRes {
+		tmpArr = strings.Split(tmp, "|")
+		if len(tmpArr) != 6 {
+			log.Error("redis key(%s)存在脏数据(%s)", key, tmp)
+			if _, err = conn.Do("SREM", key, tmp); err != nil {
+				log.Error("d.Redis.SREM error(%+v), set(%s), key(%s)", err, key, tmp)
+			}
+			continue
+		}
+		read := &model.Read{
+			Buvid:   tmpArr[0],
+			EndTime: 0,
+		}
+		read.Aid, _ = strconv.ParseInt(tmpArr[1], 10, 64)
+		read.Mid, _ = strconv.ParseInt(tmpArr[2], 10, 64)
+		read.IP = tmpArr[3]
+		read.StartTime, _ = strconv.ParseInt(tmpArr[4], 10, 64)
+		read.From = tmpArr[5]
+		res = append(res, read)
+	}
+	return
+}
+
+// ReadPing 获取上次阅读心跳时间，不存在则返回0
+func (d *Dao) ReadPing(c context.Context, buvid string, aid int64) (last int64, err error) {
+	var (
+		key  = readPingKey(buvid, aid)
+		conn = d.artRedis.Get(c)
+	)
+	defer conn.Close()
+	if last, err = redis.Int64(conn.Do("GET", key)); err != nil && err != redis.ErrNil {
+		log.Error("conn.Do(GET, %s) error(%+v)", key, err)
+		return
+	}
+	err = nil
+	return
+}
+
+// DelReadPingSet 删除阅读记录缓存
+func (d *Dao) DelReadPingSet(c context.Context, read *model.Read) (err error) {
+	if read == nil {
+		return
+	}
+	var (
+		elemKey = readPingKey(read.Buvid, read.Aid)
+		setKey  = readPingSetKey()
+		value   = fmt.Sprintf("%s|%d|%d|%s|%d|%s", read.Buvid, read.Aid, read.Mid, read.IP, read.StartTime, read.From)
+		conn    = d.artRedis.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", elemKey); err != nil {
+		log.Error("conn.Do(DEL, %s) error(%+v)", elemKey, err)
+		return
+	}
+	if _, err = conn.Do("SREM", setKey, value); err != nil {
+		log.Error("conn.Do(SREM, %s, %s) error(%+v)", setKey, value, err)
+		return
+	}
+	return
+}

app/job/openplatform/article/dao/redis_test.go

@@ -0,0 +1,69 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/library/cache/redis"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Redis(t *testing.T) {
+	Convey("test redis", t, WithDao(func(d *Dao) {
+		c := context.TODO()
+		conn := d.redis.Get(c)
+		defer conn.Close()
+
+		conn.Do("SET", "name", "echo")
+
+		if t, err := redis.Bool(conn.Do("EXISTS", "name")); err != nil {
+			fmt.Println(t)
+			fmt.Println(err)
+			_ = t
+		} else {
+			fmt.Println(t)
+			fmt.Println(err)
+			_ = t
+		}
+		fmt.Println("done")
+		err := d.PushStat(c, &StatRetry{})
+		So(err, ShouldBeNil)
+		res, err := d.PopStat(c)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+		So(d.Intercept(c, 1, 2, ""), ShouldBeFalse)
+		So(d.DupViewIntercept(c, 1, 2), ShouldBeTrue)
+		So(d.PushStat(c, nil), ShouldBeNil)
+		_, err = d.PopStat(c)
+		So(err, ShouldBeNil)
+		err = d.PushReply(c, 1, 2)
+		So(err, ShouldBeNil)
+		_, _, err = d.PopReply(c)
+		So(err, ShouldBeNil)
+		err = d.PushCDN(c, "")
+		So(err, ShouldBeNil)
+		_, err = d.PopCDN(c)
+		So(err, ShouldBeNil)
+		err = d.PushArtCache(c, nil)
+		So(err, ShouldBeNil)
+		_, err = d.PopArtCache(c)
+		So(err, ShouldBeNil)
+
+		err = d.PushGameCache(c, nil)
+		So(err, ShouldBeNil)
+		_, err = d.PopGameCache(c)
+		So(err, ShouldBeNil)
+
+		err = d.PushFlowCache(c, nil)
+		So(err, ShouldBeNil)
+		_, err = d.PopFlowCache(c)
+		So(err, ShouldBeNil)
+
+		err = d.PushDynamicCache(c, nil)
+		So(err, ShouldBeNil)
+		_, err = d.PopDynamicCache(c)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/openplatform/article/dao/reply.go

@@ -0,0 +1,82 @@
+package dao
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+	"time"
+
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_replyType       = "12"
+	_replyStateOpen  = "0" // 0: open, 1: close
+	_replyStateClose = "1"
+	_replyURL        = "http://api.bilibili.co/x/internal/v2/reply/subject/regist"
+)
+
+// OpenReply opens article's reply.
+func (d *Dao) OpenReply(c context.Context, aid, mid int64) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Second)
+		if e := d.PushReply(c, aid, mid); e != nil {
+			log.Error("d.PushReply(%d,%d) error(%+v)", aid, mid, e)
+		}
+	}()
+	params := url.Values{}
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	params.Set("oid", strconv.FormatInt(aid, 10))
+	params.Set("type", _replyType)
+	params.Set("state", _replyStateOpen)
+	var res struct {
+		Code int `json:"code"`
+	}
+	if err = d.httpClient.Post(c, _replyURL, "", params, &res); err != nil {
+		log.Error("d.httpClient.Post(%s) error(%+v)", _replyURL+"?"+params.Encode(), err)
+		PromError("reply:打开评论")
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		log.Error("d.httpClient.Post(%s) code(%d)", _replyURL+"?"+params.Encode(), res.Code)
+		PromError("reply:打开评论状态码异常")
+		err = ecode.Int(res.Code)
+	}
+	return
+}
+
+// CloseReply close article's reply.
+func (d *Dao) CloseReply(c context.Context, aid, mid int64) (err error) {
+	defer func() {
+		if err == nil {
+			return
+		}
+		time.Sleep(time.Second)
+		if e := d.PushReply(c, aid, mid); e != nil {
+			log.Error("d.PushReply(%d,%d) error(%+v)", aid, mid, e)
+		}
+	}()
+	params := url.Values{}
+	params.Set("mid", strconv.FormatInt(mid, 10))
+	params.Set("oid", strconv.FormatInt(aid, 10))
+	params.Set("type", _replyType)
+	params.Set("state", _replyStateClose)
+	var res struct {
+		Code int `json:"code"`
+	}
+	if err = d.httpClient.Post(c, _replyURL, "", params, &res); err != nil {
+		log.Error("d.httpClient.Post(%s) error(%+v)", _replyURL+"?"+params.Encode(), err)
+		PromError("reply:打开评论")
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		log.Error("d.httpClient.Post(%s) code(%d)", _replyURL+"?"+params.Encode(), res.Code)
+		PromError("reply:打开评论状态码异常")
+		err = ecode.Int(res.Code)
+	}
+	return
+}

app/job/openplatform/article/dao/reply_test.go

@@ -0,0 +1,15 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_OpenReply(t *testing.T) {
+	Convey("work", t, WithDao(func(d *Dao) {
+		err := d.OpenReply(context.Background(), 1, 2)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/openplatform/article/http/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/openplatform/article/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//app/job/openplatform/article/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/http/http.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/openplatform/article/conf"
+	"go-common/app/job/openplatform/article/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var ajSrv *service.Service
+
+// Init .
+func Init(conf *conf.Config, srv *service.Service) {
+	ajSrv = srv
+	// init outer router
+	engineOuter := bm.DefaultServer(conf.BM)
+	outerRouter(engineOuter)
+	if err := engineOuter.Start(); err != nil {
+		log.Error("xhttp.Serve error(%v)", err)
+		panic(err)
+	}
+}
+
+// outerRouter init outer router
+func outerRouter(r *bm.Engine) {
+	r.Ping(ping)
+	r.Register(register)
+	cr := r.Group("/sitemap")
+	{
+		cr.GET("/read/detail.xml", sitemap)
+	}
+}
+
+func ping(c *bm.Context) {
+	if err := ajSrv.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+func sitemap(c *bm.Context) {
+	res, l := ajSrv.SitemapXML(c)
+	c.Writer.Header().Set("Content-Type", "text/xml")
+	c.Writer.Header().Set("Content-Length", l)
+	c.Status(http.StatusOK)
+	c.Writer.Write([]byte(res))
+}

app/job/openplatform/article/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/openplatform/article/model",
+    tags = ["automanaged"],
+    deps = ["//app/interface/openplatform/article/model:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/model/model.go

@@ -0,0 +1,178 @@
+package model
+
+import (
+	"encoding/json"
+	"strconv"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+)
+
+const (
+	// ActUpdate ...
+	ActUpdate = "update"
+	// ActInsert ...
+	ActInsert = "insert"
+	// ActDelete ...
+	ActDelete = "delete"
+)
+
+// Message canal binlog message.
+type Message struct {
+	Action string          `json:"action"`
+	Table  string          `json:"table"`
+	New    json.RawMessage `json:"new"`
+	Old    json.RawMessage `json:"old"`
+}
+
+// Article db struction.
+type Article struct {
+	ID              int64  `json:"article_id"`
+	CTime           string `json:"ctime"`
+	CategoryID      int64  `json:"category_id"`
+	Title           string `json:"title"`
+	Summary         string `json:"summary"`
+	BannerURL       string `json:"banner_url"`
+	TemplateID      int    `json:"template_id"`
+	State           int    `json:"state"`
+	Mid             int64  `json:"mid"`
+	Reprint         int    `json:"reprint"`
+	ImageURLs       string `json:"image_urls"`
+	OriginImageURLs string `json:"origin_image_urls"`
+	PublishTime     int    `json:"publish_time"`
+	DeletedTime     int    `json:"deleted_time"`
+	Attributes      int32  `json:"attributes,omitempty"`
+	Reason          string `json:"reject_reason,omitempty"`
+	Words           int64  `json:"words"`
+	DynamicIntro    string `json:"dynamic_intro"`
+	MediaID         int64  `json:"media_id"`
+}
+
+// SearchArticle .
+type SearchArticle struct {
+	Article
+	Tags          string `json:"tags"`
+	Content       string `json:"content"`
+	StatsView     int64  `json:"stats_view"`
+	StatsFavorite int64  `json:"stats_favorite"`
+	StatsLikes    int64  `json:"stats_likes"`
+	StatsDisLike  int64  `json:"stats_dislike"`
+	StatsReply    int64  `json:"stats_reply"`
+	StatsShare    int64  `json:"stats_share"`
+	StatsCoin     int64  `json:"stats_coin"`
+	Keywords      string `json:"keywords"`
+}
+
+// Author db struction.
+type Author struct {
+	ID         int64 `json:"id"`
+	State      int   `json:"state"`
+	Mid        int64 `json:"mid"`
+	DailyLimit int   `json:"daily_limit"`
+}
+
+// Merge merges stat.
+func Merge(last, m *artmdl.StatMsg) (changed [][2]int64) {
+	if m.View != nil && *m.View >= 0 {
+		*last.View += *m.View
+		changed = append(changed, [2]int64{int64(artmdl.FieldView), *last.View})
+	}
+	if m.Like != nil {
+		*last.Like = *m.Like
+		changed = append(changed, [2]int64{int64(artmdl.FieldLike), *last.Like})
+	}
+	if m.Dislike != nil {
+		*last.Dislike = *m.Dislike
+	}
+	if m.Share != nil && *m.Share >= 0 {
+		*last.Share += *m.Share
+	}
+	if m.Favorite != nil && *m.Favorite >= 0 {
+		*last.Favorite = *m.Favorite
+		changed = append(changed, [2]int64{int64(artmdl.FieldFav), *last.Favorite})
+	}
+	if m.Reply != nil && *m.Reply >= 0 {
+		*last.Reply = *m.Reply
+		changed = append(changed, [2]int64{int64(artmdl.FieldReply), *last.Reply})
+	}
+	if m.Coin != nil && *m.Coin >= 0 {
+		*last.Coin = *m.Coin
+	}
+	return
+}
+
+// ReadURLs returns article's read urls.
+func ReadURLs(aid int64) []string {
+	aidStr := strconv.FormatInt(aid, 10)
+	return []string{
+		"http://www.bilibili.com/read/cv/" + aidStr,
+		"https://www.bilibili.com/read/cv/" + aidStr,
+		"http://www.bilibili.com/read/app/" + aidStr,
+		"https://www.bilibili.com/read/app/" + aidStr,
+	}
+}
+
+// GameCacheRetry .
+type GameCacheRetry struct {
+	Action string `json:"action"`
+	Aid    int64  `json:"aid"`
+}
+
+// FlowCacheRetry .
+type FlowCacheRetry struct {
+	Aid int64 `json:"aid"`
+	Mid int64 `json:"mid"`
+}
+
+// DynamicCacheRetry .
+type DynamicCacheRetry struct {
+	Aid          int64
+	Mid          int64
+	Show         bool
+	Comment      string
+	Ts           int64
+	DynamicIntro string
+}
+
+// LikeMsg msg
+type LikeMsg struct {
+	BusinessID    int64 `json:"business_id"`
+	MessageID     int64 `json:"message_id"`
+	LikesCount    int64 `json:"likes_count"`
+	DislikesCount int64 `json:"dislikes_count"`
+}
+
+// DynamicMsg msg
+type DynamicMsg struct {
+	Card struct {
+		Comment string `json:"comment"`
+		Dynamic string `json:"dynamic"`
+		OwnerID int64  `json:"owner_id"`
+		Rid     int64  `json:"rid"`
+		Show    int64  `json:"show"`
+		Stype   int64  `json:"stype"`
+		Ts      int64  `json:"ts"`
+		Type    int64  `json:"type"`
+	} `json:"card"`
+}
+
+// Setting the setting struct
+type Setting struct {
+	Recheck *Recheck
+}
+
+// Recheck setting struct
+type Recheck struct {
+	Day  int64 `json:"day"`
+	View int64 `json:"view"`
+}
+
+// Read presents user reading duration struct
+type Read struct {
+	Buvid     string
+	Aid       int64
+	Mid       int64
+	IP        string
+	From      string
+	StartTime int64
+	EndTime   int64
+}

app/job/openplatform/article/service/BUILD

@@ -0,0 +1,81 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "retry_test.go",
+        "search_test.go",
+        "service_test.go",
+        "sort_test.go",
+        "stat_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/openplatform/article/model:go_default_library",
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//app/job/openplatform/article/dao:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "article.go",
+        "author.go",
+        "cron.go",
+        "infoc.go",
+        "read.go",
+        "recommend.go",
+        "search.go",
+        "service.go",
+        "setting.go",
+        "sitemap.go",
+        "sort.go",
+        "stat.go",
+        "tag.go",
+    ],
+    importpath = "go-common/app/job/openplatform/article/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/tag/model:go_default_library",
+        "//app/interface/main/tag/rpc/client:go_default_library",
+        "//app/interface/openplatform/article/model:go_default_library",
+        "//app/interface/openplatform/article/rpc/client:go_default_library",
+        "//app/job/openplatform/article/conf:go_default_library",
+        "//app/job/openplatform/article/dao:go_default_library",
+        "//app/job/openplatform/article/model:go_default_library",
+        "//app/service/main/thumbup/model:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//library/sync/errgroup:go_default_library",
+        "//vendor/github.com/jaytaylor/html2text:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/openplatform/article/service/article.go

@@ -0,0 +1,224 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+)
+
+func (s *Service) upArticles(c context.Context, action string, newMsg []byte, oldMsg []byte) {
+	log.Info("s.upArticles action(%s) old(%s) new(%s)", action, string(oldMsg), string(newMsg))
+	if action != model.ActUpdate && action != model.ActInsert && action != model.ActDelete {
+		return
+	}
+	var (
+		err        error
+		aid        int64
+		newArticle = &model.Article{}
+		oldArticle = &model.Article{}
+	)
+	if err = json.Unmarshal(newMsg, newArticle); err != nil {
+		log.Error("json.Unmarshal(%s) error(%+v)", newMsg, err)
+		dao.PromError("article:解析过审文章databus新内容")
+		return
+	}
+	if action == model.ActUpdate {
+		if err = json.Unmarshal(oldMsg, oldArticle); err != nil {
+			log.Error("json.Unmarshal(%s) error(%+v)", oldMsg, err)
+			dao.PromError("article:解析过审文章databus旧内容")
+			return
+		}
+	}
+	aid = newArticle.ID
+	mid := newArticle.Mid
+	show := true
+	var comment string
+	if artmdl.NoDistributeAttr(newArticle.Attributes) {
+		show = false
+		comment = "禁止分发"
+	}
+	switch action {
+	case model.ActInsert:
+		s.openReply(c, aid, mid)
+		s.addArtCache(c, aid)
+		s.updateSearchArt(c, newArticle)
+		s.flowSync(c, aid, mid)
+		s.addURLNode(c, aid)
+		if comment == "" {
+			comment = "文章过审"
+		}
+	case model.ActUpdate:
+		s.updateArtCache(c, aid, oldArticle.CategoryID)
+		s.addURLNode(c, aid)
+		if !artmdl.NoDistributeAttr(oldArticle.Attributes) && artmdl.NoDistributeAttr(newArticle.Attributes) {
+			s.delSearchArt(c, aid)
+		} else {
+			s.updateSearchArt(c, newArticle)
+		}
+		if comment == "" {
+			comment = "文章修改"
+		}
+	case model.ActDelete:
+		s.closeReply(c, aid, mid)
+		s.deleteArtCache(c, aid, mid)
+		s.deleteArtRecommendCache(c, aid, newArticle.CategoryID)
+		s.delSearchArt(c, aid)
+		// s.delMediaScore(c, aid, newArticle.MediaID, mid)
+		show = false
+		comment = "文章不可见"
+	}
+	if e := s.dao.PubDynamic(c, mid, aid, show, comment, int64(newArticle.PublishTime), newArticle.DynamicIntro); e != nil {
+		s.dao.PushDynamicCache(c, &model.DynamicCacheRetry{
+			Aid:          aid,
+			Mid:          mid,
+			Show:         show,
+			Comment:      comment,
+			Ts:           int64(newArticle.PublishTime),
+			DynamicIntro: newArticle.DynamicIntro,
+		})
+	}
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	s.gameSync(c, aid, mid, action)
+	urls := model.ReadURLs(aid)
+	for _, u := range urls {
+		if err = s.dao.PurgeCDN(c, u); err == nil {
+			log.Info("s.dao.PurgeCDN(%s) success.", u)
+			dao.PromInfo("article:刷新CDN")
+		}
+	}
+}
+
+func (s *Service) gameSync(c context.Context, aid, mid int64, action string) {
+	authors, err := s.dao.GameList(c)
+	if err != nil {
+		dao.PromError("service:获得游戏数据")
+		log.Error("s.gameSync(aid: %v, mid: %v) init err: %+v", aid, mid, err)
+		return
+	}
+	var exist bool
+	for _, author := range authors {
+		if author == mid {
+			exist = true
+			break
+		}
+	}
+	if !exist {
+		return
+	}
+	if err = s.dao.GameSync(c, action, aid); err != nil {
+		log.Error("s.gameSync(%d, %d, %s) error(%+v)", aid, mid, action, err)
+		dao.PromError("service:同步游戏数据")
+		s.dao.PushGameCache(c, &model.GameCacheRetry{
+			Action: action,
+			Aid:    aid,
+		})
+		return
+	}
+	log.Info("s.gameSync(%d, %d, %s) success", aid, mid, action)
+}
+
+func (s *Service) flowSync(c context.Context, aid, mid int64) {
+	if err := s.dao.FlowSync(c, mid, aid); err != nil {
+		s.dao.PushFlowCache(c, &model.FlowCacheRetry{
+			Aid: aid,
+			Mid: mid,
+		})
+		return
+	}
+	log.Info("s.flowSync(aid: %d, mid: %d) success", aid, mid)
+}
+
+func (s *Service) openReply(c context.Context, aid, mid int64) (err error) {
+	if err = s.dao.OpenReply(c, aid, mid); err == nil {
+		log.Info("OpenReply(%d,%d) success.", aid, mid)
+		dao.PromInfo("article:打开评论区")
+	}
+	return
+}
+
+func (s *Service) closeReply(c context.Context, aid, mid int64) (err error) {
+	if err = s.dao.CloseReply(c, aid, mid); err == nil {
+		log.Info("CloseReply(%d,%d) success.", aid, mid)
+		dao.PromInfo("article:关闭评论区")
+	}
+	return
+}
+
+func (s *Service) addArtCache(c context.Context, aid int64) (err error) {
+	arg := &artmdl.ArgAid{Aid: aid}
+	if err = s.articleRPC.AddArticleCache(c, arg); err != nil {
+		log.Error("s.articleRPC.AddArticleCache(%d) error(%+v)", aid, err)
+		dao.PromError("article:新增文章缓存")
+		s.dao.PushArtCache(c, &dao.CacheRetry{
+			Action: dao.RetryAddArtCache,
+			Aid:    aid,
+		})
+	} else {
+		log.Info("s.articleRPC.AddArticleCache(%d) success", aid)
+		dao.PromInfo("article:新增文章缓存")
+	}
+	return
+}
+
+func (s *Service) updateArtCache(c context.Context, aid, cid int64) (err error) {
+	arg := &artmdl.ArgAidCid{Aid: aid, Cid: cid}
+	if err = s.articleRPC.UpdateArticleCache(c, arg); err != nil {
+		log.Error("s.articleRPC.UpdateArticleCache(%d,%d) error(%+v)", aid, cid, err)
+		dao.PromError("article:更新文章缓存")
+		s.dao.PushArtCache(c, &dao.CacheRetry{
+			Action: dao.RetryUpdateArtCache,
+			Aid:    aid,
+			Cid:    cid,
+		})
+	} else {
+		log.Info("s.articleRPC.UpdateArticleCache(%d,%d) success", aid, cid)
+		dao.PromInfo("article:更新文章缓存")
+	}
+	return
+}
+
+func (s *Service) deleteArtCache(c context.Context, aid, mid int64) (err error) {
+	arg := &artmdl.ArgAidMid{Aid: aid, Mid: mid}
+	if err = s.articleRPC.DelArticleCache(c, arg); err != nil {
+		log.Error("s.articleRPC.DelArticleCache(%d,%d) error(%+v)", aid, mid, err)
+		dao.PromError("article:删除文章缓存")
+		s.dao.PushArtCache(c, &dao.CacheRetry{
+			Action: dao.RetryDeleteArtCache,
+			Aid:    aid,
+			Mid:    mid,
+		})
+	} else {
+		log.Info("s.articleRPC.DelArticleCache(%d,%d) success", aid, mid)
+		dao.PromInfo("article:删除文章缓存")
+	}
+	return
+}
+
+func (s *Service) deleteArtRecommendCache(c context.Context, aid, cid int64) (err error) {
+	arg := &artmdl.ArgAidCid{Aid: aid, Cid: cid}
+	if err = s.articleRPC.DelRecommendArtCache(c, arg); err != nil {
+		log.Error("s.articleRPC.DelRecommendArtCache(%d,%d) error(%+v)", aid, cid, err)
+		dao.PromError("article:删除文章推荐缓存")
+		s.dao.PushArtCache(c, &dao.CacheRetry{
+			Action: dao.RetryDeleteArtRecCache,
+			Aid:    aid,
+			Cid:    cid,
+		})
+	} else {
+		log.Info("s.articleRPC.DelRecommendArtCache(%d,%d) success", aid, cid)
+		dao.PromInfo("article:删除文章推荐缓存")
+	}
+	return
+}
+
+func (s *Service) delMediaScore(c context.Context, aid, mediaID, mid int64) (err error) {
+	err = s.dao.DelScore(c, aid, mediaID, mid)
+	return
+}

app/job/openplatform/article/service/author.go

@@ -0,0 +1,38 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+)
+
+func (s *Service) upAuthors(c context.Context, action string, newMsg []byte, oldMsg []byte) {
+	log.Info("s.upAuthors action(%s) old(%s) new(%s)", action, string(oldMsg), string(newMsg))
+	var (
+		err       error
+		newAuthor = &model.Author{}
+	)
+	if err = json.Unmarshal(newMsg, newAuthor); err != nil {
+		log.Error("json.Unmarshal(%s) error(%+v)", newMsg, err)
+		dao.PromError("article:解析作者表databus新内容")
+		return
+	}
+	s.updateAuthorCache(c, newAuthor.Mid)
+}
+
+// updateAuthorCache update author cache
+func (s *Service) updateAuthorCache(c context.Context, mid int64) (err error) {
+	arg := &artmdl.ArgAuthor{Mid: mid}
+	if err = s.articleRPC.UpdateAuthorCache(c, arg); err != nil {
+		log.Error("s.articleRPC.UpdateAuthorCache(%+v) error(%+v)", arg, err)
+		dao.PromError("article:更新作者缓存")
+		return
+	}
+	log.Info("s.articleRPC.UpdateAuthorCache(%+v) success", arg)
+	dao.PromInfo("article:更新作者缓存")
+	return
+}

app/job/openplatform/article/service/cron.go

@@ -0,0 +1,21 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/library/log"
+)
+
+func (s *Service) updateSortproc() {
+	for {
+		if err := s.UpdateSort(context.TODO()); err != nil {
+			log.Error("s.UpdateSort error(%+v)", err)
+			dao.PromError("service:刷新分区投稿")
+		} else {
+			dao.PromInfo("service:刷新分区投稿")
+		}
+		time.Sleep(s.updateSortInterval)
+	}
+}

app/job/openplatform/article/service/infoc.go

@@ -0,0 +1,57 @@
+package service
+
+import (
+	"go-common/library/log"
+	binfoc "go-common/library/log/infoc"
+	"go-common/library/stat/prom"
+)
+
+// 用户阅读专栏时长上报
+type readInfo struct {
+	aid      int64
+	mid      int64
+	buvid    string
+	ip       string
+	duration int64
+	from     string
+}
+
+// ReadInfoc .
+func (s *Service) ReadInfoc(aid int64, mid int64, buvid string, ip string, duration int64, from string) {
+	s.infoc(readInfo{
+		aid:      aid,
+		mid:      mid,
+		buvid:    buvid,
+		ip:       ip,
+		duration: duration,
+		from:     from,
+	})
+}
+
+func (s *Service) infoc(i interface{}) {
+	select {
+	case s.logCh <- i:
+	default:
+		log.Warn("infocproc chan full")
+	}
+}
+
+// writeInfoc
+func (s *Service) infocproc() {
+	var (
+		readInfoc = binfoc.New(s.c.ReadInfoc)
+	)
+	for {
+		i, ok := <-s.logCh
+		if !ok {
+			log.Warn("infoc proc exit")
+			return
+		}
+		prom.BusinessInfoCount.State("infoc_channel", int64(len(s.logCh)))
+		switch l := i.(type) {
+		case readInfo:
+			readInfoc.Info(l.aid, l.mid, l.buvid, l.ip, l.duration, l.from)
+			log.Info("infocproc readInfoc param(%+v)", l)
+		}
+	}
+}

app/job/openplatform/article/service/read.go

@@ -0,0 +1,49 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/library/log"
+)
+
+func (s *Service) checkReadStatus() {
+	for {
+		var c = context.TODO()
+		readSet, err := s.dao.ReadPingSet(c)
+		if err != nil || len(readSet) == 0 {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		now := time.Now().Unix()
+		for _, read := range readSet {
+			last, err := s.dao.ReadPing(c, read.Buvid, read.Aid)
+			if err != nil {
+				time.Sleep(time.Second)
+				continue
+			}
+			if now-last < 30 {
+				continue
+			}
+			if err = s.dao.DelReadPingSet(c, read); err != nil {
+				time.Sleep(time.Second)
+				continue
+			}
+			if last == 0 {
+				log.Error("阅读心跳没取到:buvid(%s) aid(%d)", read.Buvid, read.Aid)
+				continue
+			}
+			read.EndTime = last
+			log.Info("上传用户阅读记录至数据中心: %+v", read)
+			s.ReadInfoc(read.Aid, read.Mid, read.Buvid, read.IP, read.EndTime-read.StartTime, read.From)
+		}
+		return
+	}
+}
+
+func (s *Service) checkReadStatusProc() {
+	for {
+		s.checkReadStatus()
+		time.Sleep(time.Minute)
+	}
+}

app/job/openplatform/article/service/recommend.go

@@ -0,0 +1,77 @@
+package service
+
+import (
+	"context"
+	"time"
+)
+
+func (s *Service) recommendAuthorproc() {
+	for {
+		s.calRecommendAuthor()
+		time.Sleep(time.Hour)
+	}
+}
+
+func (s *Service) calRecommendAuthor() {
+	var (
+		mids []int64
+		err  error
+		m    = make(map[int64][]int64)
+	)
+	for {
+		if mids, err = s.dao.MidsByPublishTime(context.TODO(), time.Now().Unix()-86400*s.c.Job.StatDays); err != nil {
+			time.Sleep(time.Minute)
+			continue
+		}
+		for _, mid := range mids {
+			if categoris, ok := s.statAuthorInfo(mid); ok {
+				for _, category := range categoris {
+					m[category] = append(m[category], mid)
+				}
+				s.dao.AddAuthorMostCategories(context.TODO(), mid, categoris)
+			}
+		}
+		s.dao.AddcategoriesAuthors(context.TODO(), m)
+		return
+	}
+}
+
+func (s *Service) statAuthorInfo(mid int64) (categories []int64, res bool) {
+	var (
+		m     map[int64][2]int64
+		err   error
+		words int64
+		cm    = make(map[int64]int64)
+		max   int64
+	)
+
+	if m, err = s.dao.StatByMid(context.TODO(), mid); err != nil {
+		return
+	}
+	for cid, data := range m {
+		if cate, ok := s.categoriesMap[cid]; ok {
+			if cate.ParentID != 0 {
+				cid = cate.ParentID
+			}
+			words += data[1]
+			if _, ok := cm[cid]; ok {
+				cm[cid] += data[0]
+			} else {
+				cm[cid] = data[0]
+			}
+		}
+	}
+	if words < s.c.Job.Words {
+		return
+	}
+	res = true
+	for k, v := range cm {
+		if v > max {
+			max = v
+			categories = []int64{k}
+		} else if v == max {
+			categories = append(categories, k)
+		}
+	}
+	return
+}

app/job/openplatform/article/service/retry_test.go

@@ -0,0 +1,64 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/library/cache/redis"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Retry(t *testing.T) {
+	Convey("retry", t, WithoutProcService(func(s *Service) {
+
+		var (
+			err error
+			c   = context.TODO()
+		)
+
+		SkipConvey("push retry", func() {
+			var (
+				favCnt    = int64(1)
+				replyCnt  = int64(2)
+				statRetry = &dao.StatRetry{
+					Data: &artmdl.StatMsg{
+						Aid:      888,
+						Favorite: &favCnt,
+						Reply:    &replyCnt,
+					},
+					Action: dao.RetryUpdateStatCache,
+					Count:  3,
+				}
+			)
+			err = s.dao.PushStat(c, statRetry)
+			So(err, ShouldBeNil)
+		})
+
+		SkipConvey("pop retry", func() {
+			bs, err := s.dao.PopStat(c)
+			if err != redis.ErrNil {
+				So(err, ShouldBeNil)
+			}
+			So(bs, ShouldNotBeEmpty)
+			msg := &dao.StatRetry{}
+			err = json.Unmarshal(bs, msg)
+			So(err, ShouldBeNil)
+			So(msg.Action, ShouldEqual, dao.RetryUpdateStatCache)
+			So(msg.Count, ShouldEqual, 3)
+		})
+
+		Convey("retry reply", func() {
+			err := s.dao.PushReply(context.TODO(), 8, 9)
+			So(err, ShouldBeNil)
+			aid, mid, err := s.dao.PopReply(c)
+			So(err, ShouldBeNil)
+			fmt.Println(aid)
+			fmt.Println(mid)
+		})
+	}))
+}

app/job/openplatform/article/service/search.go

@@ -0,0 +1,92 @@
+package service
+
+import (
+	"context"
+	"strings"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+
+	"go-common/library/sync/errgroup"
+
+	"github.com/jaytaylor/html2text"
+)
+
+func (s *Service) updateSearchArt(c context.Context, art *model.Article) (err error) {
+	if art == nil {
+		return
+	}
+	a := artmdl.Meta{Attributes: art.Attributes}
+	if a.AttrVal(artmdl.AttrBitNoDistribute) {
+		return
+	}
+	searchArt := &model.SearchArticle{Article: *art}
+	var (
+		group  *errgroup.Group
+		errCtx context.Context
+	)
+
+	group, errCtx = errgroup.WithContext(c)
+	group.Go(func() (err error) {
+		if searchArt.Content, err = s.dao.ArticleContent(errCtx, art.ID); err != nil {
+			return
+		}
+		searchArt.Content, err = extractText(searchArt.Content)
+		return
+	})
+	group.Go(func() (err error) {
+		var stat *artmdl.StatMsg
+		if stat, err = s.dao.Stat(errCtx, art.ID); (err != nil) || (stat == nil) {
+			return
+		}
+		searchArt.StatsDisLike = *stat.Dislike
+		searchArt.StatsFavorite = *stat.Favorite
+		searchArt.StatsLikes = *stat.Like
+		searchArt.StatsReply = *stat.Reply
+		searchArt.StatsShare = *stat.Share
+		searchArt.StatsView = *stat.View
+		searchArt.StatsCoin = *stat.Coin
+		return
+	})
+	group.Go(func() (err error) {
+		searchArt.Tags, err = s.tags(errCtx, art.ID)
+		return
+	})
+	group.Go(func() (err error) {
+		searchArt.Keywords, err = s.dao.Keywords(errCtx, art.ID)
+		return
+	})
+	if err = group.Wait(); err == nil {
+		err = s.dao.UpdateSearch(c, searchArt)
+	}
+	if err != nil {
+		dao.PromError("search:更新文章")
+		log.Error("updateSearchArt(%+v) err: %+v", searchArt, err)
+		return
+	}
+	log.Info("success updateSearchArt(id: %v, title: %v)", searchArt.ID, searchArt.Title)
+	return
+}
+
+func (s *Service) delSearchArt(c context.Context, aid int64) (err error) {
+	err = s.dao.DelSearch(c, aid)
+	return
+}
+
+func extractText(html string) (res string, err error) {
+	ops := html2text.Options{PrettyTables: false}
+	res, err = html2text.FromString(html, ops)
+	res = strings.Replace(res, "*", "", -1)
+	return
+}
+
+func (s *Service) updateSearchStats(c context.Context, stat *artmdl.StatMsg) (err error) {
+	if err = s.dao.UpdateSearchStats(c, stat); err != nil {
+		log.Error("updateSearchStats(%v) err: %+v", stat.String(), err)
+		return
+	}
+	log.Info("updateSearchStats(%v) success", stat.String())
+	return
+}

app/job/openplatform/article/service/search_test.go

@@ -0,0 +1,19 @@
+package service
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Search(t *testing.T) {
+	Convey("work", t, func() {
+		a := `
+<div><em><br>AlphaGO战胜柯洁、Google翻译准确度大幅提升、自动驾驶技术日趋成熟；这些无一不在告诉我们人工智能技术正飞速发展。那么如果让人工智能来玩STG会怎么样呢？<br></em><br></div><div><a href="http://www.bilibili.com/video/av12432">视频链接</a><br>STG也称为弹幕游戏。往往由于满屏幕都是子弹，而使一部分玩家望而却步。长久以来，初学者觉得游戏难度无法想象，简直反人类，而高级玩家却觉得还不够难，不够过瘾。这对矛盾已经成为了影响STG发展的主要矛盾。<br>av12679</div><div>av23412<br>现今，大多数STG都会为了解决或缓解这一矛盾而采取了一些措施，比如大多数游戏会设置多个难度，使初学者玩简单难度，而高手可以玩疯狂难度；有的游戏会设置教学关卡，指导初学者避弹；有一些游戏设置了自动雷击功能来为初学者降低难度。<br><br></div><div><strong><br>然而就在最近，我们在游戏《弹幕音乐绘》中，提出了一种更加别出心裁的解决方案——自动避弹。<br></strong><em><br>AlphaGO战胜柯洁、Google翻译准确度大幅提升、自动驾驶技术日趋成熟；这些无一不在告诉我们人工智能技术正飞速发展。那么如果让人工智能来玩STG会怎么样呢？<br></em><br></div><div><br>STG也称为弹幕游戏。往往由于满屏幕都是子弹，而使一部分玩家望而却步。长久以来，初学者觉得游戏难度无法想象，简直反人类，而高级玩家却觉得还不够难，不够过瘾。这对矛盾已经成为了影响STG发展的主要矛盾。<br><br></div><div><br>现今，大多数STG都会为了解决或缓解这一矛盾而采取了一些措施，比如大多数游戏会设置多个难度，使初学者玩简单难度，而高手可以玩疯狂难度；有的游戏会设置教学关卡，指导初学者避弹；有一些游戏设置了自动雷击功能来为初学者降低难度。<br><br></div><div><strong><br>然而就在最近，我们在游戏《弹幕音乐绘》中，提出了一种更加别出心裁的解决方案——自动避弹。<br></strong><br></div>
+`
+		res, err := extractText(a)
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+		// fmt.Println(res)
+	})
+}

app/job/openplatform/article/service/service.go

@@ -0,0 +1,616 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	tagrpc "go-common/app/interface/main/tag/rpc/client"
+	artmdl "go-common/app/interface/openplatform/article/model"
+	artrpc "go-common/app/interface/openplatform/article/rpc/client"
+	"go-common/app/job/openplatform/article/conf"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	thumdl "go-common/app/service/main/thumbup/model"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+)
+
+const (
+	_sharding     = 100 // goroutines for dealing the stat
+	_chanSize     = 10240
+	_articleTable = "filtered_articles" // article table name
+	_authorTable  = "article_authors"   // 作者表
+)
+
+type lastTimeStat struct {
+	time int64
+	stat *artmdl.StatMsg
+}
+
+// Service .
+type Service struct {
+	c      *conf.Config
+	dao    *dao.Dao
+	waiter *sync.WaitGroup
+	closed bool
+	// monitor              *monitor.Service
+	articleSub           *databus.Databus
+	articleStatSub       *databus.Databus
+	likeStatSub          *databus.Databus
+	replyStatSub         *databus.Databus
+	favoriteStatSub      *databus.Databus
+	coinStatSub          *databus.Databus
+	articleRPC           *artrpc.Service
+	tagRPC               *tagrpc.Service
+	statCh               [_sharding]chan *artmdl.StatMsg
+	statLastTime         [_sharding]map[int64]*lastTimeStat
+	categoriesMap        map[int64]*artmdl.Category
+	categoriesReverseMap map[int64][]*artmdl.Category
+	sitemapMap           map[int64]struct{}
+	urlListHead          *urlNode
+	urlListTail          *urlNode
+	lastURLNode          *urlNode
+	sitemapXML           string
+	likeCh               chan *thumdl.StatMsg
+	updateDbInterval     int64
+	updateSortInterval   time.Duration
+	cheatInfoc           *infoc.Infoc
+	// *Cnt means sum of consumed messages.
+	statCnt, binCnt int64
+	cheatArts       map[int64]int
+	sortLimitTime   int64
+	setting         *model.Setting
+	// infoc
+	logCh chan interface{}
+}
+
+// New creates a Service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:      c,
+		dao:    dao.New(c),
+		waiter: new(sync.WaitGroup),
+		// monitor:              monitor.New(),
+		articleSub:           databus.New(c.ArticleSub),
+		articleStatSub:       databus.New(c.ArticleStatSub),
+		replyStatSub:         databus.New(c.ReplyStatSub),
+		favoriteStatSub:      databus.New(c.FavoriteStatSub),
+		coinStatSub:          databus.New(c.CoinStatSub),
+		likeStatSub:          databus.New(c.LikeStatSub),
+		articleRPC:           artrpc.New(c.ArticleRPC),
+		tagRPC:               tagrpc.New2(c.TagRPC),
+		updateDbInterval:     int64(time.Duration(c.Job.UpdateDbInterval) / time.Second),
+		updateSortInterval:   time.Duration(c.Job.UpdateSortInterval),
+		likeCh:               make(chan *thumdl.StatMsg, 1e4),
+		cheatInfoc:           infoc.New(c.CheatInfoc),
+		categoriesMap:        make(map[int64]*artmdl.Category),
+		categoriesReverseMap: make(map[int64][]*artmdl.Category),
+		sitemapMap:           make(map[int64]struct{}),
+		sortLimitTime:        int64(time.Duration(c.Job.SortLimitTime) / time.Second),
+		logCh:                make(chan interface{}, 1024),
+	}
+	// s.monitor.SetConfig(c.HTTPClient)
+	for i := int64(0); i < _sharding; i++ {
+		i := i
+		// for stat
+		s.statCh[i] = make(chan *artmdl.StatMsg, _chanSize)
+		s.statLastTime[i] = make(map[int64]*lastTimeStat)
+		s.waiter.Add(1)
+		go s.statproc(i)
+	}
+	s.loadCategories()
+	s.loadSettings()
+	// 10 go routines with WaitGroup
+	s.waiter.Add(10)
+	go s.consumeStat()
+	go s.consumeCanal()
+	go s.checkConsumer()
+	go s.retryStat()
+	go s.retryReply()
+	go s.retryGame()
+	go s.retryFlow()
+	go s.retryDynamic()
+	go s.retryCDN()
+	go s.retryCache()
+	// other go routines
+	go s.updateSortproc()
+	go s.activityLikeproc()
+	go s.updateReadCountproc()
+	go s.updateHotspotsproc()
+	go s.updateCheatArtsproc()
+	go s.loadCategoriesproc()
+	go s.loadSettingsproc()
+	go s.recommendAuthorproc()
+	go s.checkReadStatusProc()
+	go s.infocproc()
+	go s.sitemapproc()
+	return
+}
+
+// consumeStat consumes article's stat.
+func (s *Service) consumeStat() {
+	defer s.waiter.Done()
+	for {
+		if s.closed {
+			for i := 0; i < _sharding; i++ {
+				close(s.statCh[i])
+			}
+			log.Info("databus: article-job stat consumer exit!")
+			return
+		}
+	L:
+		select {
+		case msg, ok := <-s.articleStatSub.Messages():
+			if !ok {
+				break L
+			}
+			s.statCnt++
+			msg.Commit()
+			sm := &artmdl.StatMsg{}
+			if err := json.Unmarshal(msg.Value, sm); err != nil {
+				log.Error("json.Unmarshal(%s) error(%+v)", msg.Value, err)
+				dao.PromError("service:解析计数databus消息")
+				break L
+			}
+			if sm.Aid <= 0 {
+				log.Warn("aid(%d) <=0 message(%s)", sm.Aid, msg.Value)
+				break L
+			}
+			key := sm.Aid % _sharding
+			s.statCh[key] <- sm
+			prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+			log.Info("consumeStat key:%s partition:%d offset:%d msg: %v)", msg.Key, msg.Partition, msg.Offset, sm.String())
+		case msg, ok := <-s.likeStatSub.Messages():
+			if !ok {
+				break L
+			}
+			msg.Commit()
+			like := &thumdl.StatMsg{}
+			if err := json.Unmarshal(msg.Value, like); err != nil {
+				log.Error("json.Unmarshal(%s) error(%+v)", msg.Value, err)
+				dao.PromError("service:解析like计数databus消息")
+				break L
+			}
+			if like.Type != "article" {
+				continue
+			}
+			select {
+			case s.likeCh <- like:
+			default:
+			}
+			key := like.ID % _sharding
+			s.statCh[key] <- &artmdl.StatMsg{Like: &like.Count, Dislike: &like.DislikeCount, Aid: like.ID}
+			prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+			log.Info("consumeLikeStat key:%s partition:%d offset:%d msg: %+v)", msg.Key, msg.Partition, msg.Offset, like)
+		case msg, ok := <-s.replyStatSub.Messages():
+			if !ok {
+				break L
+			}
+			msg.Commit()
+			m := &thumdl.StatMsg{}
+			if err := json.Unmarshal(msg.Value, m); err != nil {
+				log.Error("reply json.Unmarshal(%s) error(%+v)", msg.Value, err)
+				dao.PromError("service:解析reply计数databus消息")
+				break L
+			}
+			if m.Type != "article" {
+				continue
+			}
+			key := m.ID % _sharding
+			s.statCh[key] <- &artmdl.StatMsg{Reply: &m.Count, Aid: m.ID}
+			prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+			log.Info("consumeReplyStat key:%s partition:%d offset:%d msg: %+v)", msg.Key, msg.Partition, msg.Offset, m)
+		case msg, ok := <-s.favoriteStatSub.Messages():
+			if !ok {
+				break L
+			}
+			msg.Commit()
+			m := &thumdl.StatMsg{}
+			if err := json.Unmarshal(msg.Value, m); err != nil {
+				log.Error("favorite json.Unmarshal(%s) error(%+v)", msg.Value, err)
+				dao.PromError("service:解析favorite计数databus消息")
+				break L
+			}
+			if m.Type != "article" {
+				continue
+			}
+			key := m.ID % _sharding
+			s.statCh[key] <- &artmdl.StatMsg{Favorite: &m.Count, Aid: m.ID}
+			prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+			log.Info("consumeFavoriteStat key:%s partition:%d offset:%d msg: %+v)", msg.Key, msg.Partition, msg.Offset, m)
+		case msg, ok := <-s.coinStatSub.Messages():
+			if !ok {
+				break L
+			}
+			msg.Commit()
+			m := &thumdl.StatMsg{}
+			if err := json.Unmarshal(msg.Value, m); err != nil {
+				log.Error("coin json.Unmarshal(%s) error(%+v)", msg.Value, err)
+				dao.PromError("service:解析coin计数databus消息")
+				break L
+			}
+			if m.Type != "article" {
+				continue
+			}
+			key := m.ID % _sharding
+			s.statCh[key] <- &artmdl.StatMsg{Coin: &m.Count, Aid: m.ID}
+			prom.BusinessInfoCount.State(fmt.Sprintf("statChan-%v", key), int64(len(s.statCh[key])))
+			log.Info("consumeCoinStat key:%s partition:%d offset:%d msg: %+v)", msg.Key, msg.Partition, msg.Offset, m)
+		}
+	}
+}
+
+// consumeCanal consumes article's binlog databus.
+func (s *Service) consumeCanal() {
+	defer s.waiter.Done()
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.articleSub.Messages()
+		if !ok {
+			log.Info("databus: article-job binlog consumer exit!")
+			return
+		}
+		s.binCnt++
+		msg.Commit()
+		m := &model.Message{}
+		if err := json.Unmarshal(msg.Value, m); err != nil {
+			log.Error("json.Unmarshal(%s) error(%+v)", msg.Value, err)
+			continue
+		}
+		switch m.Table {
+		case _articleTable:
+			s.upArticles(c, m.Action, m.New, m.Old)
+		case _authorTable:
+			s.upAuthors(c, m.Action, m.New, m.Old)
+		}
+		log.Info("consumeCanal key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
+	}
+}
+
+func (s *Service) retryStat() {
+	defer s.waiter.Done()
+	var (
+		err error
+		bs  []byte
+		c   = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		bs, err = s.dao.PopStat(c)
+		if err != nil || bs == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		msg := &dao.StatRetry{}
+		if err = json.Unmarshal(bs, msg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%+v)", bs, err)
+			dao.PromError("service:解析计数重试消息")
+			continue
+		}
+		if msg.Count > dao.RetryStatCount {
+			continue
+		}
+		log.Info("retry: %s", bs)
+		switch msg.Action {
+		case dao.RetryUpdateStatCache:
+			if err = s.updateCache(c, msg.Data, msg.Count+1); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试计数更新缓存")
+		case dao.RetryUpdateStatDB:
+			if err = s.updateDB(c, msg.Data, msg.Count+1); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试计数更新DB")
+		}
+	}
+}
+
+func (s *Service) retryReply() {
+	defer s.waiter.Done()
+	var (
+		err      error
+		aid, mid int64
+		c        = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		aid, mid, err = s.dao.PopReply(c)
+		if err != nil || aid == 0 || mid == 0 {
+			time.Sleep(time.Second)
+			continue
+		}
+		log.Info("retry reply: aid(%d) mid(%d)", aid, mid)
+		if err = s.openReply(c, aid, mid); err != nil {
+			log.Error("s.openReply(%d,%d) error(%+v)", aid, mid, err)
+			dao.PromInfo("service:重试打开评论区")
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		log.Info("s.openReply(%d,%d) retry success", aid, mid)
+	}
+}
+
+func (s *Service) retryCDN() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		file string
+		c    = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		file, err = s.dao.PopCDN(c)
+		if err != nil || file == "" {
+			time.Sleep(time.Second)
+			continue
+		}
+		log.Info("retry CDN: file(%s)", file)
+		if err = s.dao.PurgeCDN(c, file); err != nil {
+			log.Error("s.dao.PurgeCDN(%s) error(%+v)", file, err)
+			dao.PromInfo("service:刷新CDN重试")
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		log.Info("s.dao.PurgeCDN(%s) retry success.", file)
+	}
+}
+
+func (s *Service) retryCache() {
+	defer s.waiter.Done()
+	var (
+		err error
+		bs  []byte
+		c   = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		bs, err = s.dao.PopArtCache(c)
+		if err != nil || bs == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		msg := &dao.CacheRetry{}
+		if err = json.Unmarshal(bs, msg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%+v)", bs, err)
+			dao.PromError("service:解析文章缓存重试消息")
+			continue
+		}
+		log.Info("retry cache: %s", bs)
+		switch msg.Action {
+		case dao.RetryAddArtCache:
+			if err = s.addArtCache(c, msg.Aid); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试添加文章缓存")
+		case dao.RetryUpdateArtCache:
+			if err = s.updateArtCache(c, msg.Aid, msg.Cid); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试更新文章缓存")
+		case dao.RetryDeleteArtCache:
+			if err = s.deleteArtCache(c, msg.Aid, msg.Mid); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试删除文章缓存")
+		case dao.RetryDeleteArtRecCache:
+			if err = s.deleteArtRecommendCache(c, msg.Aid, msg.Cid); err != nil {
+				time.Sleep(100 * time.Millisecond)
+			}
+			dao.PromInfo("service:重试删除文章推荐缓存")
+		}
+	}
+}
+
+// checkConsumer checks consumer state.
+func (s *Service) checkConsumer() {
+	defer s.waiter.Done()
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	var (
+		// ctx = context.TODO()
+		// c* means sum of consumed messages.
+		c1, c2 int64
+	)
+	for {
+		time.Sleep(5 * time.Hour)
+		if s.statCnt-c1 == 0 {
+			// msg := "databus: article-job stat did not consume within a minute"
+			// s.monitor.Sms(ctx, s.c.SMS.Phone, s.c.SMS.Token, msg)
+			// log.Warn(msg)
+			log.Warn("databus: article-job stat did not consume within a minute")
+		}
+		c1 = s.statCnt
+		if s.binCnt-c2 == 0 {
+			// msg := "databus: article-job binlog did not consume within a minute"
+			// s.monitor.Sms(ctx, s.c.SMS.Phone, s.c.SMS.Token, msg)
+			// log.Warn(msg)
+			log.Warn("databus: article-job binlog did not consume within a minute")
+		}
+		c2 = s.binCnt
+	}
+}
+
+// Ping reports the heath of services.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close releases resources which owned by the Service instance.
+func (s *Service) Close() (err error) {
+	defer s.waiter.Wait()
+	s.articleSub.Close()
+	s.articleStatSub.Close()
+	s.closed = true
+	log.Info("article-job has been closed.")
+	return
+}
+
+func (s *Service) retryGame() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		info *model.GameCacheRetry
+		c    = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		info, err = s.dao.PopGameCache(c)
+		if (err != nil) || (info == nil) {
+			time.Sleep(time.Second)
+			continue
+		}
+		for {
+			log.Info("retry_game: %+v", info)
+			dao.PromInfo("service:重试同步游戏")
+			if err = s.dao.GameSync(c, info.Action, info.Aid); err != nil {
+				time.Sleep(time.Millisecond * 100)
+				continue
+			}
+			break
+		}
+		log.Info("s.GameSync(%s, aid: %d) retry success", info.Action, info.Aid)
+	}
+}
+
+func (s *Service) activityLikeproc() {
+	var c = context.TODO()
+	for {
+		msg, ok := <-s.likeCh
+		if !ok {
+			log.Info("activityLikeproc: exit!")
+			return
+		}
+		s.dao.LikeSync(c, msg.ID, msg.Count)
+	}
+}
+
+func (s *Service) retryFlow() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		info *model.FlowCacheRetry
+		c    = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		info, err = s.dao.PopFlowCache(c)
+		if (err != nil) || (info == nil) {
+			time.Sleep(time.Second)
+			continue
+		}
+		for {
+			log.Info("retry_flow: %+v", info)
+			dao.PromInfo("service:重试同步flow")
+			if err = s.dao.FlowSync(c, info.Mid, info.Aid); err != nil {
+				time.Sleep(time.Second * 1)
+				continue
+			}
+			break
+		}
+		log.Info("s.FlowSync(mid:%v, aid: %d) retry success", info.Mid, info.Aid)
+	}
+}
+
+func (s *Service) retryDynamic() {
+	defer s.waiter.Done()
+	var (
+		err  error
+		info *model.DynamicCacheRetry
+		c    = context.TODO()
+	)
+	for {
+		if s.closed {
+			return
+		}
+		info, err = s.dao.PopDynamicCache(c)
+		if (err != nil) || (info == nil) {
+			time.Sleep(time.Second)
+			continue
+		}
+		for {
+			log.Info("retry_dynamic: %+v", info)
+			dao.PromInfo("service:重试同步dynamic")
+			if err = s.dao.PubDynamic(c, info.Mid, info.Aid, info.Show, info.Comment, info.Ts, info.DynamicIntro); err != nil {
+				time.Sleep(time.Second * 1)
+				continue
+			}
+			break
+		}
+		log.Info("s.PubDynamic(mid:%v, aid: %d) retry success %+v", info.Mid, info.Aid, info)
+	}
+}
+
+func (s *Service) updateReadCountproc() {
+	var c = context.TODO()
+	for {
+		err := s.articleRPC.RebuildAllListReadCount(c)
+		if err != nil {
+			log.Error("s.updateReadCountproc() err: %+v", err)
+			time.Sleep(time.Second * 5)
+			continue
+		}
+		log.Info("s.updateReadCountproc() success")
+		dao.PromError("更新文集阅读数")
+		time.Sleep(time.Duration(s.c.Job.ListReadCountInterval))
+	}
+}
+
+func (s *Service) updateHotspotsproc() {
+	var c = context.TODO()
+	var lastUpdate int64
+	for {
+		var force bool
+		duration := int64(time.Duration(s.c.Job.HotspotForceInterval) / time.Second)
+		if (time.Now().Unix() - lastUpdate) > duration {
+			force = true
+		}
+		err := s.articleRPC.UpdateHotspots(c, &artmdl.ArgForce{Force: force})
+		if err != nil {
+			log.Error("s.UpdateHotspots() err: %+v", err)
+			dao.PromError("更新热点运营文章")
+			time.Sleep(time.Second * 5)
+			continue
+		}
+		if force {
+			lastUpdate = time.Now().Unix()
+		}
+		log.Info("s.UpdateHotspots() success force:%v", force)
+		time.Sleep(time.Duration(s.c.Job.HotspotInterval))
+	}
+}
+
+func (s *Service) updateCheatArtsproc() {
+	var c = context.TODO()
+	for {
+		arts, err := s.dao.CheatArts(c)
+		if err != nil {
+			log.Error("s.updateCheatArtsproc() err: %+v", err)
+			dao.PromError("更新反作弊文章列表")
+			time.Sleep(time.Second * 5)
+			continue
+		}
+		log.Info("s.updateCheatArtsproc() success, len: %v", len(arts))
+		s.cheatArts = arts
+		time.Sleep(time.Minute)
+	}
+}

app/job/openplatform/article/service/service_test.go

@@ -0,0 +1,41 @@
+package service
+
+import (
+	"flag"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"go-common/app/job/openplatform/article/conf"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/library/queue/databus"
+)
+
+// func WithService(f func(s *Service)) func() {
+// 	return func() {
+// 		dir, _ := filepath.Abs("../goconvey.toml")
+// 		flag.Set("conf", dir)
+// 		conf.Init()
+// 		s := New(conf.Conf)
+// 		// s.dao = dao.New(conf.Conf)
+// 		f(s)
+// 	}
+// }
+
+func WithoutProcService(f func(s *Service)) func() {
+	return func() {
+		dir, _ := filepath.Abs("../goconvey.toml")
+		flag.Set("conf", dir)
+		conf.Init()
+		s := &Service{
+			c:      conf.Conf,
+			dao:    dao.New(conf.Conf),
+			waiter: new(sync.WaitGroup),
+			// articleRPC:       artrpc.New(conf.Conf.ArticleRPC),
+			articleSub:       databus.New(conf.Conf.ArticleSub),
+			articleStatSub:   databus.New(conf.Conf.ArticleStatSub),
+			updateDbInterval: int64(time.Duration(conf.Conf.Job.UpdateDbInterval) / time.Second),
+		}
+		f(s)
+	}
+}

app/job/openplatform/article/service/setting.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+)
+
+func (s *Service) loadSettings() {
+	for {
+		settings, err := s.dao.Settings(context.TODO())
+		if err != nil || len(settings) == 0 {
+			dao.PromError("service:获取配置")
+			time.Sleep(time.Second)
+			continue
+		}
+		if s.setting == nil {
+			s.setting = &model.Setting{}
+		}
+		for name, value := range settings {
+			switch name {
+			case "recheck_view":
+				var recheckView = &model.Recheck{}
+				if err = json.Unmarshal([]byte(value), recheckView); err != nil {
+					log.Error("setting.Unmarshal(%s) error(%+v)", value, err)
+					dao.PromError("service:配置项无效")
+				} else {
+					s.setting.Recheck = recheckView
+				}
+			}
+		}
+		return
+	}
+}
+
+func (s *Service) loadSettingsproc() {
+	for {
+		time.Sleep(time.Minute)
+		s.loadSettings()
+	}
+}

app/job/openplatform/article/service/sitemap.go

@@ -0,0 +1,153 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/library/log"
+)
+
+type urlNode struct {
+	id   int64
+	next *urlNode
+}
+
+const (
+	_sitemapSize     = 20000
+	_sitemapInterval = int64(60)
+)
+
+func (s *Service) sitemapproc() {
+	if s.urlListHead == nil {
+		s.initURLList(context.TODO())
+	}
+	var interval = s.c.Sitemap.Interval
+	if interval == 0 {
+		interval = _sitemapInterval
+	}
+	for {
+		if s.lastURLNode == nil || s.lastURLNode != s.urlListHead {
+			s.updateXML()
+		}
+		time.Sleep(time.Second * time.Duration(interval))
+	}
+}
+
+// add new url.
+func (s *Service) addURLNode(c context.Context, id int64) {
+	go s.pushShenma(c, id)
+	if s.urlListHead == nil {
+		s.initURLList(c)
+	}
+	if _, ok := s.sitemapMap[id]; ok {
+		return
+	}
+	var (
+		mutex sync.Mutex
+		empty struct{}
+	)
+	mutex.Lock()
+	node := &urlNode{id: id}
+	s.urlListTail.next = node
+	s.urlListTail = node
+	oldID := s.urlListHead.id
+	delete(s.sitemapMap, oldID)
+	s.sitemapMap[id] = empty
+	s.urlListHead = s.urlListHead.next
+	mutex.Unlock()
+}
+
+func (s *Service) initURLList(c context.Context) {
+	var (
+		ids        []int64
+		err        error
+		size       = s.c.Sitemap.Size
+		head, tail *urlNode
+		empty      struct{}
+		mutex      sync.Mutex
+	)
+	if size == 0 {
+		size = _sitemapSize
+	}
+	if ids, err = s.dao.LastModIDs(c, size); err != nil {
+		log.Error("s.initURLList error(%+v) size(%d)", err, size)
+		return
+	}
+	for i := len(ids); i > 0; i-- {
+		id := ids[i-1]
+		node := &urlNode{id: id}
+		if head == nil {
+			head = node
+			tail = node
+			s.sitemapMap[id] = empty
+			continue
+		}
+		tail.next = node
+		tail = node
+		s.sitemapMap[id] = empty
+	}
+	if head != nil && tail != nil {
+		mutex.Lock()
+		s.urlListHead = head
+		s.urlListTail = tail
+		mutex.Unlock()
+	}
+}
+
+func (s *Service) updateXML() {
+	var (
+		node  = s.urlListHead
+		xml   = ""
+		mutex sync.Mutex
+	)
+	for node != nil {
+		xml = `<url><loc>http://www.bilibili.com/read/cv` + strconv.FormatInt(node.id, 10) + `/</loc><lastmod>` + time.Now().Format("2006-01-02") + `</lastmod><changefreq>always</changefreq></url>` + xml
+		node = node.next
+	}
+	xml = `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.google.com/schemas/sitemap/0.84">` + xml + `</urlset>`
+	mutex.Lock()
+	s.sitemapXML = xml
+	s.lastURLNode = s.urlListHead
+	mutex.Unlock()
+}
+
+// SitemapXML .
+func (s *Service) SitemapXML(c context.Context) (string, string) {
+	return s.sitemapXML, strconv.Itoa(len(s.sitemapXML))
+}
+
+func (s *Service) pushShenma(c context.Context, id int64) {
+	url := `http://data.zhanzhang.sm.cn/push?site=www.bilibili.com&user_name=zjdxgxy@163.com&resource_name=mip_add&token=TI_a1e0216ad1a64b63e9bbadfc7579f131`
+	urlNode := "www.bilibili.com/read/mobile/" + strconv.FormatInt(id, 10)
+	resp, err := http.Post(url, "text/plain", strings.NewReader(urlNode))
+	if err != nil {
+		log.Error("s.pushShenma post error(%+v)", err)
+		return
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Error("s.pushShenma ReadAll error(%+v)", err)
+		return
+	}
+	var data struct {
+		ReturnCode int    `json:"returnCode"`
+		ErrMsg     string `json:"errMsg"`
+	}
+	if err = json.Unmarshal(body, &data); err != nil {
+		log.Error("s.pushShenma json unmarshal error(%+v)", err)
+		return
+	}
+	if data.ReturnCode == 200 {
+		log.Info("s.pushShenma success, url(%s)", urlNode)
+	} else {
+		log.Error("s.pushShenma success, errMsg(%s), url(%s)", data.ErrMsg, urlNode)
+	}
+	return
+}

app/job/openplatform/article/service/sort.go

@@ -0,0 +1,182 @@
+package service
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+)
+
+var (
+	_recommendCategory = int64(0)
+)
+
+// UpdateSort update sort
+func (s *Service) UpdateSort(c context.Context) (err error) {
+	ids := []int64{_recommendCategory}
+	for id := range s.categoriesMap {
+		ids = append(ids, id)
+	}
+	var (
+		cache   bool
+		allArts map[int64]map[int][][2]int64
+	)
+	for _, categoryID := range ids {
+		for _, field := range artmdl.SortFields {
+			if (categoryID == _recommendCategory) && (field != artmdl.FieldNew) {
+				continue
+			}
+			if cache, err = s.dao.ExpireSortCache(c, categoryID, field); err != nil {
+				dao.PromError("recommends:更新文章排序")
+				return
+			}
+			// cache不存在才更新
+			if cache {
+				continue
+			}
+			dao.PromInfo("sort:初始化排序缓存")
+			if allArts == nil {
+				allArts, _ = s.loadSortArts(context.TODO())
+			}
+			if field == artmdl.FieldNew {
+				s.updateNewArts(c, categoryID)
+			} else {
+				s.updateStatSort(c, allArts, categoryID, field)
+			}
+		}
+	}
+	return
+}
+
+func (s *Service) updateNewArts(c context.Context, categoryID int64) (err error) {
+	var arts [][2]int64
+	if categoryID == _recommendCategory {
+		if arts, err = s.dao.NewestArtIDs(c, s.c.Job.MaxNewArtsNum); err == nil {
+			log.Info("s.updateNewArts() len: %v", len(arts))
+			// 不异步
+			err = s.dao.AddSortCaches(c, _recommendCategory, artmdl.FieldNew, arts, s.c.Job.MaxNewArtsNum)
+		}
+		return
+	}
+	ids := []int64{}
+	cs := s.categoriesReverseMap[categoryID]
+	if len(cs) == 0 {
+		// 二级分区
+		ids = append(ids, categoryID)
+	} else {
+		// 一级分区聚合子分区
+		for _, s := range cs {
+			ids = append(ids, s.ID)
+		}
+	}
+	if arts, err = s.dao.NewestArtIDByCategory(c, ids, s.c.Job.MaxNewArtsNum); err == nil {
+		// 不异步
+		err = s.dao.AddSortCaches(c, categoryID, artmdl.FieldNew, arts, s.c.Job.MaxNewArtsNum)
+	}
+	return
+}
+
+func (s *Service) updateStatSort(c context.Context, allArts map[int64]map[int][][2]int64, categoryID int64, field int) (err error) {
+	if allArts == nil {
+		return
+	}
+	if categoryID == _recommendCategory {
+		//推荐下没有排序
+		return
+	}
+	arts := trimArts(allArts[categoryID][field], int(s.c.Job.MaxSortArtsNum))
+	err = s.dao.AddSortCaches(c, categoryID, field, arts, s.c.Job.MaxSortArtsNum)
+	return
+}
+
+func trimArts(arts [][2]int64, max int) (res [][2]int64) {
+	sort.Slice(arts, func(i, j int) bool {
+		return arts[i][1] > arts[j][1]
+	})
+	if len(arts) > max {
+		return arts[:max]
+	}
+	return arts
+}
+
+func (s *Service) loadSortArts(c context.Context) (res map[int64]map[int][][2]int64, err error) {
+	// init category and field
+	res = make(map[int64]map[int][][2]int64)
+	for id := range s.categoriesMap {
+		res[id] = make(map[int][][2]int64)
+	}
+	var (
+		arts      []*model.SearchArticle
+		limitTime = time.Now().Unix() - s.sortLimitTime
+	)
+	if arts, err = s.dao.SearchArts(c, limitTime); err != nil {
+		dao.PromError("sort:初始化排序计数失败")
+		return
+	}
+	for _, art := range arts {
+		if artmdl.NoDistributeAttr(art.Attributes) || artmdl.NoRegionAttr(art.Attributes) {
+			continue
+		}
+		if res[art.CategoryID] == nil {
+			res[art.CategoryID] = make(map[int][][2]int64)
+		}
+		res[art.CategoryID][artmdl.FieldFav] = append(res[art.CategoryID][artmdl.FieldFav], [2]int64{art.ID, art.StatsFavorite})
+		res[art.CategoryID][artmdl.FieldLike] = append(res[art.CategoryID][artmdl.FieldLike], [2]int64{art.ID, art.StatsLikes})
+		res[art.CategoryID][artmdl.FieldReply] = append(res[art.CategoryID][artmdl.FieldReply], [2]int64{art.ID, art.StatsReply})
+		res[art.CategoryID][artmdl.FieldView] = append(res[art.CategoryID][artmdl.FieldView], [2]int64{art.ID, art.StatsView})
+		var parentID int64
+		if id, ok := s.categoriesMap[art.CategoryID]; ok {
+			parentID = id.ParentID
+		} else {
+			continue
+		}
+		if res[parentID] == nil {
+			res[parentID] = make(map[int][][2]int64)
+		}
+		res[parentID][artmdl.FieldFav] = append(res[parentID][artmdl.FieldFav], [2]int64{art.ID, art.StatsFavorite})
+		res[parentID][artmdl.FieldLike] = append(res[parentID][artmdl.FieldLike], [2]int64{art.ID, art.StatsLikes})
+		res[parentID][artmdl.FieldReply] = append(res[parentID][artmdl.FieldReply], [2]int64{art.ID, art.StatsReply})
+		res[parentID][artmdl.FieldView] = append(res[parentID][artmdl.FieldView], [2]int64{art.ID, art.StatsView})
+	}
+	return
+}
+
+func (s *Service) loadCategoriesproc() {
+	for {
+		time.Sleep(time.Minute * 10)
+		s.loadCategories()
+	}
+}
+
+func (s *Service) loadCategories() {
+	for {
+		c, err := s.articleRPC.CategoriesMap(context.TODO(), &artmdl.ArgIP{})
+		if err != nil || len(c) == 0 {
+			dao.PromError("service:获取分类")
+			log.Error("s.articleRPC.CategoriesMap err %v", err)
+			time.Sleep(time.Second)
+			continue
+		}
+		s.categoriesMap = c
+		s.categoriesReverseMap = transformReverseCategory(c)
+		return
+	}
+}
+
+// 生成某个分类下的所有子分类
+func transformReverseCategory(cs map[int64]*artmdl.Category) (res map[int64][]*artmdl.Category) {
+	res = make(map[int64][]*artmdl.Category)
+	for _, c := range cs {
+		n := c
+		old := c
+		for (n != nil) && (n.ParentID != 0) {
+			res[n.ParentID] = append(res[n.ParentID], old)
+			n = cs[n.ParentID]
+		}
+	}
+	return
+}

app/job/openplatform/article/service/sort_test.go

@@ -0,0 +1,38 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_UpdateSort(t *testing.T) {
+	Convey("get data", t, WithoutProcService(func(s *Service) {
+		err := s.UpdateSort(context.TODO())
+		So(err, ShouldBeNil)
+	}))
+}
+func Test_loadSortArts(t *testing.T) {
+	Convey("get data", t, WithoutProcService(func(s *Service) {
+		res, err := s.loadSortArts(context.TODO())
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeEmpty)
+	}))
+}
+
+func Test_TrimArts(t *testing.T) {
+	arts := [][2]int64{
+		{1, 11},
+		{0, 10},
+		{2, 12},
+	}
+	Convey("trim arts", t, func() {
+		ns := trimArts(arts, 3)
+		expt := [][2]int64{
+			{2, 12},
+			{1, 11},
+		}
+		So(ns, ShouldResemble, expt)
+	})
+}

app/job/openplatform/article/service/stat.go

@@ -0,0 +1,216 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/app/job/openplatform/article/model"
+	"go-common/library/log"
+)
+
+func (s *Service) statproc(i int64) {
+	defer s.waiter.Done()
+	var (
+		err  error
+		ls   *lastTimeStat
+		c    = context.TODO()
+		ch   = s.statCh[i]
+		last = s.statLastTime[i]
+	)
+	for {
+		stat, ok := <-ch
+		if !ok {
+			log.Warn("statproc(%d) quit", i)
+			s.multiUpdateDB(i, last)
+			return
+		}
+		// filter view count
+		if stat.View != nil && *stat.View > 0 {
+			var ban bool
+			var reason, valid string
+			if ban = s.intercept(stat); ban {
+				log.Info("intercept view count (aid:%d, ip:%s, mid:%d)", stat.Aid, stat.IP, stat.Mid)
+				dao.PromInfo("stat:访问计数拦截")
+				reason = "访问计数拦截"
+			} else if ban = s.dao.DupViewIntercept(c, stat.Aid, stat.Mid); ban {
+				log.Info("dupintercept view count (aid:%d, ip:%s, mid:%d)", stat.Aid, stat.IP, stat.Mid)
+				dao.PromInfo("stat:重复访问计数拦截")
+				reason = "重复访问计数拦截"
+			} else if stat.CheatInfo != nil {
+				viewLv, _ := strconv.Atoi(stat.CheatInfo.Lv)
+				if limitLv, ok1 := s.cheatArts[stat.Aid]; ok1 && (viewLv <= limitLv) {
+					ban = true
+					log.Info("lvintercept view count (aid:%d, ip:%s, mid:%d)", stat.Aid, stat.IP, stat.Mid)
+					dao.PromInfo("stat:等级访问计数拦截")
+					reason = "等级访问计数拦截"
+				}
+			}
+			if ban {
+				valid = "0"
+			} else {
+				valid = "1"
+			}
+			if stat.CheatInfo != nil {
+				stat.CheatInfo.Valid = valid
+				stat.CheatInfo.Reason = reason
+			}
+			s.cheatInfo(stat.CheatInfo)
+			if ban {
+				continue
+			}
+		}
+		// get stat
+		if ls, ok = last[stat.Aid]; !ok {
+			var st *artmdl.StatMsg
+			if st, err = s.dao.Stat(c, stat.Aid); err != nil {
+				log.Error("s.dao.Stat(%d) error(%+v)", stat.Aid, err)
+				continue
+			}
+			ls = &lastTimeStat{}
+			if st == nil {
+				ls.stat = &artmdl.StatMsg{Aid: stat.Aid, View: new(int64), Like: new(int64), Dislike: new(int64), Favorite: new(int64), Reply: new(int64), Share: new(int64), Coin: new(int64)}
+				ls.time = 0 // NOTE: make sure update db in first.
+			} else {
+				ls.stat = st
+				ls.time = time.Now().Unix()
+			}
+			last[stat.Aid] = ls
+		}
+		changed := model.Merge(ls.stat, stat)
+		// update cache
+		s.updateCache(c, ls.stat, 0)
+		s.updateSortCache(c, ls.stat.Aid, changed)
+		// update db after 120s
+		if time.Now().Unix()-ls.time > s.updateDbInterval {
+			s.updateDB(c, ls.stat, 0)
+			s.updateRecheckDB(c, ls.stat)
+			s.updateSearchStats(c, ls.stat)
+			delete(last, stat.Aid) // NOTE: delete ensures that memory should be normal in 120s after channel has been closed.
+		}
+	}
+}
+
+// updateCache purge stat info in cache
+func (s *Service) updateCache(c context.Context, sm *artmdl.StatMsg, count int) (err error) {
+	stat := &artmdl.ArgStats{
+		Aid: sm.Aid,
+		Stats: &artmdl.Stats{
+			View:     *sm.View,
+			Like:     *sm.Like,
+			Dislike:  *sm.Dislike,
+			Favorite: *sm.Favorite,
+			Reply:    *sm.Reply,
+			Share:    *sm.Share,
+			Coin:     *sm.Coin,
+		},
+	}
+	if err = s.articleRPC.SetStat(context.TODO(), stat); err != nil {
+		log.Error("s.articleRPC.SetStat aid(%d) view(%d) likes(%d) dislike(%d) favorite(%d) reply(%d) share(%d) coin(%d) error(%+v)",
+			sm.Aid, *sm.View, *sm.Like, *sm.Dislike, *sm.Favorite, *sm.Reply, *sm.Share, *sm.Coin, err)
+		dao.PromError("stat:更新计数缓存")
+		s.dao.PushStat(c, &dao.StatRetry{
+			Action: dao.RetryUpdateStatCache,
+			Count:  count,
+			Data:   sm,
+		})
+		return
+	}
+	log.Info("update cache success aid(%d) view(%d) likes(%d) dislike(%d) favorite(%d) reply(%d) share(%d) coin(%d)",
+		sm.Aid, *sm.View, *sm.Like, *sm.Dislike, *sm.Favorite, *sm.Reply, *sm.Share, *sm.Coin)
+	dao.PromInfo("stat:更新计数缓存")
+	return
+}
+
+// updateSortCache update sort cache
+func (s *Service) updateSortCache(c context.Context, aid int64, changed [][2]int64) (err error) {
+	if len(changed) == 0 {
+		return
+	}
+	arg := &artmdl.ArgSort{
+		Aid:     aid,
+		Changed: changed,
+	}
+	if err = s.articleRPC.UpdateSortCache(context.TODO(), arg); err != nil {
+		log.Error("s.articleRPC.UpdateSortCache(aid:%v arg: %+v)", aid, arg)
+		dao.PromError("stat:更新排序缓存")
+		return
+	}
+	log.Info("success s.articleRPC.UpdateSortCache(aid:%v arg: %+v)", aid, arg)
+	dao.PromInfo("stat:更新排序缓存")
+	return
+}
+
+// updateDB update stat in db.
+func (s *Service) updateDB(c context.Context, stat *artmdl.StatMsg, count int) (err error) {
+	if _, err = s.dao.Update(context.TODO(), stat); err != nil {
+		dao.PromError("stat:更新计数DB")
+		s.dao.PushStat(c, &dao.StatRetry{
+			Action: dao.RetryUpdateStatDB,
+			Count:  count,
+			Data:   stat,
+		})
+		return
+	}
+	log.Info("update db success aid(%d) view(%d) likes(%d) dislike(%d) favorite(%d) reply(%d) share(%d) coin(%d)",
+		stat.Aid, *stat.View, *stat.Like, *stat.Dislike, *stat.Favorite, *stat.Reply, *stat.Share, *stat.Coin)
+	return
+}
+
+// multiUpdateDB update some stat in db.
+func (s *Service) multiUpdateDB(i int64, last map[int64]*lastTimeStat) (err error) {
+	log.Info("multiUpdateDB close(%d) start", i)
+	c := context.TODO()
+	for aid, ls := range last {
+		s.updateDB(c, ls.stat, 0)
+		log.Info("multiUpdateDB close(%d) update stats aid: %d, value: %+v", i, aid, ls.stat)
+	}
+	log.Info("multiUpdateDB close(%d) end", i)
+	return
+}
+
+// intercept intercepts illegal views.
+func (s *Service) intercept(stat *artmdl.StatMsg) bool {
+	return s.dao.Intercept(context.TODO(), stat.Aid, stat.Mid, stat.IP)
+}
+
+func (s *Service) cheatInfo(cheat *artmdl.CheatInfo) {
+	if cheat == nil {
+		return
+	}
+	log.Info("cheatInfo %+v", cheat)
+	if err := s.cheatInfoc.Info(cheat.Valid, cheat.Client, cheat.Cvid, cheat.Mid, cheat.Lv, cheat.Ts, cheat.IP, cheat.UA, cheat.Refer, cheat.Sid, cheat.Buvid, cheat.DeviceID, cheat.Build, cheat.Reason); err != nil {
+		log.Error("cheatInfo error(%+v)", err)
+	}
+}
+
+func (s *Service) updateRecheckDB(c context.Context, stat *artmdl.StatMsg) (err error) {
+	var (
+		publishTime int64
+		checkState  int
+	)
+	if s.setting.Recheck.View == 0 || s.setting.Recheck.Day == 0 {
+		return
+	}
+	if isRecheck, _ := s.dao.GetRecheckCache(c, stat.Aid); isRecheck {
+		return
+	}
+	if publishTime, checkState, err = s.dao.GetRecheckInfo(c, stat.Aid); err != nil || checkState != 0 {
+		return
+	}
+
+	if s.dao.IsAct(c, stat.Aid) {
+		return
+	}
+	if *(stat.View) > s.setting.Recheck.View {
+		if publishTime+60*60*24*s.setting.Recheck.Day+s.updateDbInterval > time.Now().Unix() {
+			if err = s.dao.UpdateRecheck(c, stat.Aid); err == nil {
+				log.Info("update recheck success aid(%d)", stat.Aid)
+				s.dao.SetRecheckCache(c, stat.Aid)
+			}
+		}
+	}
+	return
+}

app/job/openplatform/article/service/stat_test.go

@@ -0,0 +1,45 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	artmdl "go-common/app/interface/openplatform/article/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_Stat(t *testing.T) {
+
+	var (
+		err     error
+		c       = context.TODO()
+		statCnt = int64(5)
+		stat    = &artmdl.StatMsg{
+			Aid:      88,
+			View:     &statCnt,
+			Favorite: &statCnt,
+			Like:     &statCnt,
+			Dislike:  &statCnt,
+			Reply:    &statCnt,
+			Share:    &statCnt,
+		}
+	)
+
+	Convey("updateCache", t, WithoutProcService(func(s *Service) {
+		err = s.updateCache(c, stat, 0)
+		So(err, ShouldBeNil)
+	}))
+
+	Convey("updateDB", t, WithoutProcService(func(s *Service) {
+		err = s.updateDB(c, stat, 0)
+		So(err, ShouldBeNil)
+	}))
+
+	Convey("select Stat", t, WithoutProcService(func(s *Service) {
+		var stat *artmdl.StatMsg
+		stat, err = s.dao.Stat(c, 1)
+		So(err, ShouldBeNil)
+		So(stat, ShouldNotBeEmpty)
+	}))
+}

app/job/openplatform/article/service/tag.go

@@ -0,0 +1,31 @@
+package service
+
+import (
+	"context"
+	"strings"
+
+	"go-common/app/interface/main/tag/model"
+	"go-common/app/job/openplatform/article/dao"
+	"go-common/library/log"
+)
+
+// tags gets article tags.
+func (s *Service) tags(c context.Context, aid int64) (res string, err error) {
+	var (
+		tags map[int64][]*model.Tag
+		ts   []string
+		arg  = &model.ArgResTags{Type: model.PicResType, Oids: []int64{aid}}
+	)
+	if tags, err = s.tagRPC.ResTags(c, arg); err != nil {
+		log.Error("s.Tags(%d) error(%+v)", aid, err)
+		dao.PromError("rpc:获取Tag")
+		return
+	} else if tags == nil || len(tags[aid]) == 0 {
+		return
+	}
+	for _, t := range tags[aid] {
+		ts = append(ts, t.Name)
+	}
+	res = strings.Join(ts, ",")
+	return
+}