Create & Init Project...

app/job/main/stat/dao/BUILD

@@ -0,0 +1,53 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dao_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/stat/conf:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+        "sms.go",
+    ],
+    importpath = "go-common/app/job/main/stat/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/stat/conf:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/stat/dao/dao.go

@@ -0,0 +1,32 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/main/stat/conf"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao is stat job dao.
+type Dao struct {
+	c         *conf.Config
+	smsClient *bm.Client
+	db        *xsql.DB
+	clickDB   *xsql.DB
+}
+
+// New add a feed job dao.
+func New(c *conf.Config) *Dao {
+	return &Dao{
+		c:         c,
+		smsClient: bm.NewClient(c.HTTPClient),
+		db:        xsql.NewMySQL(c.DB),
+		clickDB:   xsql.NewMySQL(c.ClickDB),
+	}
+}
+
+// Ping ping health of db.
+func (d *Dao) Ping(c context.Context) (err error) {
+	return d.db.Ping(c)
+}

app/job/main/stat/dao/dao_test.go

@@ -0,0 +1,56 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+
+	"go-common/app/job/main/stat/conf"
+	"go-common/app/service/main/archive/api"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var d *Dao
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/stat-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}
+
+func Test_SendQiyeWX(t *testing.T) {
+	Convey("SendQiyeWX", t, func() {
+		d.SendQiyeWX("日志报警:stat-job topic(test) 不消费了")
+	})
+}
+
+func Test_Ping(t *testing.T) {
+	Convey("Ping", t, func() {
+		d.Ping(context.TODO())
+	})
+}
+
+func Test_Stat(t *testing.T) {
+	Convey("Stat", t, func() {
+		st, err := d.Stat(context.TODO(), 10989901)
+		So(err, ShouldBeNil)
+		Printf("%+v", st)
+
+	})
+}
+
+func Test_Update(t *testing.T) {
+	Convey("Update", t, func() {
+		_, err := d.Update(context.TODO(), &api.Stat{Aid: 10989901, Fav: 100, DisLike: 10, Like: 20})
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_MaxAID(t *testing.T) {
+	Convey("MaxAID", t, func() {
+		d.MaxAID(context.TODO())
+	})
+}

app/job/main/stat/dao/mysql.go

@@ -0,0 +1,93 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/service/main/archive/api"
+	"go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+const (
+	_sharding  = 100
+	_statSQL   = "SELECT aid,click,fav,share,reply,coin,dm,now_rank,his_rank,likes,dislike FROM archive_stat_%s WHERE aid=%d"
+	_upStatSQL = `INSERT INTO archive_stat_%s (aid,click,fav,share,reply,coin,dm,now_rank,his_rank,ctime,mtime,likes,dislike) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
+				ON DUPLICATE KEY UPDATE click=?,fav=?,share=?,reply=?,coin=?,dm=?,now_rank=?,his_rank=?,mtime=?,likes=?,dislike=?`
+	_upMStatSQL = `REPLACE INTO archive_stat_%02d (aid,click,fav,share,reply,coin,dm,now_rank,his_rank,mtime,likes,dislike) VALUES %s`
+	_clickSQL   = "SELECT aid,web,h5,outside,ios,android FROM archive_click_%02d WHERE aid=?"
+)
+
+func (d *Dao) hit(id int64) string {
+	return fmt.Sprintf("%02d", id%_sharding)
+}
+
+func statTbl(aid int64) int64 {
+	return aid % _sharding
+}
+
+// Stat returns stat info
+func (d *Dao) Stat(c context.Context, aid int64) (stat *api.Stat, err error) {
+	stat = &api.Stat{}
+	err = d.db.QueryRow(c, fmt.Sprintf(_statSQL, d.hit(aid), aid)).Scan(&stat.Aid, &stat.View, &stat.Fav, &stat.Share, &stat.Reply, &stat.Coin, &stat.Danmaku, &stat.NowRank, &stat.HisRank, &stat.Like, &stat.DisLike)
+	if err == sql.ErrNoRows {
+		err = nil
+		stat = nil
+	} else if err != nil {
+		log.Error("Stat(%v) error(%v)", aid, err)
+	}
+	return
+}
+
+// Update update stat's fields
+func (d *Dao) Update(c context.Context, stat *api.Stat) (rows int64, err error) {
+	now := time.Now()
+	res, err := d.db.Exec(c, fmt.Sprintf(_upStatSQL, d.hit(stat.Aid)), stat.Aid, stat.View, stat.Fav, stat.Share, stat.Reply, stat.Coin, stat.Danmaku, stat.NowRank, stat.HisRank, now, now, stat.Like, stat.DisLike,
+		stat.View, stat.Fav, stat.Share, stat.Reply, stat.Coin, stat.Danmaku, stat.NowRank, stat.HisRank, now, stat.Like, stat.DisLike)
+	if err != nil {
+		log.Error("UpdateStat(%d,%v) error(%v)", stat.Aid, stat, err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// MultiUpdate update some stat's fields
+func (d *Dao) MultiUpdate(c context.Context, yu int64, stats ...*api.Stat) (rows int64, err error) {
+	if len(stats) == 0 {
+		return
+	}
+	const field = `(%d,%d,%d,%d,%d,%d,%d,%d,%d,'%s',%d,%d)`
+	var (
+		fsqls = make([]string, 0, len(stats))
+		now   = time.Now().Format("2006-01-02 15:04:05")
+	)
+	for _, stat := range stats {
+		fsqls = append(fsqls, fmt.Sprintf(field, stat.Aid, stat.View, stat.Fav, stat.Share, stat.Reply, stat.Coin, stat.Danmaku, stat.NowRank, stat.HisRank, now, stat.Like, stat.DisLike))
+	}
+	res, err := d.db.Exec(c, fmt.Sprintf(_upMStatSQL, yu, strings.Join(fsqls, ",")))
+	if err != nil {
+		log.Error("upMstat error(%v)", err)
+		return
+	}
+	rows, err = res.RowsAffected()
+	return
+}
+
+// Click archive click.
+func (d *Dao) Click(c context.Context, aid int64) (cl *archive.Click3, err error) {
+	row := d.clickDB.QueryRow(c, fmt.Sprintf(_clickSQL, statTbl(aid)), aid)
+	cl = &archive.Click3{}
+	if err = row.Scan(&cl.Aid, &cl.Web, &cl.H5, &cl.Outter, &cl.Ios, &cl.Android); err != nil {
+		if err == sql.ErrNoRows {
+			cl = nil
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}

app/job/main/stat/dao/sms.go

@@ -0,0 +1,73 @@
+package dao
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	_maxAIDPath = "http://api.bilibili.co/x/internal/v2/archive/maxAid"
+)
+
+// MaxAID return max aid
+func (d *Dao) MaxAID(c context.Context) (id int64, err error) {
+	var res struct {
+		Code int   `json:"code"`
+		Data int64 `json:"data"`
+	}
+	if err = d.smsClient.Get(c, _maxAIDPath, "", nil, &res); err != nil {
+		return
+	}
+	if res.Code != 0 {
+		log.Error("d.client.MaxAid Code(%d)", res.Code)
+		return
+	}
+	log.Info("got MaxAid(%d)", res.Data)
+	id = res.Data
+	return
+}
+
+// SendQiyeWX send qiye wx
+func (d *Dao) SendQiyeWX(msg string) {
+	type wxParams struct {
+		Username  string `json:"username"`
+		Content   string `json:"content"`
+		Token     string `json:"token"`
+		Timestamp int64  `json:"timestamp"`
+		Sign      string `json:"signature"`
+	}
+	var resp struct {
+		Status int64  `json:"status"`
+		Msg    string `json:"msg"`
+	}
+	params := url.Values{}
+	params.Set("username", d.c.Monitor.Users)
+	params.Set("content", msg)
+	params.Set("token", d.c.Monitor.Token)
+	params.Set("timestamp", strconv.FormatInt(time.Now().Unix(), 10))
+	mh := md5.Sum([]byte(params.Encode() + d.c.Monitor.Secret))
+	params.Set("signature", hex.EncodeToString(mh[:]))
+	p := &wxParams{
+		Username: params.Get("username"),
+		Content:  params.Get("content"),
+		Token:    params.Get("token"),
+		Sign:     params.Get("signature"),
+	}
+	p.Timestamp, _ = strconv.ParseInt(params.Get("timestamp"), 10, 64)
+	bs, _ := json.Marshal(p)
+	payload := strings.NewReader(string(bs))
+	req, _ := http.NewRequest("POST", d.c.Monitor.URL, payload)
+	req.Header.Add("content-type", "application/json; charset=utf-8")
+	if err := d.smsClient.Do(context.TODO(), req, &resp); err != nil {
+		log.Error("d.smsClient.Do() error(%v)", err)
+	}
+}