Create & Init Project...

app/job/main/coin/dao/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/coin/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/coin/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/coin/conf:go_default_library",
+        "//app/job/main/coin/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/coin/dao/dao.go

@@ -0,0 +1,55 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/job/main/coin/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/stat/prom"
+)
+
+// Dao define dao.
+type Dao struct {
+	c                   *conf.Config
+	coinDB              *sql.DB
+	hitSettlePeriodStmt *sql.Stmt
+	getSettlePeriodStmt *sql.Stmt
+	getTotalCoinsStmt   []*sql.Stmt
+	redis               *redis.Pool
+	loginExpire         int32
+}
+
+// PromError .
+func PromError(name string) {
+	prom.BusinessErrCount.Incr(name)
+}
+
+// New new and return service.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:                 c,
+		coinDB:            sql.NewMySQL(c.DB.Coin),
+		getTotalCoinsStmt: make([]*sql.Stmt, SHARDING),
+		redis:             redis.NewPool(c.Redis),
+		loginExpire:       int32(time.Duration(c.CoinJob.LoginExpire) / time.Second),
+	}
+	for i := 0; i < SHARDING; i++ {
+		d.getTotalCoinsStmt[i] = d.coinDB.Prepared(fmt.Sprintf(_getTotalCoins, i))
+	}
+	d.hitSettlePeriodStmt = d.coinDB.Prepared(_hitSettlePeriod)
+	d.getSettlePeriodStmt = d.coinDB.Prepared(_getSettlePeriod)
+	return
+}
+
+// Ping check service health.
+func (d *Dao) Ping(c context.Context) error {
+	return d.coinDB.Ping(c)
+}
+
+// Close close sevice.
+func (d *Dao) Close() {
+	d.coinDB.Close()
+}

app/job/main/coin/dao/dao_test.go

@@ -0,0 +1,39 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/coin/conf"
+)
+
+var (
+	d   *Dao
+	ctx = context.TODO()
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.account.coin-job")
+		flag.Set("conf_appid", "coin-job")
+		flag.Set("conf_token", "RiSyxxaGuihmdbKCq9Pba1JwHRTmQjMI")
+		flag.Set("tree_id", "2130")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/coin-job-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/job/main/coin/dao/mysql.go

@@ -0,0 +1,164 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/coin/model"
+	"go-common/library/log"
+)
+
+const (
+	// SHARDING table shard.
+	SHARDING = 50
+	// coin_settle_x
+	_getSettle       = "SELECT id,mid,aid,type,coin_count,exp_sub,state FROM coin_settle_%d WHERE id> ? LIMIT 10000"
+	_hitSettlePeriod = "SELECT id,from_year,from_month,from_day,to_year,to_month,to_day FROM coin_settle_period WHERE from_year*10000+from_month*100+from_day<=? AND to_year*10000+to_month*100+to_day>?"
+	_getSettlePeriod = "SELECT id,from_year,from_month,from_day,to_year,to_month,to_day FROM coin_settle_period WHERE id=?"
+	_clearCoinCount  = "UPDATE coin_settle_%d SET coin_count=0, mtime=?"
+	_updateCoinCount = "UPDATE coin_settle_%d SET coin_count=?, mtime=? WHERE aid=? AND type=?"
+	_updateSettle    = "UPDATE coin_settle_%d SET state=1, exp_total=?, mtime=? WHERE id=?"
+	_upsertSettle    = "INSERT INTO coin_settle_%d (mid,aid,type,coin_count,ctime,mtime) VALUES (?,?,?,?,?,?) ON DUPLICATE KEY UPDATE coin_count=coin_count+?,state=0, mtime=?"
+	// coin_archive_x
+	_getTotalCoins = "SELECT aid,type,SUM(multiply) FROM coin_archive_%d WHERE timestamp>=? AND timestamp<? GROUP BY aid,type"
+)
+
+func hashField(aid, tp int64) int64 {
+	return aid*1000 + tp
+}
+
+// SettlePeriod settle coin by tableid.
+func (dao *Dao) SettlePeriod(c context.Context, id int64) (period *model.CoinSettlePeriod, err error) {
+	row := dao.getSettlePeriodStmt.QueryRow(c, id)
+	period = &model.CoinSettlePeriod{}
+	if err = row.Scan(&period.ID,
+		&period.FromYear,
+		&period.FromMonth,
+		&period.FromDay,
+		&period.ToYear,
+		&period.ToMonth,
+		&period.ToDay); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+	}
+	return
+}
+
+// HitSettlePeriod get table id.
+func (dao *Dao) HitSettlePeriod(c context.Context, now time.Time) (period *model.CoinSettlePeriod, err error) {
+	ymd, _ := strconv.Atoi(now.Format("20060102"))
+	row := dao.hitSettlePeriodStmt.QueryRow(c, ymd, ymd)
+	period = &model.CoinSettlePeriod{}
+	if err = row.Scan(&period.ID,
+		&period.FromYear,
+		&period.FromMonth,
+		&period.FromDay,
+		&period.ToYear,
+		&period.ToMonth,
+		&period.ToDay); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+		} else {
+			log.Error("row.Scan error(%v)", err)
+		}
+		return
+	}
+	if period.ID == 0 {
+		err = fmt.Errorf("zero id at(%d)", ymd)
+	}
+	return
+}
+
+// UpdateSettle update settle info.
+func (dao *Dao) UpdateSettle(c context.Context, tableID, id, expTotal int64, now time.Time) (err error) {
+	sqlStr := fmt.Sprintf(_updateSettle, tableID)
+	if _, err = dao.coinDB.Exec(c, sqlStr, expTotal, now, id); err != nil {
+		log.Error("dao.coinDB.Exec(%s, %d, %v, %d) error(%v)", sqlStr, expTotal, now, id, err)
+		PromError("db:UpdateSettle")
+	}
+	return
+}
+
+// UpdateCoinCount update coin.
+func (dao *Dao) UpdateCoinCount(c context.Context, tableID, aid, tp, count int64, now time.Time) (err error) {
+	sqlStr := fmt.Sprintf(_updateCoinCount, tableID)
+	if _, err = dao.coinDB.Exec(c, sqlStr, count, now, aid, tp); err != nil {
+		log.Error("dao.coinDB.Exec(%s, %d, %v, %d) error(%v)", sqlStr, count, now, aid, err)
+	}
+	return
+}
+
+// Every10000 get 10000 coin record.
+func (dao *Dao) Every10000(c context.Context, tableID int64, idx int64) (settles []*model.CoinSettle, maxid int64, err error) {
+	sqlStr := fmt.Sprintf(_getSettle, tableID)
+	rows, err := dao.coinDB.Query(c, sqlStr, idx)
+	if err != nil {
+		log.Error("dao.coinDB.Query(%s, %d) error(%v)", sqlStr, idx, err)
+		return
+	}
+	defer rows.Close()
+	maxid = idx
+	settles = make([]*model.CoinSettle, 0, 10000)
+	for rows.Next() {
+		settle := &model.CoinSettle{}
+		if err = rows.Scan(&settle.ID,
+			&settle.Mid,
+			&settle.Aid,
+			&settle.AvType,
+			&settle.CoinCount,
+			&settle.ExpSub,
+			&settle.State); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		if settle.ID > maxid {
+			maxid = settle.ID
+		}
+		settles = append(settles, settle)
+	}
+	return
+}
+
+// UpsertSettle update coin settle.
+func (dao *Dao) UpsertSettle(c context.Context, tableID, mid, aid, tp, coinCount int64, now time.Time) (err error) {
+	sqlStr := fmt.Sprintf(_upsertSettle, tableID)
+	if _, err = dao.coinDB.Exec(c, sqlStr, mid, aid, tp, coinCount, now, now, coinCount, now); err != nil {
+		log.Error("dao.coinDB.Exec(%s,%d,%d,%d,%v,%v,%d,%v) error(%v)", sqlStr, mid, aid, coinCount, now, now, coinCount, now, err)
+	}
+	return
+}
+
+// ClearCoinCount clear settle.
+func (dao *Dao) ClearCoinCount(c context.Context, tableID int64, now time.Time) (err error) {
+	sqlStr := fmt.Sprintf(_clearCoinCount, tableID)
+	if _, err = dao.coinDB.Exec(c, sqlStr, now); err != nil {
+		log.Error("dao.coinDB.Exec(%s, %v) error(%v)", sqlStr, now, err)
+	}
+	return
+}
+
+// TotalCoins get total coins.
+func (dao *Dao) TotalCoins(c context.Context, id int, start, end time.Time) (coins map[int64]int64, err error) {
+	rows, err := dao.getTotalCoinsStmt[id].Query(c, start.Unix(), end.Unix())
+	if err != nil {
+		log.Error("dao.getTotalCoinsStmt[%d].Query(%v, %v) error(%v)", id, start, end, err)
+		return
+	}
+	coins = make(map[int64]int64)
+	defer rows.Close()
+	for rows.Next() {
+		var aid, tp, count int64
+		if err = rows.Scan(&aid, &tp, &count); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		coins[hashField(aid, tp)] = count
+	}
+	return
+}

app/job/main/coin/dao/mysql_test.go

@@ -0,0 +1,57 @@
+package dao
+
+import (
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestSettlePeriod(t *testing.T) {
+	Convey("SettlePeriod", t, func() {
+		_, err := d.SettlePeriod(ctx, 2)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestHitSettlePeriod(t *testing.T) {
+	Convey("HitSettlePeriod", t, func() {
+		_, err := d.HitSettlePeriod(ctx, time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpdateSettle(t *testing.T) {
+	Convey("UpdateSettle", t, func() {
+		err := d.UpdateSettle(ctx, 1, 1, 10, time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpdateCoinCount(t *testing.T) {
+	Convey("UpdateCoinCount", t, func() {
+		err := d.UpdateCoinCount(ctx, 1, 1, 10, 1, time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestEvery10000(t *testing.T) {
+	Convey("Every10000", t, func() {
+		_, _, err := d.Every10000(ctx, 1, 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestUpsertSettle(t *testing.T) {
+	Convey("UpsertSettle", t, func() {
+		err := d.UpsertSettle(ctx, 1, 1, 1, 1, 1, time.Now())
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestTotalCoins(t *testing.T) {
+	Convey("TotalCoins", t, func() {
+		_, err := d.TotalCoins(ctx, 1, time.Now(), time.Now())
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/coin/dao/redis.go

@@ -0,0 +1,62 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_loginShard    = 10000
+	_loginedPrefix = "logined_%d_%d"
+)
+
+func loginKey(mid, day int64) string {
+	return fmt.Sprintf(_loginedPrefix, day, mid/_loginShard)
+}
+
+// SetLogin set user logined,
+func (d *Dao) SetLogin(c context.Context, mid, day int64) (err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if err = conn.Send("SETBIT", loginKey(mid, day), mid%_loginShard, 1); err != nil {
+		PromError("redis:SetLogin")
+		log.Error("d.SetLogin(%v,%v) redis: err: %v", mid, day, err)
+		return
+	}
+	if err = conn.Send("EXPIRE", loginKey(mid, day), d.loginExpire); err != nil {
+		PromError("redis:SetLogin")
+		log.Error("d.SetLogin(%v,%v) redis: err: %v", mid, day, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("d.SetLogin(%v,%v) redis: err: %v", mid, day, err)
+		PromError("redis:SetLogin")
+		return
+	}
+	if _, err = redis.Bool(conn.Receive()); err != nil {
+		log.Error("d.SetLogin(%v,%v) redis: err: %v", mid, day, err)
+		PromError("redis:SetLogin")
+		return
+	}
+	if _, err = conn.Receive(); err != nil {
+		log.Error("d.SetLogin(%v,%v) redis: err: %v", mid, day, err)
+		PromError("redis:SetLogin")
+	}
+	return
+}
+
+// Logined check if user logined.
+func (d *Dao) Logined(c context.Context, mid, day int64) (b bool, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if b, err = redis.Bool(conn.Do("GETBIT", loginKey(mid, day), mid%_loginShard)); err == redis.ErrNil {
+		err = nil
+	}
+	if err != nil {
+		log.Error("d.Logined(%v,%v) redis err: %+v", mid, day, err)
+	}
+	return
+}

app/job/main/coin/dao/redis_test.go

@@ -0,0 +1,22 @@
+package dao
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestSetLogin(t *testing.T) {
+	Convey("TestSetLogin", t, func() {
+		err := d.SetLogin(ctx, 1, 2)
+		if err != nil {
+			t.Errorf("dedeCoins err(%v)", err)
+		}
+		b, _ := d.Logined(ctx, 1, 2)
+		if !b {
+			t.Errorf("Logined should be true but get %v", b)
+		}
+		b, _ = d.Logined(ctx, 1, 3)
+		So(b, ShouldNotBeNil)
+	})
+}