Create & Init Project...

app/job/main/coin/service/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "award_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/coin/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "award.go",
+        "exp.go",
+        "service.go",
+        "settle.go",
+    ],
+    importpath = "go-common/app/job/main/coin/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/coin/conf:go_default_library",
+        "//app/job/main/coin/dao:go_default_library",
+        "//app/job/main/coin/model:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//app/service/main/archive/api:go_default_library",
+        "//app/service/main/archive/api/gorpc:go_default_library",
+        "//app/service/main/archive/model/archive:go_default_library",
+        "//app/service/main/coin/api/gorpc:go_default_library",
+        "//app/service/main/coin/model:go_default_library",
+        "//app/service/main/member/api:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/queue/databus/databusutil:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/coin/service/award.go

@@ -0,0 +1,145 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/coin/dao"
+	"go-common/app/job/main/coin/model"
+	accmdl "go-common/app/service/main/account/api"
+	coinmdl "go-common/app/service/main/coin/model"
+	mmdl "go-common/app/service/main/member/api"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/stat/prom"
+)
+
+var _passportLog = 53
+
+func (s *Service) awardDo(ms []interface{}) {
+	for _, m := range ms {
+		mu, ok := m.(*model.LoginLog)
+		if !ok {
+			continue
+		}
+		if mu.Business != _passportLog {
+			continue
+		}
+		prom.BusinessInfoCount.Incr("award-event")
+		err := s.award(context.TODO(), mu.Mid, mu.Timestamp, mu.IP)
+		if err != nil {
+			log.Error("s.award mid %v err %v", mu.Mid, err)
+		}
+		log.Info("conmsumer login log, mid:%v,time %d, ip: %s err: %v", mu.Mid, mu.Timestamp, mu.IP, err)
+	}
+}
+
+func split(msg *databus.Message, data interface{}) int {
+	t, ok := data.(*model.LoginLog)
+	if !ok {
+		return 0
+	}
+	return int(t.Mid)
+}
+
+func newMsg(msg *databus.Message) (res interface{}, err error) {
+	loginlog := new(model.LoginLog)
+	if err = json.Unmarshal(msg.Value, &loginlog); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+		dao.PromError("loginlog:Unmarshal")
+		return
+	}
+	var t time.Time
+	if t, err = time.ParseInLocation("2006-01-02 15:04:05", loginlog.CTime, time.Local); err != nil {
+		log.Error("time.parse(%s) error(%v)", msg.Value, err)
+		dao.PromError("loginlog:Timeparse")
+		return
+	}
+	loginlog.Timestamp = t.Unix()
+	loginlog.RawData = string(msg.Value)
+	res = loginlog
+	return
+}
+
+func newExpMsg(msg *databus.Message) (res interface{}, err error) {
+	loginlog := new(model.LoginLog)
+	explog := new(model.AddExp)
+	if err = json.Unmarshal(msg.Value, &explog); err != nil {
+		log.Error("newExpMsg json.Unmarshal(%s) error(%v)", msg.Value, err)
+		dao.PromError("loginlog:ExpUnmarshal")
+		return
+	}
+	loginlog.Mid = explog.Mid
+	loginlog.Timestamp = explog.Ts
+	loginlog.IP = explog.IP
+	loginlog.RawData = string(msg.Value)
+	res = loginlog
+	return
+}
+
+func (s *Service) award(c context.Context, mid, ts int64, ip string) (err error) {
+	if !s.c.CoinJob.Start || ts < s.c.CoinJob.StartTime {
+		return
+	}
+	if (mid == 0) || (ts == 0) || (ip == "") {
+		return
+	}
+	day := int64(time.Unix(ts, 0).Day())
+	var login bool
+	for {
+		if login, err = s.coinDao.Logined(c, mid, day); err == nil {
+			break
+		}
+		dao.PromError("redis-logined-retry")
+		time.Sleep(time.Millisecond * 500)
+	}
+	if login {
+		return
+	}
+	// false mean first login,
+	var base *mmdl.BaseInfoReply
+	base, err = s.memRPC.Base(c, &mmdl.MemberMidReq{Mid: mid})
+	if err != nil {
+		if err == ecode.MemberNotExist {
+			return
+		}
+		log.Errorv(c, log.KV("log", "memRPC"), log.KV("err", err), log.KV("mid", mid))
+		dao.PromError("登录奖励member")
+		return
+	}
+	if (base != nil) && (base.Rank == 5000) {
+		log.Infov(c, log.KV("log", "add coin but user not member"), log.KV("mid", mid))
+		return
+	}
+	var profile *accmdl.ProfileReply
+	profile, _ = s.profile(c, mid)
+	if (profile != nil) && (profile.Profile != nil) && (profile.Profile.TelStatus == 0) {
+		log.Infov(c, log.KV("log", "login award failed. no telphone"), log.KV("mid", mid))
+		return
+	}
+	if _, err = s.coinRPC.ModifyCoin(c, &coinmdl.ArgModifyCoin{Mid: mid, Count: 1, Reason: "登录奖励", IP: ip, CheckZero: 1}); err != nil {
+		dao.PromError("登录奖励RPC")
+		return
+	}
+	for {
+		if err = s.coinDao.SetLogin(c, mid, day); err == nil {
+			break
+		}
+		dao.PromError("登录奖励setLogin-retry")
+		time.Sleep(time.Millisecond * 500)
+	}
+	prom.BusinessInfoCount.Incr("award-event-success")
+	log.Info("add coin success mid: %+v", mid)
+	return
+}
+
+func (s *Service) profile(c context.Context, mid int64) (res *accmdl.ProfileReply, err error) {
+	arg := &accmdl.MidReq{Mid: mid}
+	if res, err = s.accRPC.Profile3(c, arg); err != nil {
+		dao.PromError("award:Profile3")
+		log.Errorv(c, log.KV("log", "Profile3"), log.KV("err", err))
+	}
+	return
+}

app/job/main/coin/service/award_test.go

@@ -0,0 +1,16 @@
+package service
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAward(t *testing.T) {
+	Convey("award", t, func() {
+		err := s.award(context.TODO(), 1, time.Now().Unix(), "127.0.0.1")
+		So(err, ShouldBeNil)
+	})
+}

app/job/main/coin/service/exp.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/main/coin/dao"
+	coinmdl "go-common/app/service/main/coin/model"
+	memmdl "go-common/app/service/main/member/api"
+	"go-common/library/log"
+)
+
+func (s *Service) addExp(c context.Context, mid int64, count float64, reason, ip string) (err error) {
+	argExp := &memmdl.AddExpReq{
+		Mid:     mid,
+		Count:   count,
+		Operate: "coin",
+		Reason:  reason,
+		Ip:      ip,
+	}
+	if count <= 0 {
+		log.Errorv(c, log.KV("log", "add exp count < 0"), log.KV("mid", mid), log.KV("err", err), log.KV("reason", reason), log.KV("count", count))
+		dao.PromError("exp:addExp0")
+		return
+	}
+	if _, err = s.memRPC.UpdateExp(c, argExp); err != nil {
+		log.Errorv(c, log.KV("log", "s.coinDao.IncrExp()"), log.KV("mid", mid), log.KV("err", err), log.KV("reason", reason), log.KV("count", count))
+		dao.PromError("exp:addExp")
+		return
+	}
+	return
+}
+
+func (s *Service) addCoinExp(c context.Context, mid, tp, number int64, ip string) (err error) {
+	arg := &coinmdl.ArgAddUserCoinExp{Mid: mid, Business: tp, Number: number, RealIP: ip}
+	if err = s.coinRPC.AddUserCoinExp(c, arg); err != nil {
+		log.Errorv(c, log.KV("log", "AddUserCoinExp"), log.KV("err", err))
+		dao.PromError("exp:addCoinExp")
+	}
+	return
+}

app/job/main/coin/service/service.go

@@ -0,0 +1,163 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/coin/conf"
+	"go-common/app/job/main/coin/dao"
+	"go-common/app/job/main/coin/model"
+	accrpc "go-common/app/service/main/account/api"
+	arcrpc "go-common/app/service/main/archive/api/gorpc"
+	coinrpc "go-common/app/service/main/coin/api/gorpc"
+	coinmdl "go-common/app/service/main/coin/model"
+	memrpc "go-common/app/service/main/member/api"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	"go-common/library/queue/databus/databusutil"
+)
+
+// Service coin job service.
+type Service struct {
+	coinDao  *dao.Dao
+	c        *conf.Config
+	waiter   *sync.WaitGroup
+	accRPC   accrpc.AccountClient
+	memRPC   memrpc.MemberClient
+	arcRPC   *arcrpc.Service2
+	coinRPC  *coinrpc.Service
+	databus  *databus.Databus
+	group    *databusutil.Group
+	expGroup *databusutil.Group
+}
+
+// New new and return service.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		coinDao: dao.New(c),
+		c:       c,
+		waiter:  new(sync.WaitGroup),
+		arcRPC:  arcrpc.New2(c.ArchiveRPC),
+		coinRPC: coinrpc.New(c.CoinRPC),
+	}
+	var err error
+	if s.memRPC, err = memrpc.NewClient(c.MemRPC); err != nil {
+		panic(err)
+	}
+	if s.accRPC, err = accrpc.NewClient(c.AccountRPC); err != nil {
+		panic(err)
+	}
+	s.databus = databus.New(c.Databus)
+	g := databusutil.NewGroup(c.Databusutil, databus.New(c.LoginDatabus).Messages())
+	g.New = newMsg
+	g.Split = split
+	g.Do = s.awardDo
+	g.Start()
+	s.group = g
+	eg := databusutil.NewGroup(c.Databusutil, databus.New(c.ExpDatabus).Messages())
+	eg.New = newExpMsg
+	eg.Split = split
+	eg.Do = s.awardDo
+	eg.Start()
+	s.expGroup = eg
+
+	s.waiter.Add(1)
+	go s.consumeproc()
+	go s.settleproc()
+	return
+}
+
+func (s *Service) consumeproc() {
+	defer s.waiter.Done()
+	var (
+		msg    *databus.Message
+		err    error
+		ok     bool
+		period *model.CoinSettlePeriod
+		ctx    = context.TODO()
+	)
+	for {
+		if msg, ok = <-s.databus.Messages(); !ok {
+			log.Error("s.databus.Message err(%v)", err)
+			return
+		}
+		r := &coinmdl.Record{}
+		if err = json.Unmarshal([]byte(msg.Value), r); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", msg.Value, err)
+			dao.PromError("msg:JSON")
+			continue
+		}
+		var ip = r.IPV6
+		for i := 0; i < 3; i++ {
+			if err = s.addCoinExp(ctx, r.Mid, r.AvType, r.Multiply, ip); err != nil {
+				time.Sleep(time.Millisecond * 50)
+				continue
+			}
+			break
+		}
+		if err != nil {
+			log.Errorv(ctx, log.KV("log", "fix: addCoinExp"), log.KV("mid", r.Mid), log.KV("type", r.AvType), log.KV("num", r.Multiply), log.KV("ip", ip))
+			continue
+		}
+		if err = s.updateAddCoin(ctx, r); err != nil {
+			continue
+		}
+		at := time.Unix(r.Timestamp, 0)
+		if period, err = s.coinDao.HitSettlePeriod(ctx, at); err != nil {
+			log.Errorv(ctx, log.KV("log", "s.coinDao.HitCoinPeriod"), log.KV("record", r), log.KV("err", err))
+			dao.PromError("service:HitSettlePeriod")
+			continue
+		}
+		for i := 0; ; i++ {
+			if err = s.coinDao.UpsertSettle(ctx, period.ID, r.Up, r.Aid, r.AvType, r.Multiply, time.Now()); err != nil {
+				log.Error("s.coinDao.UpsertCoinSettle(%d, %d, %d) error(%v)", r.Up, r.Aid, r.Multiply)
+				dao.PromError("service:UpsertSettle")
+				i++
+				if i > 5 {
+					// if env.DeployEnv == env.DeployEnvProd {
+					// 	s.moni.Sms(ctx, s.c.Sms.Phone, s.c.Sms.Token, "coin-job upsetSettle fail for 5 time")
+					// }
+					break
+				}
+				continue
+			}
+			break
+		}
+		log.Info("key: %s,partion:%d,offset:%d success %s", msg.Key, msg.Partition, msg.Offset, msg.Value)
+		err = msg.Commit()
+		if err != nil {
+			log.Error("msg.Commit partition:%d offset:%d err %v", msg.Partition, msg.Offset, err)
+			dao.PromError("service:msgCommit")
+		}
+	}
+}
+
+// Close close service.
+func (s *Service) Close() {
+	s.group.Close()
+	s.expGroup.Close()
+	s.databus.Close()
+}
+
+// Wait wait routine unitl all close.
+func (s *Service) Wait() {
+	s.waiter.Wait()
+}
+
+// Ping check service health.
+func (s *Service) Ping(c context.Context) error {
+	return s.coinDao.Ping(c)
+}
+
+func (s *Service) updateAddCoin(c context.Context, record *coinmdl.Record) (err error) {
+	if record == nil {
+		return
+	}
+	if err = s.coinRPC.UpdateAddCoin(c, record); err != nil {
+		log.Errorv(c, log.KV("log", "UpdateAddCoin"), log.KV("mid", record.Mid), log.KV("record", record))
+		dao.PromError("service:updateAddCoin")
+	}
+	return
+}

app/job/main/coin/service/service_test.go

@@ -0,0 +1,36 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/job/main/coin/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/coin-job-test.toml")
+	flag.Set("conf", dir)
+	if err := conf.Init(); err != nil {
+		panic("conf.Init() error")
+	}
+	s = New(conf.Conf)
+	time.Sleep(time.Second * 1)
+}
+
+func TestService(t *testing.T) {
+	Convey("mock", t, func() {
+		s.Close()
+		s.Redo(0)
+		s.Ping(context.TODO())
+		s.Wait()
+	})
+}

app/job/main/coin/service/settle.go

@@ -0,0 +1,208 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/coin/dao"
+	"go-common/app/job/main/coin/model"
+	"go-common/app/service/main/archive/api"
+	comarcmdl "go-common/app/service/main/archive/model/archive"
+	"go-common/library/log"
+)
+
+var lockRedo sync.Mutex
+var lockSettle sync.Mutex
+
+const (
+	_expPerCoin = 1
+)
+
+func (s *Service) settleproc() {
+	for {
+		now := time.Now()
+		tmp := now.AddDate(0, 1, 0)
+		// sleep one month
+		time.Sleep(time.Date(tmp.Year(), tmp.Month(), 1, 0, 0, 0, 0, tmp.Location()).Sub(now))
+		// settle coin
+		s.Settle(0)
+	}
+}
+
+// Redo redo settle.
+func (s *Service) Redo(tableID int64) (err error) {
+	lockRedo.Lock()
+	defer lockRedo.Unlock()
+	ctx := context.TODO()
+	// if tableID has been seted than run test
+	if tableID != 0 {
+		return s.redo(ctx, tableID)
+	}
+	now := time.Now()
+	day := now.Day()
+	if day <= 25 {
+		err = fmt.Errorf("redo must after every 25th of each month, today is %d ", day)
+		return
+	}
+	period, err := s.coinDao.HitSettlePeriod(ctx, now)
+	if err != nil {
+		log.Error("job s.coinDao.HitCoinPeriod error(%v)", err)
+		return
+	}
+	return s.redo(ctx, period.ID-1)
+}
+
+func (s *Service) redo(ctx context.Context, tableID int64) (err error) {
+	period, err := s.coinDao.SettlePeriod(ctx, tableID)
+	if err != nil {
+		log.Error("s.coinDao.SettlePeriod(%d) error(%v)", tableID, err)
+		return
+	}
+	if err = s.coinDao.ClearCoinCount(ctx, tableID, time.Now()); err != nil {
+		log.Error("s.coinDao.ClearCoinCount(%d) error(%v)", tableID, err)
+		return
+	}
+	startTime := time.Date(period.FromYear, time.Month(period.FromMonth), period.FromDay, 0, 0, 0, 0, time.Local)
+	endTime := time.Date(period.ToYear, time.Month(period.ToMonth), period.ToDay, 0, 0, 0, 0, time.Local)
+	var (
+		coins   map[int64]int64
+		aids    []int64
+		aidMids map[int64]int64
+		argAids *comarcmdl.ArgAids2
+		arcs    map[int64]*api.Arc
+		peer    = 100
+	)
+	for i := 0; i < dao.SHARDING; i++ {
+		if coins, err = s.coinDao.TotalCoins(ctx, i, startTime, endTime); err != nil {
+			log.Error("s.coinDao.TotalCoins(%d, %v, %v) error(%v)", i, startTime, endTime, err)
+			return
+		}
+		aids = make([]int64, 0, len(coins))
+		for aid := range coins {
+			if aid%1000 == 1 {
+				aids = append(aids, aid/1000)
+			}
+		}
+		var (
+			length = len(aids)
+			cop    = length / peer
+			mod    = length % peer
+		)
+		aidMids = make(map[int64]int64, length)
+		for i := 0; i < cop; i++ {
+			argAids = &comarcmdl.ArgAids2{
+				Aids: aids[i*peer : peer*(i+1)],
+			}
+			if arcs, err = s.arcRPC.Archives3(ctx, argAids); err != nil {
+				log.Error("s.arcRPC.Archives2 error(%v)", err)
+				return
+			}
+			for aid, arc := range arcs {
+				aidMids[aid] = arc.Author.Mid
+			}
+		}
+		if mod != 0 {
+			argAids = &comarcmdl.ArgAids2{
+				Aids: aids[cop*peer:],
+			}
+			if arcs, err = s.arcRPC.Archives3(ctx, argAids); err != nil {
+				log.Error("s.arcRPC.Archives2 error(%v)", err)
+				return
+			}
+			for aid, arc := range arcs {
+				aidMids[aid] = arc.Author.Mid
+			}
+		}
+		for aid, count := range coins {
+			if mid, ok := aidMids[aid/1000]; ok {
+				if err = s.coinDao.UpsertSettle(ctx, tableID, mid, aid/1000, aid%1000, count, time.Now()); err != nil {
+					log.Error("s.coinDao.UpdateCoinCount(%d, %d, %d) error(%v)", tableID, aid, count, err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+// Settle do settle by table.
+func (s *Service) Settle(tableID int64) (err error) {
+	lockSettle.Lock()
+	defer lockSettle.Unlock()
+	ctx := context.TODO()
+	// if tableID has been seted than run test
+	if tableID != 0 {
+		return s.settle(ctx, tableID)
+	}
+	period, err := s.coinDao.HitSettlePeriod(ctx, time.Now())
+	if err != nil {
+		log.Error("job s.coinDao.HitCoinPeriod error(%v)", err)
+		return
+	}
+	return s.settle(ctx, period.ID-1)
+}
+
+// job exec at every 1th of each month
+func (s *Service) settle(ctx context.Context, tableID int64) (err error) {
+	var (
+		settles []*model.CoinSettle
+	)
+	period, err := s.coinDao.SettlePeriod(ctx, tableID)
+	if err != nil {
+		log.Error("s.coinDao.SettlePeriod(%d) error(%v)", tableID, err)
+		return
+	}
+	var i, maxid int64
+	for {
+		if settles, maxid, err = s.coinDao.Every10000(ctx, tableID, i); err != nil {
+			log.Error("settle: job s.coinDao.Every10000(%d) error(%v)", tableID, err)
+			time.Sleep(time.Second)
+			continue
+		}
+		if maxid == i {
+			log.Info("settle: maxid %d", maxid)
+			fmt.Println("maxid", maxid)
+			return
+		}
+		for _, settle := range settles {
+			if settle.State == 1 || settle.Mid == 0 {
+				continue
+			}
+			settle.ExpTotal = settle.CoinCount*_expPerCoin - settle.ExpSub
+			if settle.ExpTotal <= 0 {
+				log.Errorv(ctx, log.KV("log", "settle: ExpTotal err"), log.KV("count", settle.ExpTotal))
+				continue
+			}
+			// add exp
+			var reason string
+			switch settle.AvType {
+			case 1:
+				reason = fmt.Sprintf("%d.%d-%d.%d视频av%d投币获得奖励", period.FromMonth, period.FromDay, period.ToMonth, period.ToDay, settle.Aid)
+			case 2:
+				reason = fmt.Sprintf("%d.%d-%d.%d文章cv%d投币获得奖励", period.FromMonth, period.FromDay, period.ToMonth, period.ToDay, settle.Aid)
+			case 3:
+				reason = fmt.Sprintf("%d.%d-%d.%d音乐mv%d投币获得奖励", period.FromMonth, period.FromDay, period.ToMonth, period.ToDay, settle.Aid)
+			}
+			for i := 0; i < 3; i++ {
+				if err = s.addExp(ctx, settle.Mid, float64(settle.ExpTotal), reason, ""); err != nil {
+					time.Sleep(time.Second)
+				} else {
+					break
+				}
+			}
+			if err != nil {
+				log.Errorv(ctx, log.KV("log", "s.accRPC.AddExp2"), log.KV("mid", settle.Mid), log.KV("err", err))
+				continue
+			}
+			if err = s.coinDao.UpdateSettle(ctx, tableID, settle.ID, settle.ExpTotal, time.Now()); err != nil {
+				log.Error("settle: s.coinDao.UpdateState(%d, %d, %d) error(%v)", tableID, settle.ID, settle.ExpTotal, err)
+				continue
+			}
+			log.Info("settle: aid(%d) add exp(%d) success", settle.Aid, settle.ExpTotal)
+		}
+		fmt.Printf("settle:%d ,len(settle) %d\n", i, len(settles))
+		i = maxid
+	}
+}