Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,60 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"active_test.go",
"retry_test.go",
"service_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/app-wall/conf:go_default_library",
"//app/job/main/app-wall/model/offer:go_default_library",
"//vendor/github.com/bsm/sarama-cluster:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"active.go",
"retry.go",
"service.go",
],
importpath = "go-common/app/job/main/app-wall/service/offer",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/app-wall/conf:go_default_library",
"//app/job/main/app-wall/dao/offer:go_default_library",
"//app/job/main/app-wall/model:go_default_library",
"//app/job/main/app-wall/model/offer:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/Shopify/sarama:go_default_library",
"//vendor/github.com/bsm/sarama-cluster:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,113 @@
package offer
import (
"context"
"fmt"
"strconv"
"strings"
"go-common/app/job/main/app-wall/model"
"go-common/app/job/main/app-wall/model/offer"
"go-common/library/log"
)
func (s *Service) activeConsumer() {
defer s.waiter.Done()
LOOP:
for {
select {
case err := <-s.consumer.Errors():
log.Error("group(%s) topic(%s) addr(%s) catch error(%+v)", s.c.Consumer.Group, s.c.Consumer.Topic, s.c.Consumer.Brokers, err)
continue
case notify := <-s.consumer.Notifications():
log.Info("notification(%v)", notify)
continue
case msg, ok := <-s.consumer.Messages():
if !ok {
log.Error("active consumer exit!")
break LOOP
}
s.consumer.MarkOffset(msg, "")
active, err := s.checkMsgIllegal(msg.Value)
if err != nil {
log.Error("s.checkMsgIllegal(%s) error(%v)", msg.Value, err)
continue
}
if active == nil {
continue
}
s.activeChan <- active
}
}
}
func (s *Service) checkMsgIllegal(msg []byte) (active *offer.ActiveMsg, err error) {
var (
msgs []string
pid int64
os string
androidid string
imei string
)
msgs = strings.Split(string(msg), "|")
if len(msgs) < 9 {
err = fmt.Errorf("active msg(%s) split len(%d)<9", msg, len(msgs))
return
}
if pid, err = strconv.ParseInt(msgs[8], 10, 64); err != nil {
return
}
if pid%10 == 3 {
os = model.TypeAndriod
if len(msgs) > 22 {
androidid = msgs[22]
}
if len(msgs) > 23 {
imei = msgs[23]
}
if imei == "" {
log.Warn("active msg(%s) imei(%s) is illegal", msg, imei)
} else {
log.Warn("active msg(%s) imei(%s) is legal", msg, imei)
}
if androidid == "" {
log.Warn("active msg(%s) androidid(%s) is illegal", msg, androidid)
}
if androidid == "" && imei == "" {
err = fmt.Errorf("active msg(%s) androidid(%s) and imei(%s) is illegal", msg, androidid, imei)
return
}
} else {
err = fmt.Errorf("active msg(%s) pid(%d) platform not android", msg, pid)
return
}
active = &offer.ActiveMsg{OS: os, IMEI: imei, Androidid: androidid, Mac: ""}
return
}
func (s *Service) activeproc() {
defer s.waiter.Done()
for {
msg, ok := <-s.activeChan
if !ok {
log.Error("active chan id closed")
break
}
s.active(msg)
}
}
func (s *Service) active(msg *offer.ActiveMsg) {
var err error
c := context.TODO()
if err = retry(func() (err error) {
return s.dao.Active(c, msg.OS, msg.IMEI, msg.Androidid, msg.Mac, "")
}, _upActiveRetry, _sleep); err != nil {
log.Error("%+v", err)
if err = s.syncRetry(c, offer.ActionActive, msg.OS, msg.IMEI, msg.Androidid, msg.Mac); err != nil {
log.Error("%+v", err)
}
return
}
log.Info("active device os(%s) imei(%s) androidid(%s) mac(%s) success", msg.OS, msg.IMEI, msg.Androidid, msg.Mac)
}

View File

@@ -0,0 +1,81 @@
package offer
import (
"reflect"
"testing"
"go-common/app/job/main/app-wall/model/offer"
)
func TestService_activeConsumer(t *testing.T) {
tests := []struct {
name string
s *Service
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.activeConsumer()
})
}
}
func TestService_checkMsgIllegal(t *testing.T) {
type args struct {
msg []byte
}
tests := []struct {
name string
s *Service
args args
wantActive *offer.ActiveMsg
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotActive, err := tt.s.checkMsgIllegal(tt.args.msg)
if (err != nil) != tt.wantErr {
t.Errorf("Service.checkMsgIllegal() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotActive, tt.wantActive) {
t.Errorf("Service.checkMsgIllegal() = %v, want %v", gotActive, tt.wantActive)
}
})
}
}
func TestService_activeproc(t *testing.T) {
tests := []struct {
name string
s *Service
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.activeproc()
})
}
}
func TestService_active(t *testing.T) {
type args struct {
msg *offer.ActiveMsg
}
tests := []struct {
name string
s *Service
args args
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.active(tt.args.msg)
})
}
}

View File

@@ -0,0 +1,72 @@
package offer
import (
"context"
"encoding/json"
"time"
"go-common/app/job/main/app-wall/model/offer"
"go-common/library/log"
)
const (
_upActiveRetry = 5
_sleep = 100 * time.Millisecond
)
func (s *Service) retryproc() {
defer s.waiter.Done()
var (
bs []byte
err error
)
c := context.TODO()
msg := &offer.Retry{}
for {
if s.closed {
break
}
if bs, err = s.dao.PopFail(c); err != nil {
time.Sleep(5 * time.Second)
continue
}
if len(bs) == 0 {
time.Sleep(5 * time.Second)
continue
}
if err = json.Unmarshal(bs, msg); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", bs, err)
continue
}
log.Info("retry action(%s) data(%s)", msg.Action, bs)
switch msg.Action {
case offer.ActionActive:
if msg.Data != nil {
if err = retry(func() (err error) {
return s.dao.Active(c, msg.Data.OS, msg.Data.IMEI, msg.Data.Androidid, msg.Data.Mac, "")
}, _upActiveRetry, _sleep); err != nil {
log.Error("%+v", err)
if err = s.syncRetry(c, offer.ActionActive, msg.Data.OS, msg.Data.IMEI, msg.Data.Androidid, msg.Data.Mac); err != nil {
log.Error("%+v", err)
}
return
}
}
}
}
}
func retry(callback func() error, retry int, sleep time.Duration) (err error) {
for i := 0; i < retry; i++ {
if err = callback(); err == nil {
return
}
time.Sleep(sleep)
}
return
}
func (s *Service) syncRetry(c context.Context, action, os, imei, androidid, mac string) (err error) {
retry := &offer.Retry{Action: action, Data: &offer.Data{OS: os, IMEI: imei, Androidid: androidid, Mac: mac}}
return s.dao.PushFail(c, retry)
}

View File

@@ -0,0 +1,69 @@
package offer
import (
"context"
"testing"
"time"
)
func TestService_retryproc(t *testing.T) {
tests := []struct {
name string
s *Service
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.retryproc()
})
}
}
func Test_retry(t *testing.T) {
type args struct {
callback func() error
retry int
sleep time.Duration
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := retry(tt.args.callback, tt.args.retry, tt.args.sleep); (err != nil) != tt.wantErr {
t.Errorf("retry() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestService_syncRetry(t *testing.T) {
type args struct {
c context.Context
action string
os string
imei string
androidid string
mac string
}
tests := []struct {
name string
s *Service
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := tt.s.syncRetry(tt.args.c, tt.args.action, tt.args.os, tt.args.imei, tt.args.androidid, tt.args.mac); (err != nil) != tt.wantErr {
t.Errorf("Service.syncRetry() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -0,0 +1,88 @@
package offer
import (
"context"
"strings"
"sync"
"time"
"go-common/app/job/main/app-wall/conf"
offerDao "go-common/app/job/main/app-wall/dao/offer"
"go-common/app/job/main/app-wall/model/offer"
"go-common/library/log"
"github.com/Shopify/sarama"
cluster "github.com/bsm/sarama-cluster"
)
// Service struct
type Service struct {
c *conf.Config
dao *offerDao.Dao
consumer *cluster.Consumer
activeChan chan *offer.ActiveMsg
closed bool
waiter sync.WaitGroup
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: offerDao.New(c),
activeChan: make(chan *offer.ActiveMsg, 10240),
closed: false,
}
var err error
if s.consumer, err = s.NewConsumer(); err != nil {
log.Error("%+v", err)
panic(err)
}
s.waiter.Add(1)
go s.activeConsumer()
s.waiter.Add(1)
go s.activeproc()
// retry consumer
for i := 0; i < 4; i++ {
s.waiter.Add(1)
go s.retryproc()
}
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() {
s.closed = true
s.consumer.Close()
s.waiter.Wait()
log.Info("app-wall-job closed.")
}
// NewConsumer new cluster consumer.
func (s *Service) NewConsumer() (*cluster.Consumer, error) {
// cluster config
cfg := cluster.NewConfig()
// NOTE cluster auto commit offset interval
cfg.Consumer.Offsets.CommitInterval = time.Second * 1
// NOTE set fetch.wait.max.ms
cfg.Consumer.MaxWaitTime = time.Millisecond * 100
// NOTE errors that occur during offset management,if enabled, c.Errors channel must be read
cfg.Consumer.Return.Errors = true
// NOTE notifications that occur during consumer, if enabled, c.Notifications channel must be read
cfg.Group.Return.Notifications = true
// The initial offset to use if no offset was previously committed.
// default: OffsetOldest
if strings.ToLower(s.c.Consumer.Offset) != "new" {
cfg.Consumer.Offsets.Initial = sarama.OffsetOldest
} else {
cfg.Consumer.Offsets.Initial = sarama.OffsetNewest
}
// new cluster consumer
log.Info("s.c.Consumer.Brokers: %v", s.c.Consumer.Brokers)
return cluster.NewConsumer(s.c.Consumer.Brokers, s.c.Consumer.Group, []string{s.c.Consumer.Topic}, cfg)
}

View File

@@ -0,0 +1,98 @@
package offer
import (
"context"
"flag"
"path/filepath"
"reflect"
"testing"
"go-common/app/job/main/app-wall/conf"
cluster "github.com/bsm/sarama-cluster"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
)
func init() {
dir, _ := filepath.Abs("../../cmd/app-wall-job-test.toml")
flag.Set("conf", dir)
conf.Init()
s = New(conf.Conf)
}
func TestNew(t *testing.T) {
type args struct {
c *conf.Config
}
tests := []struct {
name string
args args
wantS *Service
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotS := New(tt.args.c); !reflect.DeepEqual(gotS, tt.wantS) {
t.Errorf("New() = %v, want %v", gotS, tt.wantS)
}
})
}
}
func TestService_Ping(t *testing.T) {
type args struct {
c context.Context
}
tests := []struct {
name string
s *Service
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := tt.s.Ping(tt.args.c); (err != nil) != tt.wantErr {
t.Errorf("Service.Ping() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestService_Close(t *testing.T) {
tests := []struct {
name string
s *Service
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.s.Close()
})
}
}
func TestService_NewConsumer(t *testing.T) {
tests := []struct {
name string
s *Service
want *cluster.Consumer
wantErr error
}{
// TODO: Add test cases.
}
for _, tt := range tests {
Convey(tt.name, t, func() {
got, err := tt.s.NewConsumer()
So(err, ShouldEqual, tt.wantErr)
So(got, ShouldResemble, tt.want)
})
}
}

View File

@@ -0,0 +1,59 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = ["unicom_test.go"],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/job/main/app-wall/conf:go_default_library",
"//app/job/main/app-wall/model/unicom:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"infoc.go",
"service.go",
"unicom.go",
],
importpath = "go-common/app/job/main/app-wall/service/unicom",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/main/app-wall/conf:go_default_library",
"//app/job/main/app-wall/dao/seq:go_default_library",
"//app/job/main/app-wall/dao/unicom:go_default_library",
"//app/job/main/app-wall/model/unicom:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/log/infoc:go_default_library",
"//library/queue/databus:go_default_library",
"//library/queue/databus/report:go_default_library",
"//library/stat/prom:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,88 @@
package unicom
import (
"strconv"
"time"
"go-common/app/job/main/app-wall/conf"
"go-common/library/log"
binfoc "go-common/library/log/infoc"
)
type infoc struct {
usermob string
phone string
mid string
leve string
integral string
flow string
unicomtype string
now string
}
type packInfoc struct {
usermob string
phone string
mid string
requestNo string
packName string
packIntegral string
packType string
now string
}
// unicomInfoc unicom user infoc
func (s *Service) unicomInfoc(usermob string, phone, leve, integral, flow int, utype string, mid int64, now time.Time) {
select {
case s.logCh[mid%s.c.ChanDBNum] <- infoc{usermob, strconv.Itoa(phone), strconv.FormatInt(mid, 10),
strconv.Itoa(leve), strconv.Itoa(integral), strconv.Itoa(flow), utype, strconv.FormatInt(now.Unix(), 10)}:
default:
log.Warn("unicomInfoc log buffer is full")
}
}
func (s *Service) unicomInfocproc(i int64) {
var (
cliChan = s.logCh[i]
packinf = binfoc.New(conf.Conf.UnicomUserInfoc2)
)
for {
i, ok := <-cliChan
if !ok {
log.Warn("infoc proc exit")
return
}
switch v := i.(type) {
case infoc:
packinf.Info(v.now, "0", v.usermob, v.phone, v.mid, v.leve, v.integral, v.flow, v.unicomtype)
log.Info("unicomInfocproc log mid(%v) phone(%v)", v.mid, v.phone)
}
}
}
// unicomPackInfoc unicom pack infoc
func (s *Service) unicomPackInfoc(usermob, packName, orderNumber string, phone, packIntegral, packType int, mid int64, now time.Time) {
select {
case s.packCh <- packInfoc{usermob, strconv.Itoa(phone), strconv.FormatInt(mid, 10),
orderNumber, packName, strconv.Itoa(packIntegral), strconv.Itoa(packType), strconv.FormatInt(now.Unix(), 10)}:
default:
log.Warn("unicomPackInfoc log buffer is full")
}
}
func (s *Service) unicomPackInfocproc() {
var (
packinf = binfoc.New(conf.Conf.UnicomPackInfoc)
)
for {
i, ok := <-s.packCh
if !ok {
log.Warn("infoc proc exit")
return
}
switch v := i.(type) {
case packInfoc:
packinf.Info(v.now, "0", v.usermob, v.phone, v.mid, v.requestNo, v.packName, v.packIntegral, v.packType)
}
}
}

View File

@@ -0,0 +1,122 @@
package unicom
import (
"sync"
"time"
"go-common/app/job/main/app-wall/conf"
seqDao "go-common/app/job/main/app-wall/dao/seq"
unicomDao "go-common/app/job/main/app-wall/dao/unicom"
"go-common/app/job/main/app-wall/model/unicom"
"go-common/library/log"
"go-common/library/queue/databus"
"go-common/library/stat/prom"
)
type Service struct {
c *conf.Config
dao *unicomDao.Dao
seqdao *seqDao.Dao
clickSub *databus.Databus
closed bool
// waiter
waiter sync.WaitGroup
cliChan []chan *unicom.ClickMsg
dbcliChan []chan *unicom.UserBind
// infoc
logCh []chan interface{}
packCh chan interface{}
packLogCh chan interface{}
integralLogCh []chan interface{}
// prom
pHit *prom.Prom
pMiss *prom.Prom
// tick
tick time.Duration
lastmonth map[int]bool
}
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: unicomDao.New(c),
clickSub: databus.New(c.ReportDatabus),
seqdao: seqDao.New(c),
// infoc
packCh: make(chan interface{}, 1024),
packLogCh: make(chan interface{}, 1024),
// close
closed: false,
// prom
pHit: prom.CacheHit,
pMiss: prom.CacheMiss,
lastmonth: map[int]bool{},
// tick
tick: time.Duration(c.Tick),
}
for i := int64(0); i < s.c.ChanNum; i++ {
s.cliChan = append(s.cliChan, make(chan *unicom.ClickMsg, 300000))
}
for i := int64(0); i < s.c.ChanDBNum; i++ {
s.dbcliChan = append(s.dbcliChan, make(chan *unicom.UserBind, 1024))
s.integralLogCh = append(s.integralLogCh, make(chan interface{}, 1024))
s.logCh = append(s.logCh, make(chan interface{}, 1024))
}
for i := int64(0); i < s.c.ChanNum; i++ {
s.waiter.Add(1)
go s.cliChanProc(i)
}
for i := int64(0); i < s.c.ChanDBNum; i++ {
s.waiter.Add(1)
go s.unicomInfocproc(i)
go s.addUserIntegralLogproc(i)
}
for i := int64(0); i < s.c.ChanDBNum; i++ {
s.waiter.Add(1)
go s.dbcliChanProc(i)
}
s.waiter.Add(1)
go s.clickConsumer()
s.waiter.Add(1)
now := time.Now()
if s.c.Monthly {
// s.updatemonth(now)
s.upBindAll()
}
s.waiter.Add(1)
s.loadUnicomIPOrder(now)
s.loadUnicomFlow()
go s.loadproc()
s.waiter.Add(1)
go s.unicomPackInfocproc()
go s.addUserPackLogproc()
return
}
// Close Service
func (s *Service) Close() {
s.closed = true
time.Sleep(time.Second * 2)
s.clickSub.Close()
for i := 0; i < len(s.cliChan); i++ {
close(s.cliChan[i])
}
for i := 0; i < len(s.dbcliChan); i++ {
close(s.dbcliChan[i])
close(s.integralLogCh[i])
close(s.logCh[i])
}
s.waiter.Wait()
log.Info("app-wall-job unicom flow closed.")
}
// cacheproc load cache
func (s *Service) loadproc() {
for {
time.Sleep(s.tick)
now := time.Now()
s.loadUnicomFlow()
s.updatemonth(now)
s.loadUnicomIPOrder(now)
}
}

View File

@@ -0,0 +1,653 @@
package unicom
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"go-common/app/job/main/app-wall/model/unicom"
"go-common/library/cache/memcache"
"go-common/library/ecode"
"go-common/library/log"
"go-common/library/queue/databus/report"
)
const (
_initIPUnicomKey = "ipunicom_%v_%v"
)
func (s *Service) clickConsumer() {
defer s.waiter.Done()
msgs := s.clickSub.Messages()
for {
msg, ok := <-msgs
if !ok || s.closed {
log.Info("s.clickSub.Cloesd")
return
}
msg.Commit()
var (
sbs [][]byte
err error
)
if err = json.Unmarshal(msg.Value, &sbs); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", msg.Value, err)
continue
}
for _, bs := range sbs {
var (
err error
click *unicom.ClickMsg
)
if click, err = s.checkMsgIllegal(bs); err != nil {
log.Error("s.checkMsgIllegal(%s) error(%v)", strings.Replace(string(bs), "\001", "|", -1), err)
continue
}
log.Info("clickConsumer s.checkMsgIllegal(%s)", strings.Replace(string(bs), "\001", "|", -1))
s.cliChan[click.AID%s.c.ChanNum] <- click
}
}
}
func (s *Service) checkMsgIllegal(msg []byte) (click *unicom.ClickMsg, err error) {
var (
aid int64
clickMsg []string
plat int64
bvID string
mid int64
lv int64
ctime int64
stime int64
epid int64
ip string
seasonType int
userAgent string
)
clickMsg = strings.Split(string(msg), "\001")
if len(clickMsg) < 10 {
err = errors.New("click msg error")
return
}
if aid, err = strconv.ParseInt(clickMsg[1], 10, 64); err != nil {
err = fmt.Errorf("aid(%s) error", clickMsg[1])
return
}
if aid <= 0 {
err = fmt.Errorf("wocao aid(%s) error", clickMsg[1])
return
}
if plat, err = strconv.ParseInt(clickMsg[0], 10, 64); err != nil {
err = fmt.Errorf("plat(%s) error", clickMsg[0])
return
}
if plat != 3 && plat != 4 {
err = fmt.Errorf("plat(%d) is not android or ios", plat)
return
}
userAgent = clickMsg[10]
bvID = clickMsg[8]
if bvID == "" {
err = fmt.Errorf("bvID(%s) is illegal", clickMsg[8])
return
}
if clickMsg[4] != "" && clickMsg[4] != "0" {
if mid, err = strconv.ParseInt(clickMsg[4], 10, 64); err != nil {
err = fmt.Errorf("mid(%s) is illegal", clickMsg[4])
return
}
}
if clickMsg[5] != "" {
if lv, err = strconv.ParseInt(clickMsg[5], 10, 64); err != nil {
err = fmt.Errorf("lv(%s) is illegal", clickMsg[5])
return
}
}
if ctime, err = strconv.ParseInt(clickMsg[6], 10, 64); err != nil {
err = fmt.Errorf("ctime(%s) is illegal", clickMsg[6])
return
}
if stime, err = strconv.ParseInt(clickMsg[7], 10, 64); err != nil {
err = fmt.Errorf("stime(%s) is illegal", clickMsg[7])
return
}
if ip = clickMsg[9]; ip == "" {
err = errors.New("ip is illegal")
return
}
if clickMsg[17] != "" {
if epid, err = strconv.ParseInt(clickMsg[17], 10, 64); err != nil {
err = fmt.Errorf("epid(%s) is illegal", clickMsg[17])
return
}
if clickMsg[15] != "null" {
if seasonType, err = strconv.Atoi(clickMsg[15]); err != nil {
err = fmt.Errorf("seasonType(%s) is illegal", clickMsg[15])
return
}
}
}
click = &unicom.ClickMsg{
Plat: int8(plat),
AID: aid,
MID: mid,
Lv: int8(lv),
CTime: ctime,
STime: stime,
BvID: bvID,
IP: ip,
KafkaBs: msg,
EpID: epid,
SeasonType: seasonType,
UserAgent: userAgent,
}
return
}
func (s *Service) cliChanProc(i int64) {
defer s.waiter.Done()
var (
cli *unicom.ClickMsg
cliChan = s.cliChan[i]
)
for {
var (
ub *unicom.UserBind
c = context.TODO()
err error
ok bool
count int
addFlow int
now = time.Now()
u *unicom.Unicom
cardType string
)
if cli, ok = <-cliChan; !ok || s.closed {
return
}
if count, err = s.dao.UserPackReceiveCache(c, cli.MID); err != nil {
log.Error("s.dao.UserBindCache error(%v) mid(%v) count(%v)", err, cli.MID, count)
continue
}
if count > 0 {
log.Info("s.dao.UserBindCache mid(%v) count(%v)", cli.MID, count)
continue
}
if ub, err = s.dao.UserBindCache(c, cli.MID); err != nil {
continue
}
if ub == nil || ub.Phone == 0 {
continue
}
res := s.unicomInfo(c, ub.Usermob, now)
if u, ok = res[ub.Usermob]; !ok || u == nil {
continue
}
switch u.Spid {
case 10019:
cardType = "22卡"
case 10020:
cardType = "33卡"
case 10021:
cardType = "小电视卡"
default:
log.Info("unicom spid equal 979 (%v)", ub)
continue
}
ub.Integral = ub.Integral + 10
switch cli.Lv {
case 0, 1, 2, 3:
addFlow = 10
case 4:
addFlow = 15
case 5:
addFlow = 20
case 6:
addFlow = 30
}
ub.Flow = ub.Flow + addFlow
if err = s.dao.AddUserBindCache(c, ub.Mid, ub); err != nil {
log.Error("s.dao.AddUserBindCache error(%v)", err)
continue
}
if err = s.dao.AddUserPackReceiveCache(c, ub.Mid, 1, now); err != nil {
log.Error("s.dao.AddUserPackReceiveCache error(%v)", err)
continue
}
s.dbcliChan[ub.Mid%s.c.ChanDBNum] <- ub
log.Info("unicom mobile cliChanProc userbind(%v)", ub)
s.unicomInfoc(ub.Usermob, ub.Phone, int(cli.Lv), 10, addFlow, cardType, ub.Mid, now)
s.addUserIntegralLog(&unicom.UserIntegralLog{Phone: ub.Phone, Mid: ub.Mid, UnicomDesc: cardType, Type: 0, Integral: 10, Flow: addFlow, Desc: "每日礼包"})
}
}
func (s *Service) dbcliChanProc(i int64) {
defer s.waiter.Done()
var (
ub *unicom.UserBind
dbcliChan = s.dbcliChan[i]
)
for {
var (
c = context.TODO()
ok bool
row int64
err error
)
if ub, ok = <-dbcliChan; !ok || s.closed {
return
}
if row, err = s.dao.UpUserIntegral(c, ub); err != nil || row == 0 {
log.Error("s.dao.UpUserIntegral ub(%v) error(%v) or result==0", ub, err)
continue
}
log.Info("unicom mobile dbcliChanProc userbind(%v)", ub)
}
}
// unicomInfo
func (s *Service) unicomInfo(c context.Context, usermob string, now time.Time) (res map[string]*unicom.Unicom) {
var (
err error
u []*unicom.Unicom
)
res = map[string]*unicom.Unicom{}
if u, err = s.dao.UnicomCache(c, usermob); err == nil && len(u) > 0 {
s.pHit.Incr("unicoms_cache")
} else {
if u, err = s.dao.OrdersUserFlow(context.TODO(), usermob); err != nil {
log.Error("unicom_s.dao.OrdersUserFlow error(%v)", err)
return
}
s.pMiss.Incr("unicoms_cache")
}
if len(u) > 0 {
row := &unicom.Unicom{}
for _, user := range u {
if user.TypeInt == 1 && now.Unix() <= int64(user.Endtime) {
*row = *user
break
} else if user.TypeInt == 0 {
if user.Spid == 979 {
continue
}
if int64(row.Ordertime) > int64(user.Ordertime) {
continue
}
*row = *user
}
}
if row.Spid == 0 {
return
}
res[usermob] = row
}
return
}
func (s *Service) upBindAll() {
var (
orders []*unicom.UserBind
err error
start = 0
end = 1000
)
for {
var tmp []*unicom.UserBind
if tmp, err = s.dao.BindAll(context.TODO(), start, end); err != nil {
log.Error("s.dao.BindAll error(%v)", err)
return
}
start = end + start
if len(tmp) == 0 {
break
}
orders = append(orders, tmp...)
}
for _, b := range orders {
var (
c = context.TODO()
u *unicom.Unicom
ok bool
now = time.Now()
integral int
ub *unicom.UserBind
err error
cardType string
)
if now.Month() == b.Monthly.Month() && now.Year() == b.Monthly.Year() {
continue
}
res := s.unicomInfo(c, b.Usermob, now)
if u, ok = res[b.Usermob]; !ok || u == nil {
continue
}
switch u.Spid {
case 10019:
integral = 220
cardType = "22卡"
case 10020:
integral = 330
cardType = "33卡"
case 10021:
integral = 660
cardType = "小电视卡"
default:
continue
}
if ub, err = s.dao.UserBindCache(c, b.Mid); err != nil {
continue
}
if ub == nil || ub.Phone == 0 {
continue
}
ub.Integral = ub.Integral + integral
ub.Monthly = now
if err = s.dao.AddUserBindCache(c, ub.Mid, ub); err != nil {
log.Error("s.dao.AddUserBindCache error(%v)", err)
continue
}
s.dbcliChan[ub.Mid%s.c.ChanDBNum] <- ub
log.Info("unicom mobile upBindAll userbind(%v)", ub)
s.unicomInfoc(ub.Usermob, ub.Phone, 0, integral, 0, cardType, ub.Mid, now)
s.addUserIntegralLog(&unicom.UserIntegralLog{Phone: ub.Phone, Mid: ub.Mid, UnicomDesc: cardType, Type: 1, Integral: integral, Flow: 0, Desc: "每月礼包"})
}
}
func (s *Service) updatemonth(now time.Time) {
m := int(now.Month())
if lmonth, ok := s.lastmonth[m]; !ok || !lmonth {
if now.Day() == 1 {
s.upBindAll()
s.lastmonth[m] = true
if m = m + 1; m > 12 {
m = 1
}
s.lastmonth[m] = false
log.Info("updatepro user monthly integral success")
}
}
}
func (s *Service) loadUnicomFlow() {
var (
list map[string]*unicom.UnicomUserFlow
err error
)
if list, err = s.dao.UserFlowListCache(context.TODO()); err != nil {
log.Error("load unicom s.dao.UserFlowListCache error(%v)", err)
return
}
log.Info("load unicom flow total len(%v)", len(list))
for key, u := range list {
var (
c = context.TODO()
requestNo int64
orderstatus string
msg string
)
if err = s.dao.UserFlowCache(c, key); err != nil {
if err == memcache.ErrNotFound {
if err = s.returnPoints(c, u); err != nil {
if err != ecode.NothingFound {
log.Error("load unicom s.returnPoints error(%v)", err)
continue
}
err = nil
}
log.Info("load unicom userbind timeout flow(%v)", u)
} else {
log.Error("load unicom s.dao.UserFlowCache error(%v)", err)
continue
}
} else {
if requestNo, err = s.seqdao.SeqID(c); err != nil {
log.Error("load unicom s.seqdao.SeqID error(%v)", err)
continue
}
if orderstatus, msg, err = s.dao.FlowQry(c, u.Phone, requestNo, u.Outorderid, u.Orderid, time.Now()); err != nil {
log.Error("load unicom s.dao.FlowQry error(%v) msg(%s)", err, msg)
continue
}
log.Info("load unicom userbind flow(%v) orderstatus(%s)", u, orderstatus)
if orderstatus == "00" {
continue
} else if orderstatus != "01" {
if err = s.returnPoints(c, u); err != nil {
if err != ecode.NothingFound {
log.Error("load unicom s.returnPoints error(%v)", err)
continue
}
err = nil
}
}
}
delete(list, key)
if err = s.dao.DeleteUserFlowCache(c, key); err != nil {
log.Error("load unicom s.dao.DeleteUserFlowCache error(%v)", err)
continue
}
}
if err = s.dao.AddUserFlowListCache(context.TODO(), list); err != nil {
log.Error("load unicom s.dao.AddUserFlowListCache error(%v)", err)
return
}
log.Info("load unicom flow last len(%v) success", len(list))
}
// returnPoints retutn user integral and flow
func (s *Service) returnPoints(c context.Context, u *unicom.UnicomUserFlow) (err error) {
var (
userbind *unicom.UserBind
result int64
)
if userbind, err = s.unicomBindInfo(c, u.Mid); err != nil {
return
}
ub := &unicom.UserBind{}
*ub = *userbind
ub.Flow = ub.Flow + u.Flow
ub.Integral = ub.Integral + u.Integral
if err = s.dao.AddUserBindCache(c, ub.Mid, ub); err != nil {
log.Error("unicom s.dao.AddUserBindCache error(%v)", err)
return
}
if result, err = s.dao.UpUserIntegral(c, ub); err != nil || result == 0 {
log.Error("unicom s.dao.UpUserIntegral error(%v) or result==0", err)
return
}
var packInt int
if u.Integral > 0 {
packInt = u.Integral
} else {
packInt = u.Flow
}
ul := &unicom.UserPackLog{
Phone: u.Phone,
Usermob: ub.Usermob,
Mid: u.Mid,
RequestNo: u.Outorderid,
Type: 0,
Desc: u.Desc + ",领取失败并返还",
Integral: packInt,
}
s.addUserPackLog(ul)
s.addUserIntegralLog(&unicom.UserIntegralLog{Phone: u.Phone, Mid: u.Mid, UnicomDesc: "", Type: 2, Integral: u.Integral, Flow: u.Flow, Desc: u.Desc + ",领取失败并返还"})
log.Info("unicom_pack(%v) mid(%v)", u.Desc+",领取失败并返还", userbind.Mid)
s.unicomPackInfoc(userbind.Usermob, u.Desc+",领取失败并返还", u.Orderid, userbind.Phone, packInt, 0, userbind.Mid, time.Now())
return
}
// unicomBindInfo unicom bind info
func (s *Service) unicomBindInfo(c context.Context, mid int64) (res *unicom.UserBind, err error) {
if res, err = s.dao.UserBindCache(c, mid); err != nil {
if res, err = s.dao.UserBind(c, mid); err != nil {
log.Error("s.dao.UserBind error(%v)", err)
return
}
if res == nil {
err = ecode.NothingFound
return
}
if err = s.dao.AddUserBindCache(c, mid, res); err != nil {
log.Error("s.dao.AddUserBindCache mid(%d) error(%v)", mid, err)
return
}
}
return
}
// loadUnicomIPOrder load unciom ip order update
func (s *Service) loadUnicomIPOrder(now time.Time) {
var (
dbips map[string]*unicom.UnicomIP
err error
)
if dbips, err = s.loadUnicomIP(context.TODO()); err != nil {
log.Error("s.loadUnicomIP", err)
return
}
if len(dbips) == 0 {
log.Error("db cache ip len 0")
return
}
unicomIP, err := s.dao.UnicomIP(context.TODO(), now)
if err != nil {
log.Error("s.dao.UnicomIP(%v)", err)
return
}
if len(unicomIP) == 0 {
log.Info("unicom ip orders is null")
return
}
tx, err := s.dao.BeginTran(context.TODO())
if err != nil {
log.Error("s.dao.BeginTran error(%v)", err)
return
}
for _, uip := range unicomIP {
key := fmt.Sprintf(_initIPUnicomKey, uip.Ipbegin, uip.Ipend)
if _, ok := dbips[key]; ok {
delete(dbips, key)
continue
}
var (
result int64
)
if result, err = s.dao.InUnicomIPSync(tx, uip, time.Now()); err != nil || result == 0 {
tx.Rollback()
log.Error("s.dao.InUnicomIPSync error(%v)", err)
return
}
}
for _, uold := range dbips {
var (
result int64
)
if result, err = s.dao.UpUnicomIP(tx, uold.Ipbegin, uold.Ipend, 0, time.Now()); err != nil || result == 0 {
tx.Rollback()
log.Error("s.dao.UpUnicomIP error(%v)", err)
return
}
}
if err = tx.Commit(); err != nil {
log.Error("tx.Commit error(%v)", err)
return
}
log.Info("update unicom ip success")
}
// loadUnicomIP load unicom ip
func (s *Service) loadUnicomIP(c context.Context) (res map[string]*unicom.UnicomIP, err error) {
var unicomIP []*unicom.UnicomIP
unicomIP, err = s.dao.IPSync(c)
if err != nil {
log.Error("s.dao.IPSync error(%v)", err)
return
}
tmp := map[string]*unicom.UnicomIP{}
for _, u := range unicomIP {
key := fmt.Sprintf(_initIPUnicomKey, u.Ipbegin, u.Ipend)
tmp[key] = u
}
res = tmp
log.Info("loadUnicomIPCache success")
return
}
func (s *Service) addUserPackLog(u *unicom.UserPackLog) {
select {
case s.packLogCh <- u:
default:
log.Warn("user pack log buffer is full")
}
}
func (s *Service) addUserIntegralLog(u *unicom.UserIntegralLog) {
select {
case s.integralLogCh[u.Mid%s.c.ChanDBNum] <- u:
default:
log.Warn("user add integral and flow log buffer is full")
}
}
func (s *Service) addUserPackLogproc() {
for {
i, ok := <-s.packLogCh
if !ok || s.closed {
log.Warn("user pack log proc exit")
return
}
var (
c = context.TODO()
result int64
err error
)
switch v := i.(type) {
case *unicom.UserPackLog:
if result, err = s.dao.InUserPackLog(c, v); err != nil || result == 0 {
log.Error("s.dao.UpUserIntegral error(%v) or result==0", err)
continue
}
log.Info("unicom user flow or integral back mid(%d) phone(%d)", v.Mid, v.Phone)
}
}
}
func (s *Service) addUserIntegralLogproc(i int64) {
var (
dbcliChan = s.integralLogCh[i]
)
for {
i, ok := <-dbcliChan
if !ok || s.closed {
log.Warn("user pack log proc exit")
return
}
var (
logID = 91
)
switch v := i.(type) {
case *unicom.UserIntegralLog:
// if result, err = s.dao.InUserIntegralLog(c, v); err != nil || result == 0 {
// log.Error("s.dao.InUserIntegralLog error(%v) or result==0", err)
// continue
// }
report.User(&report.UserInfo{
Mid: v.Mid,
Business: logID,
Action: "unicom_userpack_add",
Ctime: time.Now(),
Content: map[string]interface{}{
"phone": v.Phone,
"pack_desc": v.Desc,
"integral": v.Integral,
},
})
}
}
}

View File

@@ -0,0 +1,62 @@
package unicom
import (
"context"
"flag"
"path/filepath"
"testing"
"time"
"go-common/app/job/main/app-wall/conf"
"go-common/app/job/main/app-wall/model/unicom"
. "github.com/smartystreets/goconvey/convey"
)
var (
s *Service
)
func WithService(f func(s *Service)) func() {
return func() {
f(s)
}
}
func init() {
dir, _ := filepath.Abs("../../cmd/app-wall-job-test.toml")
flag.Set("conf", dir)
conf.Init()
s = New(conf.Conf)
time.Sleep(time.Second)
}
func TestAddUserPackLogproc(t *testing.T) {
Convey("Unicom addUserPackLogproc", t, WithService(func(s *Service) {
s.addUserPackLogproc()
}))
}
func TestAddUserIntegralLogproc(t *testing.T) {
Convey("Unicom addUserIntegralLogproc", t, WithService(func(s *Service) {
s.addUserIntegralLogproc(1)
}))
}
func TestAddUserIntegralLog(t *testing.T) {
Convey("Unicom addUserIntegralLog", t, WithService(func(s *Service) {
s.addUserIntegralLog(&unicom.UserIntegralLog{})
}))
}
func TestLoadUnicomIP(t *testing.T) {
Convey("Unicom loadUnicomIP", t, WithService(func(s *Service) {
s.loadUnicomIP(context.TODO())
}))
}
func TestLoadUnicomIPOrder(t *testing.T) {
Convey("Unicom loadUnicomIPOrder", t, WithService(func(s *Service) {
s.loadUnicomIPOrder(time.Now())
}))
}