Create & Init Project...

app/job/main/app-wall/service/offer/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "active_test.go",
+        "retry_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/app-wall/conf:go_default_library",
+        "//app/job/main/app-wall/model/offer:go_default_library",
+        "//vendor/github.com/bsm/sarama-cluster:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "active.go",
+        "retry.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/main/app-wall/service/offer",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/app-wall/conf:go_default_library",
+        "//app/job/main/app-wall/dao/offer:go_default_library",
+        "//app/job/main/app-wall/model:go_default_library",
+        "//app/job/main/app-wall/model/offer:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/bsm/sarama-cluster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/app-wall/service/offer/active.go

@@ -0,0 +1,113 @@
+package offer
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"go-common/app/job/main/app-wall/model"
+	"go-common/app/job/main/app-wall/model/offer"
+	"go-common/library/log"
+)
+
+func (s *Service) activeConsumer() {
+	defer s.waiter.Done()
+LOOP:
+	for {
+		select {
+		case err := <-s.consumer.Errors():
+			log.Error("group(%s) topic(%s) addr(%s) catch error(%+v)", s.c.Consumer.Group, s.c.Consumer.Topic, s.c.Consumer.Brokers, err)
+			continue
+		case notify := <-s.consumer.Notifications():
+			log.Info("notification(%v)", notify)
+			continue
+		case msg, ok := <-s.consumer.Messages():
+			if !ok {
+				log.Error("active consumer exit!")
+				break LOOP
+			}
+			s.consumer.MarkOffset(msg, "")
+			active, err := s.checkMsgIllegal(msg.Value)
+			if err != nil {
+				log.Error("s.checkMsgIllegal(%s) error(%v)", msg.Value, err)
+				continue
+			}
+			if active == nil {
+				continue
+			}
+			s.activeChan <- active
+		}
+	}
+}
+
+func (s *Service) checkMsgIllegal(msg []byte) (active *offer.ActiveMsg, err error) {
+	var (
+		msgs      []string
+		pid       int64
+		os        string
+		androidid string
+		imei      string
+	)
+	msgs = strings.Split(string(msg), "|")
+	if len(msgs) < 9 {
+		err = fmt.Errorf("active msg(%s) split len(%d)<9", msg, len(msgs))
+		return
+	}
+	if pid, err = strconv.ParseInt(msgs[8], 10, 64); err != nil {
+		return
+	}
+	if pid%10 == 3 {
+		os = model.TypeAndriod
+		if len(msgs) > 22 {
+			androidid = msgs[22]
+		}
+		if len(msgs) > 23 {
+			imei = msgs[23]
+		}
+		if imei == "" {
+			log.Warn("active msg(%s) imei(%s) is illegal", msg, imei)
+		} else {
+			log.Warn("active msg(%s) imei(%s) is legal", msg, imei)
+		}
+		if androidid == "" {
+			log.Warn("active msg(%s) androidid(%s) is illegal", msg, androidid)
+		}
+		if androidid == "" && imei == "" {
+			err = fmt.Errorf("active msg(%s) androidid(%s) and imei(%s) is illegal", msg, androidid, imei)
+			return
+		}
+	} else {
+		err = fmt.Errorf("active msg(%s) pid(%d) platform not android", msg, pid)
+		return
+	}
+	active = &offer.ActiveMsg{OS: os, IMEI: imei, Androidid: androidid, Mac: ""}
+	return
+}
+
+func (s *Service) activeproc() {
+	defer s.waiter.Done()
+	for {
+		msg, ok := <-s.activeChan
+		if !ok {
+			log.Error("active chan id closed")
+			break
+		}
+		s.active(msg)
+	}
+}
+
+func (s *Service) active(msg *offer.ActiveMsg) {
+	var err error
+	c := context.TODO()
+	if err = retry(func() (err error) {
+		return s.dao.Active(c, msg.OS, msg.IMEI, msg.Androidid, msg.Mac, "")
+	}, _upActiveRetry, _sleep); err != nil {
+		log.Error("%+v", err)
+		if err = s.syncRetry(c, offer.ActionActive, msg.OS, msg.IMEI, msg.Androidid, msg.Mac); err != nil {
+			log.Error("%+v", err)
+		}
+		return
+	}
+	log.Info("active device os(%s) imei(%s) androidid(%s) mac(%s) success", msg.OS, msg.IMEI, msg.Androidid, msg.Mac)
+}

app/job/main/app-wall/service/offer/active_test.go

@@ -0,0 +1,81 @@
+package offer
+
+import (
+	"reflect"
+	"testing"
+
+	"go-common/app/job/main/app-wall/model/offer"
+)
+
+func TestService_activeConsumer(t *testing.T) {
+	tests := []struct {
+		name string
+		s    *Service
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.s.activeConsumer()
+		})
+	}
+}
+
+func TestService_checkMsgIllegal(t *testing.T) {
+	type args struct {
+		msg []byte
+	}
+	tests := []struct {
+		name       string
+		s          *Service
+		args       args
+		wantActive *offer.ActiveMsg
+		wantErr    bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotActive, err := tt.s.checkMsgIllegal(tt.args.msg)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Service.checkMsgIllegal() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotActive, tt.wantActive) {
+				t.Errorf("Service.checkMsgIllegal() = %v, want %v", gotActive, tt.wantActive)
+			}
+		})
+	}
+}
+
+func TestService_activeproc(t *testing.T) {
+	tests := []struct {
+		name string
+		s    *Service
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.s.activeproc()
+		})
+	}
+}
+
+func TestService_active(t *testing.T) {
+	type args struct {
+		msg *offer.ActiveMsg
+	}
+	tests := []struct {
+		name string
+		s    *Service
+		args args
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.s.active(tt.args.msg)
+		})
+	}
+}

app/job/main/app-wall/service/offer/retry.go

@@ -0,0 +1,72 @@
+package offer
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/job/main/app-wall/model/offer"
+	"go-common/library/log"
+)
+
+const (
+	_upActiveRetry = 5
+	_sleep         = 100 * time.Millisecond
+)
+
+func (s *Service) retryproc() {
+	defer s.waiter.Done()
+	var (
+		bs  []byte
+		err error
+	)
+	c := context.TODO()
+	msg := &offer.Retry{}
+	for {
+		if s.closed {
+			break
+		}
+		if bs, err = s.dao.PopFail(c); err != nil {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		if len(bs) == 0 {
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		if err = json.Unmarshal(bs, msg); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", bs, err)
+			continue
+		}
+		log.Info("retry action(%s) data(%s)", msg.Action, bs)
+		switch msg.Action {
+		case offer.ActionActive:
+			if msg.Data != nil {
+				if err = retry(func() (err error) {
+					return s.dao.Active(c, msg.Data.OS, msg.Data.IMEI, msg.Data.Androidid, msg.Data.Mac, "")
+				}, _upActiveRetry, _sleep); err != nil {
+					log.Error("%+v", err)
+					if err = s.syncRetry(c, offer.ActionActive, msg.Data.OS, msg.Data.IMEI, msg.Data.Androidid, msg.Data.Mac); err != nil {
+						log.Error("%+v", err)
+					}
+					return
+				}
+			}
+		}
+	}
+}
+
+func retry(callback func() error, retry int, sleep time.Duration) (err error) {
+	for i := 0; i < retry; i++ {
+		if err = callback(); err == nil {
+			return
+		}
+		time.Sleep(sleep)
+	}
+	return
+}
+
+func (s *Service) syncRetry(c context.Context, action, os, imei, androidid, mac string) (err error) {
+	retry := &offer.Retry{Action: action, Data: &offer.Data{OS: os, IMEI: imei, Androidid: androidid, Mac: mac}}
+	return s.dao.PushFail(c, retry)
+}

app/job/main/app-wall/service/offer/retry_test.go

@@ -0,0 +1,69 @@
+package offer
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestService_retryproc(t *testing.T) {
+	tests := []struct {
+		name string
+		s    *Service
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.s.retryproc()
+		})
+	}
+}
+
+func Test_retry(t *testing.T) {
+	type args struct {
+		callback func() error
+		retry    int
+		sleep    time.Duration
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := retry(tt.args.callback, tt.args.retry, tt.args.sleep); (err != nil) != tt.wantErr {
+				t.Errorf("retry() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestService_syncRetry(t *testing.T) {
+	type args struct {
+		c         context.Context
+		action    string
+		os        string
+		imei      string
+		androidid string
+		mac       string
+	}
+	tests := []struct {
+		name    string
+		s       *Service
+		args    args
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.s.syncRetry(tt.args.c, tt.args.action, tt.args.os, tt.args.imei, tt.args.androidid, tt.args.mac); (err != nil) != tt.wantErr {
+				t.Errorf("Service.syncRetry() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

app/job/main/app-wall/service/offer/service.go

@@ -0,0 +1,88 @@
+package offer
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/job/main/app-wall/conf"
+	offerDao "go-common/app/job/main/app-wall/dao/offer"
+	"go-common/app/job/main/app-wall/model/offer"
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	cluster "github.com/bsm/sarama-cluster"
+)
+
+// Service struct
+type Service struct {
+	c          *conf.Config
+	dao        *offerDao.Dao
+	consumer   *cluster.Consumer
+	activeChan chan *offer.ActiveMsg
+	closed     bool
+	waiter     sync.WaitGroup
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:          c,
+		dao:        offerDao.New(c),
+		activeChan: make(chan *offer.ActiveMsg, 10240),
+		closed:     false,
+	}
+	var err error
+	if s.consumer, err = s.NewConsumer(); err != nil {
+		log.Error("%+v", err)
+		panic(err)
+	}
+	s.waiter.Add(1)
+	go s.activeConsumer()
+	s.waiter.Add(1)
+	go s.activeproc()
+	// retry consumer
+	for i := 0; i < 4; i++ {
+		s.waiter.Add(1)
+		go s.retryproc()
+	}
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.closed = true
+	s.consumer.Close()
+	s.waiter.Wait()
+	log.Info("app-wall-job closed.")
+}
+
+// NewConsumer new cluster consumer.
+func (s *Service) NewConsumer() (*cluster.Consumer, error) {
+	// cluster config
+	cfg := cluster.NewConfig()
+	// NOTE cluster auto commit offset interval
+	cfg.Consumer.Offsets.CommitInterval = time.Second * 1
+	// NOTE set fetch.wait.max.ms
+	cfg.Consumer.MaxWaitTime = time.Millisecond * 100
+	// NOTE errors that occur during offset management,if enabled, c.Errors channel must be read
+	cfg.Consumer.Return.Errors = true
+	// NOTE notifications that occur during consumer, if enabled, c.Notifications channel must be read
+	cfg.Group.Return.Notifications = true
+	// The initial offset to use if no offset was previously committed.
+	// default: OffsetOldest
+	if strings.ToLower(s.c.Consumer.Offset) != "new" {
+		cfg.Consumer.Offsets.Initial = sarama.OffsetOldest
+	} else {
+		cfg.Consumer.Offsets.Initial = sarama.OffsetNewest
+	}
+	// new cluster consumer
+	log.Info("s.c.Consumer.Brokers: %v", s.c.Consumer.Brokers)
+	return cluster.NewConsumer(s.c.Consumer.Brokers, s.c.Consumer.Group, []string{s.c.Consumer.Topic}, cfg)
+}

app/job/main/app-wall/service/offer/service_test.go

@@ -0,0 +1,98 @@
+package offer
+
+import (
+	"context"
+	"flag"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"go-common/app/job/main/app-wall/conf"
+
+	cluster "github.com/bsm/sarama-cluster"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	s *Service
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/app-wall-job-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	s = New(conf.Conf)
+}
+
+func TestNew(t *testing.T) {
+	type args struct {
+		c *conf.Config
+	}
+	tests := []struct {
+		name  string
+		args  args
+		wantS *Service
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotS := New(tt.args.c); !reflect.DeepEqual(gotS, tt.wantS) {
+				t.Errorf("New() = %v, want %v", gotS, tt.wantS)
+			}
+		})
+	}
+}
+
+func TestService_Ping(t *testing.T) {
+	type args struct {
+		c context.Context
+	}
+	tests := []struct {
+		name    string
+		s       *Service
+		args    args
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.s.Ping(tt.args.c); (err != nil) != tt.wantErr {
+				t.Errorf("Service.Ping() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestService_Close(t *testing.T) {
+	tests := []struct {
+		name string
+		s    *Service
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.s.Close()
+		})
+	}
+}
+
+func TestService_NewConsumer(t *testing.T) {
+	tests := []struct {
+		name    string
+		s       *Service
+		want    *cluster.Consumer
+		wantErr error
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		Convey(tt.name, t, func() {
+			got, err := tt.s.NewConsumer()
+			So(err, ShouldEqual, tt.wantErr)
+			So(got, ShouldResemble, tt.want)
+		})
+	}
+}