Create & Init Project...

app/job/main/appstatic/dao/push/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "appres_test.go",
+        "dao_test.go",
+        "push_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "appres.go",
+        "dao.go",
+        "push.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/push",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/push/appres.go

@@ -0,0 +1,95 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/library/log"
+	"go-common/library/naming/discovery"
+	"go-common/library/net/rpc/warden"
+)
+
+func errlog(step string, err error) {
+	log.Error("CallPush Step [%s] Err [%v]", step, err)
+}
+
+// build grpc client for app-resource
+func (d *Dao) grpcClient(addr string) (appres.AppResourceClient, error) {
+	client := warden.NewClient(d.c.AppresClient)
+	cc, err := client.Dial(context.Background(), addr)
+	if err != nil {
+		return nil, err
+	}
+	return appres.NewAppResourceClient(cc), nil
+}
+
+// CallRefresh picks ip addrs from discovery, and call grpc method
+func (d *Dao) CallRefresh(ctx context.Context) (err error) {
+	var (
+		addrs    []string
+		client   appres.AppResourceClient
+		arg      = &appres.NoArgRequest{}
+		succCall int
+	)
+	if addrs, err = d.pickAddrs(); err != nil {
+		errlog("pickAddrs", err)
+		return
+	}
+	// loop grpc call, ignore error
+	for _, addr := range addrs {
+		if client, err = d.grpcClient(addr); err != nil {
+			errlog(fmt.Sprintf("grpcDial Addr [%s]", addr), err)
+			continue
+		}
+		if _, err = client.ModuleUpdateCache(ctx, arg); err != nil {
+			errlog(fmt.Sprintf("grpcCall Addr [%s] ", addr), err)
+			continue
+		}
+		succCall++
+	}
+	if succCall == 0 { // addrs must be greater than 0, if succ call is zero, return error
+		return fmt.Errorf("CallRefresh Addrs [%d], Zero Succ!", len(addrs))
+	}
+	log.Info("CallPush Refresh Succ [%d], Total [%d]", succCall, len(addrs))
+	err = nil
+	return
+}
+
+// pick all the app-resource grpc instances from discovery
+func (d *Dao) pickAddrs() (grpcAddrs []string, err error) {
+	dis := discovery.New(nil)
+	defer dis.Close()
+	b := dis.Build(d.c.Cfg.Grpc.ApiAppID)
+	defer b.Close()
+	ch := b.Watch()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+	defer cancel()
+	select {
+	case <-ch:
+	case <-ctx.Done():
+		err = fmt.Errorf("查找节点超时 请检查appid是否填写正确")
+		return
+	}
+	ins, ok := b.Fetch(context.Background())
+	if !ok {
+		err = fmt.Errorf("discovery 拉取失败")
+		return
+	}
+	for _, vs := range ins {
+		for _, v := range vs {
+			for _, addr := range v.Addrs {
+				if strings.Contains(addr, "grpc://") {
+					ip := strings.Replace(addr, "grpc://", "", -1)
+					grpcAddrs = append(grpcAddrs, ip)
+				}
+			}
+		}
+	}
+	if len(grpcAddrs) == 0 {
+		err = fmt.Errorf("discovery 找不到服务节点")
+	}
+	return
+}

app/job/main/appstatic/dao/push/appres_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/interface/main/app-resource/api/v1"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestPusherrlog(t *testing.T) {
+	var (
+		step = ""
+		err  error
+	)
+	convey.Convey("errlog", t, func(ctx convey.C) {
+		errlog(step, err)
+		ctx.Convey("No return values", func(ctx convey.C) {
+		})
+	})
+}
+
+func TestPushgrpcClient(t *testing.T) {
+	var (
+		grpcAddrs []string
+		err       error
+		p1        v1.AppResourceClient
+	)
+	convey.Convey("pickAddrs", t, func(ctx convey.C) {
+		grpcAddrs, err = d.pickAddrs()
+		ctx.Convey("Then err should be nil.grpcAddrs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(grpcAddrs, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("grpcClient", t, func(ctx convey.C) {
+		fmt.Println("Call ", grpcAddrs[0])
+		p1, err = d.grpcClient(grpcAddrs[0])
+		ctx.Convey("Then err should be nil.p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("callRefresh", t, func(ctx convey.C) {
+		err = d.CallRefresh(context.Background())
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+
+}

app/job/main/appstatic/dao/push/dao.go

@@ -0,0 +1,33 @@
+package push
+
+import (
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c         *conf.Config
+	db        *xsql.DB
+	client    *bm.Client
+	redis     *redis.Pool
+	appresCli appres.AppResourceClient
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+		redis:  redis.NewPool(c.Redis),
+	}
+	var err error
+	if d.appresCli, err = appres.NewClient(c.AppresClient); err != nil {
+		panic(err)
+	}
+	return
+}

app/job/main/appstatic/dao/push/dao_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/push/push.go

@@ -0,0 +1,94 @@
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_diffFinish = "SELECT COUNT(1) FROM resource_file WHERE resource_id = ? AND url = ? AND file_type = ? AND is_deleted = 0"
+	_pushMsg    = "SELECT resource.id, resource.pool_id, resource_pool.`name` FROM resource LEFT JOIN resource_pool ON resource.pool_id = resource_pool.id WHERE resource.id = ? LIMIT 1"
+	_platform   = "SELECT b.value FROM resource_config a LEFT JOIN resource_limit b ON a.id = b.config_id WHERE a.resource_id = ? AND b.`column` = 'mobi_app' AND a.is_deleted = 0 AND b.is_deleted = 0"
+	_diffPkg    = 1
+)
+
+// CallPush calls the push server api
+func (d *Dao) CallPush(ctx context.Context, platform string, msg string, ip string) (err error) {
+	var (
+		cfg    = d.c.Cfg.Push
+		params = url.Values{}
+	)
+	params.Set("operation", fmt.Sprintf("%d", cfg.Operation))
+	params.Set("platform", platform)
+	params.Set("message", msg)
+	params.Set("speed", fmt.Sprintf("%d", cfg.QPS))
+	var res struct {
+		Code int `json:"code"`
+	}
+	if err = d.client.Post(ctx, cfg.URL, ip, params, &res); err != nil {
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		err = errors.Wrap(ecode.Int(res.Code), cfg.URL+"?"+params.Encode())
+	}
+	return
+}
+
+// DiffFinish checks whether the resource's diff calculation has been finished or not
+func (d *Dao) DiffFinish(c context.Context, resID string) (res bool, err error) {
+	count := 0
+	row := d.db.QueryRow(c, _diffFinish, resID, "", _diffPkg)
+	if err = row.Scan(&count); err != nil {
+		log.Error("d.DiffFinish err(%v)", err)
+		return
+	}
+	if count == 0 {
+		res = true
+	}
+	return
+}
+
+// PushMsg combines the resource pool info to prepare the msg to call PUSH
+func (d *Dao) PushMsg(c context.Context, resID string) (res string, err error) {
+	var (
+		msg  model.PushMsg
+		data []byte
+	)
+	row := d.db.QueryRow(c, _pushMsg, resID)
+	if err = row.Scan(&msg.ResID, &msg.ModID, &msg.ModName); err != nil {
+		log.Error("d.PushMsg err(%v)", err)
+	}
+	if data, err = json.Marshal(msg); err != nil {
+		log.Error("PushMsg Info ResID %d, Json Err %v", resID, err)
+		return
+	}
+	res = string(data)
+	return
+}
+
+// Platform picks the mobi_app value to distinguish the platform to push
+func (d *Dao) Platform(c context.Context, resID string) (res []string, err error) {
+	rows, err := d.db.Query(c, _platform, resID)
+	if err != nil {
+		log.Error("db.Query(%d) error(%v)", resID, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mobiApp string
+		if err = rows.Scan(&mobiApp); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		res = append(res, mobiApp)
+	}
+	return
+}

app/job/main/appstatic/dao/push/push_test.go

@@ -0,0 +1,55 @@
+package push
+
+import (
+	"testing"
+
+	"fmt"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_CallPush(t *testing.T) {
+	Convey("Http error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(-400).JSON("")
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Business Code error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code":-400}"`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Everything is fine", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code" : 0}`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_DiffFinish(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.DiffFinish(ctx, "57")
+		So(err, ShouldBeNil)
+		So(data, ShouldBeTrue)
+	}))
+}
+
+func TestDao_PushMsg(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.PushMsg(ctx, "57")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}
+
+func TestDao_Platform(t *testing.T) {
+	Convey("TestDao_Platform", t, WithDao(func(d *Dao) {
+		data, err := d.Platform(ctx, "77")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}

app/job/main/appstatic/dao/push/redis.go

@@ -0,0 +1,46 @@
+package push
+
+import (
+	"context"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const _pushKey = "appstatic-admin-topush"
+
+// ZrangeList picks up all the to push resIDs, ctime
+func (d *Dao) ZrangeList(c context.Context) (resIDs map[string]int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	defer conn.Close()
+	// get all the resIDs in one shot
+	if err = conn.Send("ZRANGE", _pushKey, 0, -1, "WITHSCORES"); err != nil {
+		log.Error("conn.Do(ZREVRANGE, %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() err(%v)", err)
+		return
+	}
+	if resIDs, err = redis.Int64Map(conn.Receive()); err != nil {
+		log.Error("redis.Int64s()err(%v)", err)
+		return
+	}
+	return
+}
+
+// ZRem ZREM trim from trim queue.
+func (d *Dao) ZRem(c context.Context, resID string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	if _, err = conn.Do("ZREM", key, resID); err != nil {
+		log.Error("conn.Send(ZADD %s - %v) error(%v)", key, resID, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/appstatic/dao/push/redis_test.go

@@ -0,0 +1,30 @@
+package push
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_ZrangeList(t *testing.T) {
+	Convey("TestDao_ZrangeList", t, WithDao(func(d *Dao) {
+		data, err := d.ZrangeList(context.TODO())
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+		Printf("%+v", data)
+	}))
+}
+
+func TestDao_ZRem(t *testing.T) {
+	var (
+		conn = d.redis.Get(ctx)
+		key  = _pushKey
+		id   = "999"
+	)
+	Convey("everything is fine", t, WithDao(func(d *Dao) {
+		conn.Do("ZADD", key, id)
+		err := d.ZRem(ctx, id)
+		So(err, ShouldBeNil)
+	}))
+}