Create & Init Project...

app/job/main/appstatic/dao/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/appstatic/dao/caldiff:all-srcs",
+        "//app/job/main/appstatic/dao/push:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/caldiff/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cal_diff_test.go",
+        "dao_test.go",
+        "download_test.go",
+        "upbfs_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cal_diff.go",
+        "dao.go",
+        "download.go",
+        "upbfs.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/caldiff",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/caldiff/cal_diff.go

@@ -0,0 +1,110 @@
+package caldiff
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+const (
+	_calDiffFmt = "SELECT id,`name`,type, md5, size, url, resource_id, file_type, from_ver FROM resource_file " +
+		"WHERE file_type = ? AND url = ? AND is_deleted = 0 %s ORDER BY id DESC %s"
+	_saveFile  = "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	_parseRes  = "SELECT id, `name`, version, pool_id FROM resource WHERE %s"
+	_parseFile = "SELECT id, `name`, type, md5, size, url, resource_id, file_type, from_ver FROM resource_file WHERE" +
+		" %s AND is_deleted = 0 %s"
+	_updateStatus = "UPDATE resource_file SET file_type = ? WHERE id = ?"
+	_diffPkg      = 1
+)
+
+var (
+	_calDiffNew   = fmt.Sprintf(_calDiffFmt, " AND ctime = mtime", "LIMIT 1")
+	_parseResID   = fmt.Sprintf(_parseRes, " id = ?")
+	_parseResVer  = fmt.Sprintf(_parseRes, " pool_id = ? AND version = ?")
+	_getReadyFile = fmt.Sprintf(_parseFile, " resource_id = ? AND file_type = ? AND url != ?", "LIMIT 1")
+)
+
+// UpdateStatus updates the file's status
+func (d *Dao) UpdateStatus(c context.Context, status int, id int) (err error) {
+	if _, err = d.db.Exec(c, _updateStatus, status, id); err != nil {
+		log.Error("UpdateStatus ID %d, Err %v", id, err)
+	}
+	return
+}
+
+// ReadyFile takes the already generated file
+func (d *Dao) ReadyFile(c context.Context, resID int, ftype int) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _getReadyFile, resID, ftype, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _getReadyFile, resID, ftype, err)
+	}
+	return
+}
+
+// ParseResVer takes one resource info
+func (d *Dao) ParseResVer(c context.Context, poolID int, version int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResVer, poolID, version)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE pool_id = ? AND version = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _parseResVer, poolID, version, err)
+	}
+	return
+}
+
+// ParseResID takes one resource info
+func (d *Dao) ParseResID(c context.Context, resID int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResID, resID)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE id = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d) error(%v)", _parseResID, resID, err)
+	}
+	return
+}
+
+// DiffNew picks the recently created diff packages
+func (d *Dao) DiffNew(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _calDiffNew, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// DiffRetry picks the recently created diff packages
+func (d *Dao) DiffRetry(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	query := fmt.Sprintf(_calDiffFmt, " AND ctime != mtime AND mtime < date_sub(now(), INTERVAL "+d.c.Cfg.Diff.Retry+")", "LIMIT 1")
+	row := d.db.QueryRow(c, query, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// SaveFile saves the file info
+func (d *Dao) SaveFile(c context.Context, fileID int, file *model.FileInfo) (err error) {
+	// "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	if _, err = d.db.Exec(c, _saveFile, file.URL, _diffPkg, file.Md5, file.Size, file.Name, fileID); err != nil {
+		log.Error("SaveFile ID %d, Err %v", fileID, err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/cal_diff_test.go

@@ -0,0 +1,94 @@
+package caldiff
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/appstatic/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+const (
+	_availableRes = "SELECT id FROM resource ORDER BY id DESC"
+)
+
+func TestDao_DiffNew(t *testing.T) {
+	Convey("TestDao_DiffNew", t, WithDao(func(d *Dao) {
+		file, err := d.DiffNew(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_DiffRetry(t *testing.T) {
+	Convey("TestDao_DiffRetry", t, WithDao(func(d *Dao) {
+		file, err := d.DiffRetry(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_SaveFile(t *testing.T) {
+	Convey("TestDao_SaveFile", t, WithDao(func(d *Dao) {
+		err := d.SaveFile(ctx, 1, &model.FileInfo{
+			Name: "123",
+			Size: 123,
+			Type: "1",
+			Md5:  "1234",
+			URL:  "xxx",
+		})
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_ParseResID(t *testing.T) {
+	Convey("TestDao_ParseResID", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		res, err := d.ParseResID(ctx, int(r.ID))
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+		fmt.Println(r.ID)
+	}))
+}
+
+func TestDao_ParseResVer(t *testing.T) {
+	Convey("TestDao_ParseResVer", t, WithDao(func(d *Dao) {
+		dd, err := d.ParseResVer(ctx, 23, 1)
+		So(err, ShouldBeNil)
+		fmt.Println(err)
+		So(dd, ShouldNotBeNil)
+		data, err2 := (json.Marshal(dd))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_ReadyFile(t *testing.T) {
+	Convey("TestDao_ReadyFile", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		dd, err := d.ReadyFile(ctx, int(r.ID), 0) // ftype = 0 full package
+		So(err, ShouldBeNil)
+		So(dd, ShouldNotBeNil)
+		data, _ := json.Marshal(dd)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_UpdateStatus(t *testing.T) {
+	Convey("TestDao_UpdateStatus", t, WithDao(func(d *Dao) {
+		err := d.UpdateStatus(ctx, 2, 304)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/appstatic/dao/caldiff/dao.go

@@ -0,0 +1,24 @@
+package caldiff
+
+import (
+	"go-common/app/job/main/appstatic/conf"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c      *conf.Config
+	db     *xsql.DB
+	client *bm.Client
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/dao_test.go

@@ -0,0 +1,53 @@
+package caldiff
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/caldiff/download.go

@@ -0,0 +1,42 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"go-common/library/log"
+)
+
+const (
+	errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
+)
+
+// DownloadFile downloads one file from url to local
+func (d *Dao) DownloadFile(ctx context.Context, api string, fileName string) (bts int64, err error) {
+	var (
+		res []byte
+		f   *os.File
+		req *http.Request
+	)
+	if req, err = d.client.NewRequest(http.MethodGet, api, "", url.Values{}); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if res, err = d.client.Raw(ctx, req); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if f, err = os.Create(fileName); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("CreateFile(%s)", fileName), err)
+		return
+	}
+	if bts, err = io.Copy(f, bytes.NewReader(res)); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("SaveFile(%s)", fileName), err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/download_test.go

@@ -0,0 +1,29 @@
+package caldiff
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_DownloadFile(t *testing.T) {
+	var (
+		newPath = fmt.Sprintf("%s/%s", d.c.Cfg.Diff.Folder, "123.zip")
+		url     = "http://i0.hdslb.com/bfs/face/ca14680bcfe4a956d1b6c06fbc1f6a6529257746.jpg"
+	)
+	Convey("TestDao_DownloadFile", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		data, err := d.DownloadFile(ctx, url, newPath)
+		So(err, ShouldBeNil)
+		So(data, ShouldBeGreaterThan, 0)
+		fmt.Println(data)
+	}))
+	Convey("CreateFile Error", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		_, err := d.DownloadFile(ctx, url, "/test/test.txt")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+}

app/job/main/appstatic/dao/caldiff/upbfs.go

@@ -0,0 +1,89 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// bfs info
+const (
+	_uploadURL = "/bfs/%s/%s"
+	_template  = "%s\n%s\n%s\n%d\n"
+	_method    = "PUT"
+	_bucket    = "app-static"
+)
+
+// Upload upload picture or log file to bfs
+func (d *Dao) Upload(c context.Context, fileName string, fileType string, timing int64, data []byte, bfs *conf.Bfs) (location string, err error) {
+	var (
+		req    *http.Request
+		resp   *http.Response
+		code   int
+		client = &http.Client{Timeout: time.Duration(bfs.Timeout) * time.Millisecond}
+		url    = fmt.Sprintf(bfs.Host+_uploadURL, _bucket, fileName)
+	)
+	// prepare the data of the file and init the request
+	buf := new(bytes.Buffer)
+	_, err = buf.Write(data)
+	if err != nil {
+		log.Error("Upload.buf.Write.error(%v)", err)
+		err = ecode.RequestErr
+		return
+	}
+	if req, err = http.NewRequest(_method, url, buf); err != nil {
+		log.Error("http.NewRequest() Upload(%v) error(%v)", url, err)
+		return
+	}
+	// request setting
+	authorization := authorize(bfs.Key, bfs.Secret, _method, _bucket, fileName, timing)
+	req.Header.Set("Date", fmt.Sprint(timing))
+	req.Header.Set("Authorization", authorization)
+	req.Header.Set("Content-Type", fileType)
+	resp, err = client.Do(req)
+	// response treatment
+	if err != nil {
+		log.Error("Bfs client.Do(%s) error(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err = errors.Wrap(ecode.Int(resp.StatusCode), "Bfs Status Code Error")
+		return
+	}
+	code, err = strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = errors.Wrap(ecode.Int(code), "Bfs Header Code Error")
+		return
+	}
+	location = resp.Header.Get("Location")
+	return
+}
+
+// authorize returns authorization for upload file to bfs
+func authorize(key, secret, method, bucket, file string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf(_template, method, bucket, file, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}

app/job/main/appstatic/dao/caldiff/upbfs_test.go

@@ -0,0 +1,57 @@
+package caldiff
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoUpload(t *testing.T) {
+	var (
+		c        = context.Background()
+		fileName = "test.txt"
+		fileType = "txt"
+		timing   = time.Now().Unix()
+		bfs      = d.c.Bfs
+		data     = []byte("123")
+		url      = fmt.Sprintf(bfs.Host+_uploadURL, "app-static", fileName)
+	)
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		ctx.Convey("http code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(-400)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("business code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).JSON(`{"code":-400}`)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("everything is fine", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).SetHeader("Location", "test").SetHeader("code", "200").JSON(`{"code":200}`)
+			location, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(location, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoauthorize(t *testing.T) {
+	var (
+		key    = "key"
+		secret = "secret"
+		method = "put"
+		bucket = "tv-cover"
+		file   = "file"
+		expire = int64(0)
+	)
+	convey.Convey("authorize", t, func(ctx convey.C) {
+		authorization := authorize(key, secret, method, bucket, file, expire)
+		ctx.Convey("Then authorization should not be nil.", func(ctx convey.C) {
+			ctx.So(authorization, convey.ShouldNotBeNil)
+		})
+	})
+}

app/job/main/appstatic/dao/push/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "appres_test.go",
+        "dao_test.go",
+        "push_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "appres.go",
+        "dao.go",
+        "push.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/push",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/push/appres.go

@@ -0,0 +1,95 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/library/log"
+	"go-common/library/naming/discovery"
+	"go-common/library/net/rpc/warden"
+)
+
+func errlog(step string, err error) {
+	log.Error("CallPush Step [%s] Err [%v]", step, err)
+}
+
+// build grpc client for app-resource
+func (d *Dao) grpcClient(addr string) (appres.AppResourceClient, error) {
+	client := warden.NewClient(d.c.AppresClient)
+	cc, err := client.Dial(context.Background(), addr)
+	if err != nil {
+		return nil, err
+	}
+	return appres.NewAppResourceClient(cc), nil
+}
+
+// CallRefresh picks ip addrs from discovery, and call grpc method
+func (d *Dao) CallRefresh(ctx context.Context) (err error) {
+	var (
+		addrs    []string
+		client   appres.AppResourceClient
+		arg      = &appres.NoArgRequest{}
+		succCall int
+	)
+	if addrs, err = d.pickAddrs(); err != nil {
+		errlog("pickAddrs", err)
+		return
+	}
+	// loop grpc call, ignore error
+	for _, addr := range addrs {
+		if client, err = d.grpcClient(addr); err != nil {
+			errlog(fmt.Sprintf("grpcDial Addr [%s]", addr), err)
+			continue
+		}
+		if _, err = client.ModuleUpdateCache(ctx, arg); err != nil {
+			errlog(fmt.Sprintf("grpcCall Addr [%s] ", addr), err)
+			continue
+		}
+		succCall++
+	}
+	if succCall == 0 { // addrs must be greater than 0, if succ call is zero, return error
+		return fmt.Errorf("CallRefresh Addrs [%d], Zero Succ!", len(addrs))
+	}
+	log.Info("CallPush Refresh Succ [%d], Total [%d]", succCall, len(addrs))
+	err = nil
+	return
+}
+
+// pick all the app-resource grpc instances from discovery
+func (d *Dao) pickAddrs() (grpcAddrs []string, err error) {
+	dis := discovery.New(nil)
+	defer dis.Close()
+	b := dis.Build(d.c.Cfg.Grpc.ApiAppID)
+	defer b.Close()
+	ch := b.Watch()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+	defer cancel()
+	select {
+	case <-ch:
+	case <-ctx.Done():
+		err = fmt.Errorf("查找节点超时 请检查appid是否填写正确")
+		return
+	}
+	ins, ok := b.Fetch(context.Background())
+	if !ok {
+		err = fmt.Errorf("discovery 拉取失败")
+		return
+	}
+	for _, vs := range ins {
+		for _, v := range vs {
+			for _, addr := range v.Addrs {
+				if strings.Contains(addr, "grpc://") {
+					ip := strings.Replace(addr, "grpc://", "", -1)
+					grpcAddrs = append(grpcAddrs, ip)
+				}
+			}
+		}
+	}
+	if len(grpcAddrs) == 0 {
+		err = fmt.Errorf("discovery 找不到服务节点")
+	}
+	return
+}

app/job/main/appstatic/dao/push/appres_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/interface/main/app-resource/api/v1"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestPusherrlog(t *testing.T) {
+	var (
+		step = ""
+		err  error
+	)
+	convey.Convey("errlog", t, func(ctx convey.C) {
+		errlog(step, err)
+		ctx.Convey("No return values", func(ctx convey.C) {
+		})
+	})
+}
+
+func TestPushgrpcClient(t *testing.T) {
+	var (
+		grpcAddrs []string
+		err       error
+		p1        v1.AppResourceClient
+	)
+	convey.Convey("pickAddrs", t, func(ctx convey.C) {
+		grpcAddrs, err = d.pickAddrs()
+		ctx.Convey("Then err should be nil.grpcAddrs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(grpcAddrs, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("grpcClient", t, func(ctx convey.C) {
+		fmt.Println("Call ", grpcAddrs[0])
+		p1, err = d.grpcClient(grpcAddrs[0])
+		ctx.Convey("Then err should be nil.p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("callRefresh", t, func(ctx convey.C) {
+		err = d.CallRefresh(context.Background())
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+
+}

app/job/main/appstatic/dao/push/dao.go

@@ -0,0 +1,33 @@
+package push
+
+import (
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c         *conf.Config
+	db        *xsql.DB
+	client    *bm.Client
+	redis     *redis.Pool
+	appresCli appres.AppResourceClient
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+		redis:  redis.NewPool(c.Redis),
+	}
+	var err error
+	if d.appresCli, err = appres.NewClient(c.AppresClient); err != nil {
+		panic(err)
+	}
+	return
+}

app/job/main/appstatic/dao/push/dao_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/push/push.go

@@ -0,0 +1,94 @@
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_diffFinish = "SELECT COUNT(1) FROM resource_file WHERE resource_id = ? AND url = ? AND file_type = ? AND is_deleted = 0"
+	_pushMsg    = "SELECT resource.id, resource.pool_id, resource_pool.`name` FROM resource LEFT JOIN resource_pool ON resource.pool_id = resource_pool.id WHERE resource.id = ? LIMIT 1"
+	_platform   = "SELECT b.value FROM resource_config a LEFT JOIN resource_limit b ON a.id = b.config_id WHERE a.resource_id = ? AND b.`column` = 'mobi_app' AND a.is_deleted = 0 AND b.is_deleted = 0"
+	_diffPkg    = 1
+)
+
+// CallPush calls the push server api
+func (d *Dao) CallPush(ctx context.Context, platform string, msg string, ip string) (err error) {
+	var (
+		cfg    = d.c.Cfg.Push
+		params = url.Values{}
+	)
+	params.Set("operation", fmt.Sprintf("%d", cfg.Operation))
+	params.Set("platform", platform)
+	params.Set("message", msg)
+	params.Set("speed", fmt.Sprintf("%d", cfg.QPS))
+	var res struct {
+		Code int `json:"code"`
+	}
+	if err = d.client.Post(ctx, cfg.URL, ip, params, &res); err != nil {
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		err = errors.Wrap(ecode.Int(res.Code), cfg.URL+"?"+params.Encode())
+	}
+	return
+}
+
+// DiffFinish checks whether the resource's diff calculation has been finished or not
+func (d *Dao) DiffFinish(c context.Context, resID string) (res bool, err error) {
+	count := 0
+	row := d.db.QueryRow(c, _diffFinish, resID, "", _diffPkg)
+	if err = row.Scan(&count); err != nil {
+		log.Error("d.DiffFinish err(%v)", err)
+		return
+	}
+	if count == 0 {
+		res = true
+	}
+	return
+}
+
+// PushMsg combines the resource pool info to prepare the msg to call PUSH
+func (d *Dao) PushMsg(c context.Context, resID string) (res string, err error) {
+	var (
+		msg  model.PushMsg
+		data []byte
+	)
+	row := d.db.QueryRow(c, _pushMsg, resID)
+	if err = row.Scan(&msg.ResID, &msg.ModID, &msg.ModName); err != nil {
+		log.Error("d.PushMsg err(%v)", err)
+	}
+	if data, err = json.Marshal(msg); err != nil {
+		log.Error("PushMsg Info ResID %d, Json Err %v", resID, err)
+		return
+	}
+	res = string(data)
+	return
+}
+
+// Platform picks the mobi_app value to distinguish the platform to push
+func (d *Dao) Platform(c context.Context, resID string) (res []string, err error) {
+	rows, err := d.db.Query(c, _platform, resID)
+	if err != nil {
+		log.Error("db.Query(%d) error(%v)", resID, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mobiApp string
+		if err = rows.Scan(&mobiApp); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		res = append(res, mobiApp)
+	}
+	return
+}

app/job/main/appstatic/dao/push/push_test.go

@@ -0,0 +1,55 @@
+package push
+
+import (
+	"testing"
+
+	"fmt"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_CallPush(t *testing.T) {
+	Convey("Http error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(-400).JSON("")
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Business Code error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code":-400}"`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Everything is fine", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code" : 0}`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_DiffFinish(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.DiffFinish(ctx, "57")
+		So(err, ShouldBeNil)
+		So(data, ShouldBeTrue)
+	}))
+}
+
+func TestDao_PushMsg(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.PushMsg(ctx, "57")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}
+
+func TestDao_Platform(t *testing.T) {
+	Convey("TestDao_Platform", t, WithDao(func(d *Dao) {
+		data, err := d.Platform(ctx, "77")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}

app/job/main/appstatic/dao/push/redis.go

@@ -0,0 +1,46 @@
+package push
+
+import (
+	"context"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const _pushKey = "appstatic-admin-topush"
+
+// ZrangeList picks up all the to push resIDs, ctime
+func (d *Dao) ZrangeList(c context.Context) (resIDs map[string]int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	defer conn.Close()
+	// get all the resIDs in one shot
+	if err = conn.Send("ZRANGE", _pushKey, 0, -1, "WITHSCORES"); err != nil {
+		log.Error("conn.Do(ZREVRANGE, %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() err(%v)", err)
+		return
+	}
+	if resIDs, err = redis.Int64Map(conn.Receive()); err != nil {
+		log.Error("redis.Int64s()err(%v)", err)
+		return
+	}
+	return
+}
+
+// ZRem ZREM trim from trim queue.
+func (d *Dao) ZRem(c context.Context, resID string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	if _, err = conn.Do("ZREM", key, resID); err != nil {
+		log.Error("conn.Send(ZADD %s - %v) error(%v)", key, resID, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/appstatic/dao/push/redis_test.go

@@ -0,0 +1,30 @@
+package push
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_ZrangeList(t *testing.T) {
+	Convey("TestDao_ZrangeList", t, WithDao(func(d *Dao) {
+		data, err := d.ZrangeList(context.TODO())
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+		Printf("%+v", data)
+	}))
+}
+
+func TestDao_ZRem(t *testing.T) {
+	var (
+		conn = d.redis.Get(ctx)
+		key  = _pushKey
+		id   = "999"
+	)
+	Convey("everything is fine", t, WithDao(func(d *Dao) {
+		conn.Do("ZADD", key, id)
+		err := d.ZRem(ctx, id)
+		So(err, ShouldBeNil)
+	}))
+}