Create & Init Project...

app/job/main/appstatic/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/appstatic/cmd:all-srcs",
+        "//app/job/main/appstatic/conf:all-srcs",
+        "//app/job/main/appstatic/dao:all-srcs",
+        "//app/job/main/appstatic/http:all-srcs",
+        "//app/job/main/appstatic/model:all-srcs",
+        "//app/job/main/appstatic/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/CHANGELOG.md

@@ -0,0 +1,13 @@
+# appstatic-job
+
+### v1.0.2
+1. 刷完app-resource的GRPC后sleep 2秒
+
+### v1.0.1
+1. 新增接broadcast逻辑：
+* 拆分dao层，分出cal_diff（增量包计算）和push（请求broadcast推送）两个dao的包出来
+* 对接app-resource，计算增量包完成后请求app-resource刷新缓存，成功后再请求broadcast推送
+
+### v1.0.0
+1. 项目初始化，从appstatic-admin中迁移出增量包计算逻辑
+

app/job/main/appstatic/CONTRIBUTORS.md

@@ -0,0 +1,12 @@
+# Owner
+renwei
+liweijia
+peiyifei
+
+# Author 
+zhaoshichen
+
+# Reviewer
+liuxuan
+wuhao
+guanyanliang

app/job/main/appstatic/OWNERS

@@ -0,0 +1,21 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- liweijia
+- peiyifei
+- renwei
+- zhaoshichen
+- zhapuyu
+labels:
+- job
+- job/main/appstatic
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- guanyanliang
+- liuxuan
+- liweijia
+- renwei
+- wuhao
+- zhaoshichen

app/job/main/appstatic/README.md

@@ -0,0 +1,10 @@
+#### appstatic-job
+
+##### 项目简介
+> 1.静态资源管理job
+
+##### 编译环境
+> 请只用golang v1.7.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  

app/job/main/appstatic/cmd/BUILD

@@ -0,0 +1,41 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["appstatic-job-test.toml"],
+    importpath = "go-common/app/job/main/appstatic/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/http:go_default_library",
+        "//app/job/main/appstatic/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/cmd/appstatic-job-test.toml

@@ -0,0 +1,80 @@
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/appstatic-job.pid"
+dir = "./"
+perf = "0.0.0.0:7020"
+family = "appstatic-job"
+
+[log]
+    dir = "/data/log/appstatic-job/"
+    stdout = true
+
+[cfg]
+    [cfg.diff]
+        FreDiff = "30s"
+        folder = "/tmp"
+        Retry = "5 MINUTE"
+    [cfg.push]
+        qps = 1000
+        operation = 1001
+        url = "http://api.bilibili.co/x/internal/broadcast/push/all"
+        timeout = 30
+        fre = "2s"
+        Pause = "2s"
+    [cfg.grpc]
+        ApiAppID = "app.resource"
+        Method = "/app.resource.v1.AppResource/ModuleUpdateCache"
+
+[HTTPServer]
+    addr = "0.0.0.0:7021"
+    maxListen = 1000
+    timeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+
+[HTTPClient]
+    dial      = "1s"
+    timeout   = "3s"
+    keepAlive = "60s"
+    key = "f265dcfa28272742"
+    secret = "437facc22dc8698b5544669bcc12348d"
+    [HTTPClient.breaker]
+    window  ="10s"
+    sleep   ="10ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[mysql]
+    addr = "172.16.33.205"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_show?timeout=1m&readTimeout=1m&writeTimeout=1m&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 10
+    idle = 5
+    queryTimeout = "1m"
+    execTimeout = "1m"
+    tranTimeout = "1m"
+    [mysql.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+        name = "appstatic-job"
+        proto = "tcp"
+        addr = "172.18.33.61:6895"
+        active = 10
+        idle = 5
+        dialTimeout = "1s"
+        readTimeout = "1s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+        expire = "24h"
+
+[bfs]
+    key    = "fed7d6b5948c614f"
+    secret = "280e9140721cff3879cd13f59fc28a"
+    host   = "http://uat-bfs.bilibili.co"
+    timeout = 1000
+

app/job/main/appstatic/cmd/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/app/job/main/appstatic/http"
+	"go-common/app/job/main/appstatic/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	defer log.Close()
+	log.Info("appstatic-job start")
+	srv := service.New(conf.Conf)
+	http.Init(conf.Conf, srv)
+	initSignal(srv)
+}
+
+func initSignal(srv *service.Service) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("appstatic-job get a signal: %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			if err := srv.Close(); err != nil {
+				log.Error("srv close consumer error(%v)", err)
+			}
+			return
+		case syscall.SIGHUP:
+			// TODO: reload
+		default:
+			return
+		}
+	}
+}

app/job/main/appstatic/conf/BUILD

@@ -0,0 +1,37 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/main/appstatic/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/conf/conf.go

@@ -0,0 +1,118 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	xlog "go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc/warden"
+	"go-common/library/net/trace"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// Config .
+type Config struct {
+	Log          *xlog.Config
+	Tracer       *trace.Config
+	HTTPServer   *bm.ServerConfig
+	HTTPClient   *bm.ClientConfig
+	AppresClient *warden.ClientConfig
+	MySQL        *sql.Config
+	Cfg          *Cfg // push cfg.
+	Redis        *redis.Config
+	Bfs          *Bfs
+}
+
+// Cfg .
+type Cfg struct {
+	Push *PushCfg
+	Diff *DiffCfg
+	Grpc *GrpcCfg
+}
+
+// DiffCfg represents the diff calc config
+type DiffCfg struct {
+	FreDiff xtime.Duration // diff calculation frequency
+	Folder  string
+	Retry   string
+}
+
+// Bfs represents  the bfs config
+type Bfs struct {
+	Key     string
+	Secret  string
+	Host    string
+	Timeout int
+	OldURL  string
+	NewURL  string
+}
+
+// PushCfg def.
+type PushCfg struct {
+	QPS       int            // qps limit
+	Operation int            // operation number
+	URL       string         // push url
+	Timeout   int64          // timeout
+	Fre       xtime.Duration // frequency
+	Pause     xtime.Duration // pause between call app-resource and broadcast
+}
+
+// GrpcCfg def.
+type GrpcCfg struct {
+	ApiAppID string
+	Method   string
+}
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init .
+func Init() (err error) {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	err = load()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/main/appstatic/dao/BUILD

@@ -0,0 +1,25 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/main/appstatic/dao/caldiff:all-srcs",
+        "//app/job/main/appstatic/dao/push:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/caldiff/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cal_diff_test.go",
+        "dao_test.go",
+        "download_test.go",
+        "upbfs_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cal_diff.go",
+        "dao.go",
+        "download.go",
+        "upbfs.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/caldiff",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/caldiff/cal_diff.go

@@ -0,0 +1,110 @@
+package caldiff
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+const (
+	_calDiffFmt = "SELECT id,`name`,type, md5, size, url, resource_id, file_type, from_ver FROM resource_file " +
+		"WHERE file_type = ? AND url = ? AND is_deleted = 0 %s ORDER BY id DESC %s"
+	_saveFile  = "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	_parseRes  = "SELECT id, `name`, version, pool_id FROM resource WHERE %s"
+	_parseFile = "SELECT id, `name`, type, md5, size, url, resource_id, file_type, from_ver FROM resource_file WHERE" +
+		" %s AND is_deleted = 0 %s"
+	_updateStatus = "UPDATE resource_file SET file_type = ? WHERE id = ?"
+	_diffPkg      = 1
+)
+
+var (
+	_calDiffNew   = fmt.Sprintf(_calDiffFmt, " AND ctime = mtime", "LIMIT 1")
+	_parseResID   = fmt.Sprintf(_parseRes, " id = ?")
+	_parseResVer  = fmt.Sprintf(_parseRes, " pool_id = ? AND version = ?")
+	_getReadyFile = fmt.Sprintf(_parseFile, " resource_id = ? AND file_type = ? AND url != ?", "LIMIT 1")
+)
+
+// UpdateStatus updates the file's status
+func (d *Dao) UpdateStatus(c context.Context, status int, id int) (err error) {
+	if _, err = d.db.Exec(c, _updateStatus, status, id); err != nil {
+		log.Error("UpdateStatus ID %d, Err %v", id, err)
+	}
+	return
+}
+
+// ReadyFile takes the already generated file
+func (d *Dao) ReadyFile(c context.Context, resID int, ftype int) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _getReadyFile, resID, ftype, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _getReadyFile, resID, ftype, err)
+	}
+	return
+}
+
+// ParseResVer takes one resource info
+func (d *Dao) ParseResVer(c context.Context, poolID int, version int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResVer, poolID, version)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE pool_id = ? AND version = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _parseResVer, poolID, version, err)
+	}
+	return
+}
+
+// ParseResID takes one resource info
+func (d *Dao) ParseResID(c context.Context, resID int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResID, resID)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE id = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d) error(%v)", _parseResID, resID, err)
+	}
+	return
+}
+
+// DiffNew picks the recently created diff packages
+func (d *Dao) DiffNew(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _calDiffNew, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// DiffRetry picks the recently created diff packages
+func (d *Dao) DiffRetry(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	query := fmt.Sprintf(_calDiffFmt, " AND ctime != mtime AND mtime < date_sub(now(), INTERVAL "+d.c.Cfg.Diff.Retry+")", "LIMIT 1")
+	row := d.db.QueryRow(c, query, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// SaveFile saves the file info
+func (d *Dao) SaveFile(c context.Context, fileID int, file *model.FileInfo) (err error) {
+	// "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	if _, err = d.db.Exec(c, _saveFile, file.URL, _diffPkg, file.Md5, file.Size, file.Name, fileID); err != nil {
+		log.Error("SaveFile ID %d, Err %v", fileID, err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/cal_diff_test.go

@@ -0,0 +1,94 @@
+package caldiff
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/appstatic/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+const (
+	_availableRes = "SELECT id FROM resource ORDER BY id DESC"
+)
+
+func TestDao_DiffNew(t *testing.T) {
+	Convey("TestDao_DiffNew", t, WithDao(func(d *Dao) {
+		file, err := d.DiffNew(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_DiffRetry(t *testing.T) {
+	Convey("TestDao_DiffRetry", t, WithDao(func(d *Dao) {
+		file, err := d.DiffRetry(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_SaveFile(t *testing.T) {
+	Convey("TestDao_SaveFile", t, WithDao(func(d *Dao) {
+		err := d.SaveFile(ctx, 1, &model.FileInfo{
+			Name: "123",
+			Size: 123,
+			Type: "1",
+			Md5:  "1234",
+			URL:  "xxx",
+		})
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_ParseResID(t *testing.T) {
+	Convey("TestDao_ParseResID", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		res, err := d.ParseResID(ctx, int(r.ID))
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+		fmt.Println(r.ID)
+	}))
+}
+
+func TestDao_ParseResVer(t *testing.T) {
+	Convey("TestDao_ParseResVer", t, WithDao(func(d *Dao) {
+		dd, err := d.ParseResVer(ctx, 23, 1)
+		So(err, ShouldBeNil)
+		fmt.Println(err)
+		So(dd, ShouldNotBeNil)
+		data, err2 := (json.Marshal(dd))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_ReadyFile(t *testing.T) {
+	Convey("TestDao_ReadyFile", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		dd, err := d.ReadyFile(ctx, int(r.ID), 0) // ftype = 0 full package
+		So(err, ShouldBeNil)
+		So(dd, ShouldNotBeNil)
+		data, _ := json.Marshal(dd)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_UpdateStatus(t *testing.T) {
+	Convey("TestDao_UpdateStatus", t, WithDao(func(d *Dao) {
+		err := d.UpdateStatus(ctx, 2, 304)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/appstatic/dao/caldiff/dao.go

@@ -0,0 +1,24 @@
+package caldiff
+
+import (
+	"go-common/app/job/main/appstatic/conf"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c      *conf.Config
+	db     *xsql.DB
+	client *bm.Client
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/dao_test.go

@@ -0,0 +1,53 @@
+package caldiff
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/caldiff/download.go

@@ -0,0 +1,42 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"go-common/library/log"
+)
+
+const (
+	errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
+)
+
+// DownloadFile downloads one file from url to local
+func (d *Dao) DownloadFile(ctx context.Context, api string, fileName string) (bts int64, err error) {
+	var (
+		res []byte
+		f   *os.File
+		req *http.Request
+	)
+	if req, err = d.client.NewRequest(http.MethodGet, api, "", url.Values{}); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if res, err = d.client.Raw(ctx, req); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if f, err = os.Create(fileName); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("CreateFile(%s)", fileName), err)
+		return
+	}
+	if bts, err = io.Copy(f, bytes.NewReader(res)); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("SaveFile(%s)", fileName), err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/download_test.go

@@ -0,0 +1,29 @@
+package caldiff
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_DownloadFile(t *testing.T) {
+	var (
+		newPath = fmt.Sprintf("%s/%s", d.c.Cfg.Diff.Folder, "123.zip")
+		url     = "http://i0.hdslb.com/bfs/face/ca14680bcfe4a956d1b6c06fbc1f6a6529257746.jpg"
+	)
+	Convey("TestDao_DownloadFile", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		data, err := d.DownloadFile(ctx, url, newPath)
+		So(err, ShouldBeNil)
+		So(data, ShouldBeGreaterThan, 0)
+		fmt.Println(data)
+	}))
+	Convey("CreateFile Error", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		_, err := d.DownloadFile(ctx, url, "/test/test.txt")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+}

app/job/main/appstatic/dao/caldiff/upbfs.go

@@ -0,0 +1,89 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// bfs info
+const (
+	_uploadURL = "/bfs/%s/%s"
+	_template  = "%s\n%s\n%s\n%d\n"
+	_method    = "PUT"
+	_bucket    = "app-static"
+)
+
+// Upload upload picture or log file to bfs
+func (d *Dao) Upload(c context.Context, fileName string, fileType string, timing int64, data []byte, bfs *conf.Bfs) (location string, err error) {
+	var (
+		req    *http.Request
+		resp   *http.Response
+		code   int
+		client = &http.Client{Timeout: time.Duration(bfs.Timeout) * time.Millisecond}
+		url    = fmt.Sprintf(bfs.Host+_uploadURL, _bucket, fileName)
+	)
+	// prepare the data of the file and init the request
+	buf := new(bytes.Buffer)
+	_, err = buf.Write(data)
+	if err != nil {
+		log.Error("Upload.buf.Write.error(%v)", err)
+		err = ecode.RequestErr
+		return
+	}
+	if req, err = http.NewRequest(_method, url, buf); err != nil {
+		log.Error("http.NewRequest() Upload(%v) error(%v)", url, err)
+		return
+	}
+	// request setting
+	authorization := authorize(bfs.Key, bfs.Secret, _method, _bucket, fileName, timing)
+	req.Header.Set("Date", fmt.Sprint(timing))
+	req.Header.Set("Authorization", authorization)
+	req.Header.Set("Content-Type", fileType)
+	resp, err = client.Do(req)
+	// response treatment
+	if err != nil {
+		log.Error("Bfs client.Do(%s) error(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err = errors.Wrap(ecode.Int(resp.StatusCode), "Bfs Status Code Error")
+		return
+	}
+	code, err = strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = errors.Wrap(ecode.Int(code), "Bfs Header Code Error")
+		return
+	}
+	location = resp.Header.Get("Location")
+	return
+}
+
+// authorize returns authorization for upload file to bfs
+func authorize(key, secret, method, bucket, file string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf(_template, method, bucket, file, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}

app/job/main/appstatic/dao/caldiff/upbfs_test.go

@@ -0,0 +1,57 @@
+package caldiff
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoUpload(t *testing.T) {
+	var (
+		c        = context.Background()
+		fileName = "test.txt"
+		fileType = "txt"
+		timing   = time.Now().Unix()
+		bfs      = d.c.Bfs
+		data     = []byte("123")
+		url      = fmt.Sprintf(bfs.Host+_uploadURL, "app-static", fileName)
+	)
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		ctx.Convey("http code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(-400)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("business code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).JSON(`{"code":-400}`)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("everything is fine", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).SetHeader("Location", "test").SetHeader("code", "200").JSON(`{"code":200}`)
+			location, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(location, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoauthorize(t *testing.T) {
+	var (
+		key    = "key"
+		secret = "secret"
+		method = "put"
+		bucket = "tv-cover"
+		file   = "file"
+		expire = int64(0)
+	)
+	convey.Convey("authorize", t, func(ctx convey.C) {
+		authorization := authorize(key, secret, method, bucket, file, expire)
+		ctx.Convey("Then authorization should not be nil.", func(ctx convey.C) {
+			ctx.So(authorization, convey.ShouldNotBeNil)
+		})
+	})
+}

app/job/main/appstatic/dao/push/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "appres_test.go",
+        "dao_test.go",
+        "push_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "appres.go",
+        "dao.go",
+        "push.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/push",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/app-resource/api/v1:go_default_library",
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/push/appres.go

@@ -0,0 +1,95 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/library/log"
+	"go-common/library/naming/discovery"
+	"go-common/library/net/rpc/warden"
+)
+
+func errlog(step string, err error) {
+	log.Error("CallPush Step [%s] Err [%v]", step, err)
+}
+
+// build grpc client for app-resource
+func (d *Dao) grpcClient(addr string) (appres.AppResourceClient, error) {
+	client := warden.NewClient(d.c.AppresClient)
+	cc, err := client.Dial(context.Background(), addr)
+	if err != nil {
+		return nil, err
+	}
+	return appres.NewAppResourceClient(cc), nil
+}
+
+// CallRefresh picks ip addrs from discovery, and call grpc method
+func (d *Dao) CallRefresh(ctx context.Context) (err error) {
+	var (
+		addrs    []string
+		client   appres.AppResourceClient
+		arg      = &appres.NoArgRequest{}
+		succCall int
+	)
+	if addrs, err = d.pickAddrs(); err != nil {
+		errlog("pickAddrs", err)
+		return
+	}
+	// loop grpc call, ignore error
+	for _, addr := range addrs {
+		if client, err = d.grpcClient(addr); err != nil {
+			errlog(fmt.Sprintf("grpcDial Addr [%s]", addr), err)
+			continue
+		}
+		if _, err = client.ModuleUpdateCache(ctx, arg); err != nil {
+			errlog(fmt.Sprintf("grpcCall Addr [%s] ", addr), err)
+			continue
+		}
+		succCall++
+	}
+	if succCall == 0 { // addrs must be greater than 0, if succ call is zero, return error
+		return fmt.Errorf("CallRefresh Addrs [%d], Zero Succ!", len(addrs))
+	}
+	log.Info("CallPush Refresh Succ [%d], Total [%d]", succCall, len(addrs))
+	err = nil
+	return
+}
+
+// pick all the app-resource grpc instances from discovery
+func (d *Dao) pickAddrs() (grpcAddrs []string, err error) {
+	dis := discovery.New(nil)
+	defer dis.Close()
+	b := dis.Build(d.c.Cfg.Grpc.ApiAppID)
+	defer b.Close()
+	ch := b.Watch()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100)
+	defer cancel()
+	select {
+	case <-ch:
+	case <-ctx.Done():
+		err = fmt.Errorf("查找节点超时 请检查appid是否填写正确")
+		return
+	}
+	ins, ok := b.Fetch(context.Background())
+	if !ok {
+		err = fmt.Errorf("discovery 拉取失败")
+		return
+	}
+	for _, vs := range ins {
+		for _, v := range vs {
+			for _, addr := range v.Addrs {
+				if strings.Contains(addr, "grpc://") {
+					ip := strings.Replace(addr, "grpc://", "", -1)
+					grpcAddrs = append(grpcAddrs, ip)
+				}
+			}
+		}
+	}
+	if len(grpcAddrs) == 0 {
+		err = fmt.Errorf("discovery 找不到服务节点")
+	}
+	return
+}

app/job/main/appstatic/dao/push/appres_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go-common/app/interface/main/app-resource/api/v1"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestPusherrlog(t *testing.T) {
+	var (
+		step = ""
+		err  error
+	)
+	convey.Convey("errlog", t, func(ctx convey.C) {
+		errlog(step, err)
+		ctx.Convey("No return values", func(ctx convey.C) {
+		})
+	})
+}
+
+func TestPushgrpcClient(t *testing.T) {
+	var (
+		grpcAddrs []string
+		err       error
+		p1        v1.AppResourceClient
+	)
+	convey.Convey("pickAddrs", t, func(ctx convey.C) {
+		grpcAddrs, err = d.pickAddrs()
+		ctx.Convey("Then err should be nil.grpcAddrs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(grpcAddrs, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("grpcClient", t, func(ctx convey.C) {
+		fmt.Println("Call ", grpcAddrs[0])
+		p1, err = d.grpcClient(grpcAddrs[0])
+		ctx.Convey("Then err should be nil.p1 should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(p1, convey.ShouldNotBeNil)
+		})
+	})
+	convey.Convey("callRefresh", t, func(ctx convey.C) {
+		err = d.CallRefresh(context.Background())
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+
+}

app/job/main/appstatic/dao/push/dao.go

@@ -0,0 +1,33 @@
+package push
+
+import (
+	appres "go-common/app/interface/main/app-resource/api/v1"
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c         *conf.Config
+	db        *xsql.DB
+	client    *bm.Client
+	redis     *redis.Pool
+	appresCli appres.AppResourceClient
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+		redis:  redis.NewPool(c.Redis),
+	}
+	var err error
+	if d.appresCli, err = appres.NewClient(c.AppresClient); err != nil {
+		panic(err)
+	}
+	return
+}

app/job/main/appstatic/dao/push/dao_test.go

@@ -0,0 +1,53 @@
+package push
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/push/push.go

@@ -0,0 +1,94 @@
+package push
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_diffFinish = "SELECT COUNT(1) FROM resource_file WHERE resource_id = ? AND url = ? AND file_type = ? AND is_deleted = 0"
+	_pushMsg    = "SELECT resource.id, resource.pool_id, resource_pool.`name` FROM resource LEFT JOIN resource_pool ON resource.pool_id = resource_pool.id WHERE resource.id = ? LIMIT 1"
+	_platform   = "SELECT b.value FROM resource_config a LEFT JOIN resource_limit b ON a.id = b.config_id WHERE a.resource_id = ? AND b.`column` = 'mobi_app' AND a.is_deleted = 0 AND b.is_deleted = 0"
+	_diffPkg    = 1
+)
+
+// CallPush calls the push server api
+func (d *Dao) CallPush(ctx context.Context, platform string, msg string, ip string) (err error) {
+	var (
+		cfg    = d.c.Cfg.Push
+		params = url.Values{}
+	)
+	params.Set("operation", fmt.Sprintf("%d", cfg.Operation))
+	params.Set("platform", platform)
+	params.Set("message", msg)
+	params.Set("speed", fmt.Sprintf("%d", cfg.QPS))
+	var res struct {
+		Code int `json:"code"`
+	}
+	if err = d.client.Post(ctx, cfg.URL, ip, params, &res); err != nil {
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		err = errors.Wrap(ecode.Int(res.Code), cfg.URL+"?"+params.Encode())
+	}
+	return
+}
+
+// DiffFinish checks whether the resource's diff calculation has been finished or not
+func (d *Dao) DiffFinish(c context.Context, resID string) (res bool, err error) {
+	count := 0
+	row := d.db.QueryRow(c, _diffFinish, resID, "", _diffPkg)
+	if err = row.Scan(&count); err != nil {
+		log.Error("d.DiffFinish err(%v)", err)
+		return
+	}
+	if count == 0 {
+		res = true
+	}
+	return
+}
+
+// PushMsg combines the resource pool info to prepare the msg to call PUSH
+func (d *Dao) PushMsg(c context.Context, resID string) (res string, err error) {
+	var (
+		msg  model.PushMsg
+		data []byte
+	)
+	row := d.db.QueryRow(c, _pushMsg, resID)
+	if err = row.Scan(&msg.ResID, &msg.ModID, &msg.ModName); err != nil {
+		log.Error("d.PushMsg err(%v)", err)
+	}
+	if data, err = json.Marshal(msg); err != nil {
+		log.Error("PushMsg Info ResID %d, Json Err %v", resID, err)
+		return
+	}
+	res = string(data)
+	return
+}
+
+// Platform picks the mobi_app value to distinguish the platform to push
+func (d *Dao) Platform(c context.Context, resID string) (res []string, err error) {
+	rows, err := d.db.Query(c, _platform, resID)
+	if err != nil {
+		log.Error("db.Query(%d) error(%v)", resID, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mobiApp string
+		if err = rows.Scan(&mobiApp); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		res = append(res, mobiApp)
+	}
+	return
+}

app/job/main/appstatic/dao/push/push_test.go

@@ -0,0 +1,55 @@
+package push
+
+import (
+	"testing"
+
+	"fmt"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_CallPush(t *testing.T) {
+	Convey("Http error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(-400).JSON("")
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Business Code error", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code":-400}"`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+	Convey("Everything is fine", t, WithDao(func(d *Dao) {
+		httpMock("POST", d.c.Cfg.Push.URL).Reply(200).JSON(`{"code" : 0}`)
+		err := d.CallPush(ctx, "ios", "", "")
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_DiffFinish(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.DiffFinish(ctx, "57")
+		So(err, ShouldBeNil)
+		So(data, ShouldBeTrue)
+	}))
+}
+
+func TestDao_PushMsg(t *testing.T) {
+	Convey("TestDao_DiffFinish", t, WithDao(func(d *Dao) {
+		data, err := d.PushMsg(ctx, "57")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}
+
+func TestDao_Platform(t *testing.T) {
+	Convey("TestDao_Platform", t, WithDao(func(d *Dao) {
+		data, err := d.Platform(ctx, "77")
+		fmt.Println(data)
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+	}))
+}

app/job/main/appstatic/dao/push/redis.go

@@ -0,0 +1,46 @@
+package push
+
+import (
+	"context"
+
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const _pushKey = "appstatic-admin-topush"
+
+// ZrangeList picks up all the to push resIDs, ctime
+func (d *Dao) ZrangeList(c context.Context) (resIDs map[string]int64, err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	defer conn.Close()
+	// get all the resIDs in one shot
+	if err = conn.Send("ZRANGE", _pushKey, 0, -1, "WITHSCORES"); err != nil {
+		log.Error("conn.Do(ZREVRANGE, %s) error(%v)", key, err)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush() err(%v)", err)
+		return
+	}
+	if resIDs, err = redis.Int64Map(conn.Receive()); err != nil {
+		log.Error("redis.Int64s()err(%v)", err)
+		return
+	}
+	return
+}
+
+// ZRem ZREM trim from trim queue.
+func (d *Dao) ZRem(c context.Context, resID string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+		key  = _pushKey
+	)
+	if _, err = conn.Do("ZREM", key, resID); err != nil {
+		log.Error("conn.Send(ZADD %s - %v) error(%v)", key, resID, err)
+	}
+	conn.Close()
+	return
+}

app/job/main/appstatic/dao/push/redis_test.go

@@ -0,0 +1,30 @@
+package push
+
+import (
+	"context"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_ZrangeList(t *testing.T) {
+	Convey("TestDao_ZrangeList", t, WithDao(func(d *Dao) {
+		data, err := d.ZrangeList(context.TODO())
+		So(err, ShouldBeNil)
+		So(data, ShouldNotBeNil)
+		Printf("%+v", data)
+	}))
+}
+
+func TestDao_ZRem(t *testing.T) {
+	var (
+		conn = d.redis.Get(ctx)
+		key  = _pushKey
+		id   = "999"
+	)
+	Convey("everything is fine", t, WithDao(func(d *Dao) {
+		conn.Do("ZADD", key, id)
+		err := d.ZRem(ctx, id)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/appstatic/http/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/main/appstatic/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/http/http.go

@@ -0,0 +1,29 @@
+package http
+
+import (
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/app/job/main/appstatic/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var apsSrv *service.Service
+
+// Init .
+func Init(c *conf.Config, srv *service.Service) {
+	apsSrv = srv
+	engineIn := bm.DefaultServer(c.HTTPServer)
+	route(engineIn)
+	// init inner server
+	if err := engineIn.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+func route(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+func ping(c *bm.Context) {
+}

app/job/main/appstatic/model/BUILD

@@ -0,0 +1,27 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/main/appstatic/model",
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/model/model.go

@@ -0,0 +1,45 @@
+package model
+
+import "time"
+
+// PushMsg is used to push to the mobile clients to indicate the mod name to request
+type PushMsg struct {
+	ResID   int    `json:"res_id" gorm:"column:id"`
+	ModID   int    `json:"mod_id" gorm:"column:pool_id"`
+	ModName string `json:"mod_name" gorm:"column:name"`
+}
+
+// ResourceFile represents the table structure
+type ResourceFile struct {
+	ID         int       `json:"id"`
+	Name       string    `json:"name"`
+	Type       string    `json:"type"`
+	Md5        string    `json:"md5"`
+	Size       int       `json:"size"`
+	URL        string    `json:"url"`
+	ResourceID int       `json:"resource_id"`
+	Ctime      time.Time `json:"ctime"`
+	Mtime      time.Time `json:"mtime"`
+	FileType   int8      `json:"file_type"`
+	FromVer    int64     `json:"from_ver"`
+	IsDeleted  int8      `json:"is_deleted"`
+}
+
+//FileInfo : the uploaded file information
+type FileInfo struct {
+	Name string `json:"name"`
+	Size int64  `json:"size"`
+	Type string `json:"type"`
+	Md5  string `json:"md5"`
+	URL  string `json:"url"`
+}
+
+// Resource reprensents the resource table
+type Resource struct {
+	ID      int64     `json:"id" params:"id"`
+	Name    string    `json:"name" params:"name"`
+	Version int64     `json:"version" params:"version"`
+	PoolID  int64     `json:"pool_id" params:"pool_id"`
+	Ctime   time.Time `json:"ctime" params:"ctime"`
+	Mtime   time.Time `json:"mtime" params:"mtime"`
+}

app/job/main/appstatic/service/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cal_diff.go",
+        "push.go",
+        "service.go",
+        "upbfs.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/dao/caldiff:go_default_library",
+        "//app/job/main/appstatic/dao/push:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/service/cal_diff.go

@@ -0,0 +1,255 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+const (
+	errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
+	_zipType  = "application/zip"
+	_tmpSlash = "*"
+	// file type
+	_fullPackage           = 0
+	_diffPackge            = 1
+	_calculationInProgress = 2
+)
+
+// Sync modified season data to the license owner
+func (s *Service) calDiffproc() {
+	defer s.waiter.Done()
+	var (
+		file *model.ResourceFile
+		err  error
+	)
+	for {
+		if s.daoClosed {
+			log.Info("DB closed!")
+			return
+		}
+		time.Sleep(time.Duration(s.c.Cfg.Diff.FreDiff))
+		// check recently created files
+		if file, err = s.dao.DiffNew(ctx); err != nil {
+			log.Error(errFormat, "callDiffproc", "diffNew", err)
+			continue
+		}
+		// if recently created file not found, we retry failed files
+		if file == nil {
+			if file, err = s.dao.DiffRetry(ctx); err != nil {
+				log.Error(errFormat, "callDiffproc", "DiffRetry", err)
+				continue
+			}
+			if file == nil {
+				log.Error(errFormat, "callDiffproc", "DiffRetry", "No Diff To Calculate")
+				continue
+			}
+		}
+		// begin calculate
+		log.Info("Calculate Diff for (fileID:%d, ResID: %d, FileName:%s)", file.ID, file.ResourceID, file.Name)
+		if err := s.calDiff(file); err != nil {
+			log.Error(errFormat, "calDiffproc", "calDiff", err)
+			continue
+		}
+	}
+}
+
+// get the file info from DB and download the file to local
+func (s *Service) catchFile(file *model.ResourceFile, version int) (newPath string, err error) {
+	var (
+		size   int64
+		newPkg *model.ResourceFile
+	)
+	// get the new package info
+	if newPkg, err = s.getFile(file.ResourceID, _fullPackage, version); err != nil {
+		log.Error(errFormat, "calDiff", fmt.Sprintf("getFileNew %v", file), err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	// save the new package
+	newPkg.Name = strings.Replace(newPkg.Name, "/", _tmpSlash, 1)    // avoid open file error
+	newPath = fmt.Sprintf("%s/%s", s.c.Cfg.Diff.Folder, newPkg.Name) // combine the local path
+	if size, err = s.dao.DownloadFile(ctx, newPkg.URL, newPath); err != nil {
+		log.Error(errFormat, "calDiff", fmt.Sprintf("saveFile %v", file), err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	// check result file
+	if _, err = os.Stat(newPath); os.IsNotExist(err) {
+		log.Error(errFormat, "diffCmd", "IsNotExist", newPath+" - File Not Exist")
+		return
+	}
+	log.Info("Save File From URL [%s] in [%s], Size: %d", newPkg.URL, newPath, size)
+	return
+}
+
+// calculate the diff for one file struct and upload the result to BFS and fill the URL
+func (s *Service) calDiff(file *model.ResourceFile) (err error) {
+	var (
+		newPath    string
+		oldPath    string
+		patchPath  string
+		patchURL   string
+		patchFInfo *model.FileInfo
+	)
+	// update the status of the file, to avoid being picked by another
+	if err = s.chgFStatus(file.ID, _calculationInProgress); err != nil {
+		log.Error(errFormat, "calDiff", "chgFStatus", err)
+		return
+	}
+	// save the new file
+	if newPath, err = s.catchFile(file, 0); err != nil {
+		log.Error(errFormat, "callDiff", "catchFile", err)
+		return
+	}
+	// save the old file
+	if oldPath, err = s.catchFile(file, int(file.FromVer)); err != nil {
+		log.Error(errFormat, "callDiff", "catchFile", err)
+		return
+	}
+	// exec bsdiff to get the patch file and upload it
+	file.Name = strings.Replace(file.Name, "/", _tmpSlash, 1)
+	patchPath = s.c.Cfg.Diff.Folder + "/" + file.Name
+	if patchFInfo, patchURL, err = s.diffCmd(file.Name, patchPath, newPath, oldPath); err != nil {
+		log.Error(errFormat, "calDiff", "diffCmd", err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	log.Info("Upload Path File From [%s] to [%s], Size: %d", patchPath, patchURL)
+	// save the url to the file
+	if err = s.dao.SaveFile(ctx, file.ID, &model.FileInfo{
+		Name: patchFInfo.Name,
+		Size: patchFInfo.Size,
+		Md5:  patchFInfo.Md5,
+		URL:  patchURL}); err != nil {
+		log.Error(errFormat, "calDiff", "updateURL", err)
+		return
+	}
+	// delete all the packages used
+	if err = delPkgs(newPath, oldPath, patchPath); err != nil {
+		log.Error(errFormat, "calDiff", "delPkgs", err)
+	}
+	return
+}
+
+// delete all the packages used to generate the diff pkg
+func delPkgs(newPath string, oldPath string, patchPath string) (err error) {
+	if err = deleteFile(newPath); err != nil {
+		log.Error(errFormat, "delPkgs", "NewPath", err)
+		return
+	}
+	if err = deleteFile(oldPath); err != nil {
+		log.Error(errFormat, "delPkgs", "oldPath", err)
+		return
+	}
+	if err = deleteFile(patchPath); err != nil {
+		log.Error(errFormat, "delPkgs", "patchPath", err)
+	}
+	return
+}
+
+// delete one file
+func deleteFile(path string) (err error) {
+	// check file
+	if _, err = os.Stat(path); os.IsNotExist(err) {
+		err = fmt.Errorf("File %s Not exist", path)
+		log.Error(errFormat, "deleteFile", "IsNotExist", "File Not Exist")
+		return
+	}
+	if err = os.Remove(path); err != nil {
+		log.Error(errFormat, "deleteFile", "Remove", err)
+		return
+	}
+	log.Info("Delete %s Succesfully", path)
+	return
+}
+
+// execute the bsdiff command to have the result
+func (s *Service) diffCmd(patchName string, patchPath string, newPath string, oldPath string) (patchFInfo *model.FileInfo, location string, err error) {
+	var (
+		content []byte
+		fInfo   os.FileInfo
+	)
+	begin := time.Now()
+	cmd := exec.Command("bsdiff", oldPath, newPath, patchPath)
+	// exec Command
+	if err = cmd.Run(); err != nil {
+		log.Error(errFormat, "diffCmd", "cmdRun", err)
+		return
+	}
+	timeCost := time.Since(begin)
+	log.Info("BSDiff Command Finished. Time Cost: %v, Cmd:%s %s %s %s", timeCost, "bsdiff", oldPath, newPath, patchPath)
+	// check patch file
+	if fInfo, err = os.Stat(patchPath); os.IsNotExist(err) {
+		log.Error(errFormat, "diffCmd", "patchFileCheck", "File Not Exist")
+		return
+	}
+	log.Info("Patch File Generated, Name: %s, Size: %s.", fInfo.Name(), fInfo.Size())
+	// read patch file and upload
+	if content, err = ioutil.ReadFile(patchPath); err != nil {
+		log.Error(errFormat, "diffCmd", "ReadFile_Patch", err)
+		return
+	}
+	if patchFInfo, err = s.ParseFile(content); err != nil {
+		log.Error(errFormat, "diffCmd", "ParsePatchFile", err)
+		return
+	}
+	patchFInfo.Name = rename(patchName, patchFInfo.Md5)
+	// upload patch file to bfs
+	location, err = s.Upload(context.Background(), patchFInfo.Name, _zipType, time.Now().Unix(), content)
+	if err != nil {
+		log.Error(errFormat, "diffCmd", "UploadPatch", err)
+	}
+	return
+}
+
+// split the patchName, pick the ModID (res[0]), VersionInfo (res[1]), Insert the md5 inside
+func rename(patchName string, md5 string) (newName string) {
+	res := strings.Split(patchName, _tmpSlash)
+	if len(res) != 2 {
+		log.Error("patchName %s can't split", patchName)
+		return patchName
+	}
+	return res[0] + "_" + md5 + "/" + res[1]
+}
+
+// get file object ( struct )
+func (s *Service) getFile(resID int, fileType int, version int) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	var (
+		res    *model.Resource // current version
+		resHis *model.Resource // history version
+		poolID int
+	)
+	if res, err = s.dao.ParseResID(ctx, resID); err != nil {
+		log.Error("[getFile]-[findPool %d]-Error(%v)", resID, err)
+		return
+	}
+	poolID = int(res.PoolID)
+	if version != 0 { // full pkg of the history version
+		if resHis, err = s.dao.ParseResVer(ctx, poolID, version); err != nil {
+			log.Error("[getFile]-[findVersion]-Error(%v)", err)
+			return
+		}
+		resID = int(resHis.ID)
+	}
+	if file, err = s.dao.ReadyFile(ctx, resID, fileType); err != nil {
+		log.Error(errFormat, "getUrl", "First", err)
+	}
+	return
+}
+
+// change the file's status
+func (s *Service) chgFStatus(fileID int, status int) (err error) {
+	if err = s.dao.UpdateStatus(ctx, status, fileID); err != nil {
+		log.Error(errFormat, "chgFStatus", "update", err)
+	}
+	return
+}

app/job/main/appstatic/service/push.go

@@ -0,0 +1,124 @@
+package service
+
+import (
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	_platIOS     = "ios"
+	_platAndroid = "android"
+	_platAll     = ""
+	_vIPad       = "ipad"
+	_vIPhone     = "iphone"
+	_vAndroid    = "android"
+	_vAndroidB   = "android_b"
+)
+
+func (s *Service) push(resIDs map[string]int64) {
+	var (
+		now     = time.Now().Unix()
+		timeout = s.c.Cfg.Push.Timeout
+		err     error
+		msg     string
+	)
+	for resID, timeV := range resIDs {
+		finish := false
+		needPush := false
+		// distinguish whether the resource is ready to push. calc finish or timeout
+		if finish, err = s.pushDao.DiffFinish(ctx, resID); err != nil { // check whether diff cal finish
+			continue
+		}
+		if now-timeV > timeout { // check whether it's already timeout
+			needPush = true
+			log.Info("CallPush [%v] Because of Timeout", resID)
+		} else if finish {
+			needPush = true
+			log.Info("CallPush [%v] Because of DiffFinish", resID)
+		} else {
+			log.Info("CallPush Jump [%v]", resID)
+			continue
+		}
+		// prepare api call
+		if msg, err = s.pushDao.PushMsg(ctx, resID); err != nil { // prepare msg
+			continue
+		}
+		if needPush {
+			if err = s.pushDao.CallRefresh(ctx); err != nil {
+				log.Error("CallPush [%d] app-resource refresh error [%v]", resID, err)
+				continue
+			}
+			time.Sleep(time.Duration(s.c.Cfg.Push.Pause))
+			if err = s.pushDao.CallPush(ctx, s.platform(resID), msg, ""); err != nil {
+				log.Error("CallPush [%v] Error [%v]", resID, err)
+				continue
+			}
+			log.Info("CallPush [%v] Succ, Platform: %s, Delete Key", resID)
+			if err = s.pushDao.ZRem(ctx, resID); err != nil {
+				continue
+			}
+		}
+	}
+}
+
+// distinguish the resource's platform info
+func (s *Service) platform(resID string) (platform string) {
+	var (
+		err          error
+		ios, android bool
+		mobiAPPs     []string
+	)
+	platform = _platAll // default value
+	if mobiAPPs, err = s.pushDao.Platform(ctx, resID); err != nil {
+		return
+	}
+	for _, value := range mobiAPPs {
+		switch value {
+		case _vAndroid:
+			android = true
+		case _vAndroidB:
+			android = true
+		case _vIPad:
+			ios = true
+		case _vIPhone:
+			ios = true
+		default:
+			log.Error("ResourceID %d, Limit Wrong Value %s", resID, value)
+		}
+	}
+	if ios && !android {
+		return _platIOS
+	}
+	if !ios && android {
+		return _platAndroid
+	}
+	return // other case like all or none, just return the default value
+}
+
+func (s *Service) pushproc() {
+	var (
+		resIDs map[string]int64
+		err    error
+	)
+	defer s.waiter.Done()
+	for {
+		if s.daoClosed {
+			log.Info("DB closed!")
+			return
+		}
+		time.Sleep(time.Duration(s.c.Cfg.Push.Fre))
+		// pick to push resIDs from redis
+		if resIDs, err = s.pushDao.ZrangeList(ctx); err != nil {
+			log.Error("Get ToPush List Err %v", err)
+			continue
+		}
+		if len(resIDs) == 0 {
+			log.Info("No ToPush Data, Sleep")
+			continue
+		}
+		// push the data
+		log.Info("ToPush Treat Data: %d", len(resIDs))
+		s.push(resIDs)
+	}
+}

app/job/main/appstatic/service/service.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/app/job/main/appstatic/dao/caldiff"
+	"go-common/app/job/main/appstatic/dao/push"
+	"go-common/library/log"
+)
+
+var ctx = context.Background()
+
+// Service .
+type Service struct {
+	c         *conf.Config
+	dao       *caldiff.Dao
+	pushDao   *push.Dao
+	waiter    *sync.WaitGroup
+	daoClosed bool
+}
+
+// New creates a Service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:       c,
+		dao:     caldiff.New(c),
+		pushDao: push.New(c),
+		waiter:  new(sync.WaitGroup),
+	}
+	s.waiter.Add(1)
+	go s.pushproc()
+	s.waiter.Add(1)
+	go s.calDiffproc()
+	return
+}
+
+// Close releases resources which owned by the Service instance.
+func (s *Service) Close() (err error) {
+	log.Info("Close dao!")
+	s.daoClosed = true
+	log.Info("Wait waiter!")
+	s.waiter.Wait()
+	log.Info("appstatic-job has been closed.")
+	return
+}

app/job/main/appstatic/service/upbfs.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"net/http"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+// ParseFile analyses file info
+func (s *Service) ParseFile(content []byte) (file *model.FileInfo, err error) {
+	fType := http.DetectContentType(content)
+	// file md5
+	md5hash := md5.New()
+	if _, err = io.Copy(md5hash, bytes.NewReader(content)); err != nil {
+		log.Error("resource uploadFile.Copy error(%v)", err)
+		return
+	}
+	md5 := md5hash.Sum(nil)
+	fMd5 := hex.EncodeToString(md5[:])
+	file = &model.FileInfo{
+		Md5:  fMd5,
+		Type: fType,
+		Size: int64(len(content)),
+	}
+	return
+}
+
+// Upload can upload a file object: store the info in Redis, and transfer the file to Bfs
+func (s *Service) Upload(c context.Context, fileName string, fileType string, timing int64, body []byte) (location string, err error) {
+	if location, err = s.dao.Upload(c, fileName, fileType, timing, body, s.c.Bfs); err != nil { // bfs
+		log.Error("s.upload.UploadBfs() error(%v)", err)
+	}
+	return
+}