Create & Init Project...

app/job/main/appstatic/dao/caldiff/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cal_diff_test.go",
+        "dao_test.go",
+        "download_test.go",
+        "upbfs_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cal_diff.go",
+        "dao.go",
+        "download.go",
+        "upbfs.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/dao/caldiff",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/dao/caldiff/cal_diff.go

@@ -0,0 +1,110 @@
+package caldiff
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+const (
+	_calDiffFmt = "SELECT id,`name`,type, md5, size, url, resource_id, file_type, from_ver FROM resource_file " +
+		"WHERE file_type = ? AND url = ? AND is_deleted = 0 %s ORDER BY id DESC %s"
+	_saveFile  = "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	_parseRes  = "SELECT id, `name`, version, pool_id FROM resource WHERE %s"
+	_parseFile = "SELECT id, `name`, type, md5, size, url, resource_id, file_type, from_ver FROM resource_file WHERE" +
+		" %s AND is_deleted = 0 %s"
+	_updateStatus = "UPDATE resource_file SET file_type = ? WHERE id = ?"
+	_diffPkg      = 1
+)
+
+var (
+	_calDiffNew   = fmt.Sprintf(_calDiffFmt, " AND ctime = mtime", "LIMIT 1")
+	_parseResID   = fmt.Sprintf(_parseRes, " id = ?")
+	_parseResVer  = fmt.Sprintf(_parseRes, " pool_id = ? AND version = ?")
+	_getReadyFile = fmt.Sprintf(_parseFile, " resource_id = ? AND file_type = ? AND url != ?", "LIMIT 1")
+)
+
+// UpdateStatus updates the file's status
+func (d *Dao) UpdateStatus(c context.Context, status int, id int) (err error) {
+	if _, err = d.db.Exec(c, _updateStatus, status, id); err != nil {
+		log.Error("UpdateStatus ID %d, Err %v", id, err)
+	}
+	return
+}
+
+// ReadyFile takes the already generated file
+func (d *Dao) ReadyFile(c context.Context, resID int, ftype int) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _getReadyFile, resID, ftype, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _getReadyFile, resID, ftype, err)
+	}
+	return
+}
+
+// ParseResVer takes one resource info
+func (d *Dao) ParseResVer(c context.Context, poolID int, version int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResVer, poolID, version)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE pool_id = ? AND version = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d,%d) error(%v)", _parseResVer, poolID, version, err)
+	}
+	return
+}
+
+// ParseResID takes one resource info
+func (d *Dao) ParseResID(c context.Context, resID int) (res *model.Resource, err error) {
+	res = &model.Resource{}
+	row := d.db.QueryRow(c, _parseResID, resID)
+	// "SELECT id, `name`, version, pool_id FROM resource WHERE id = ?"
+	if err = row.Scan(&res.ID, &res.Name, &res.Version, &res.PoolID); err != nil {
+		log.Error("db.QueryRow(%s) (%d) error(%v)", _parseResID, resID, err)
+	}
+	return
+}
+
+// DiffNew picks the recently created diff packages
+func (d *Dao) DiffNew(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	row := d.db.QueryRow(c, _calDiffNew, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// DiffRetry picks the recently created diff packages
+func (d *Dao) DiffRetry(c context.Context) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	query := fmt.Sprintf(_calDiffFmt, " AND ctime != mtime AND mtime < date_sub(now(), INTERVAL "+d.c.Cfg.Diff.Retry+")", "LIMIT 1")
+	row := d.db.QueryRow(c, query, _diffPkg, "")
+	if err = row.Scan(&file.ID, &file.Name, &file.Type, &file.Md5, &file.Size, &file.URL, &file.ResourceID, &file.FileType, &file.FromVer); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			file = nil
+		} else {
+			log.Error("db.QueryRow(%s) error(%v)", _calDiffNew, err)
+			return
+		}
+	}
+	return
+}
+
+// SaveFile saves the file info
+func (d *Dao) SaveFile(c context.Context, fileID int, file *model.FileInfo) (err error) {
+	// "UPDATE resource_file SET url = ?, file_type = ?, md5 = ?, size = ?, `name` = ? WHERE id = ?"
+	if _, err = d.db.Exec(c, _saveFile, file.URL, _diffPkg, file.Md5, file.Size, file.Name, fileID); err != nil {
+		log.Error("SaveFile ID %d, Err %v", fileID, err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/cal_diff_test.go

@@ -0,0 +1,94 @@
+package caldiff
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"go-common/app/job/main/appstatic/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+const (
+	_availableRes = "SELECT id FROM resource ORDER BY id DESC"
+)
+
+func TestDao_DiffNew(t *testing.T) {
+	Convey("TestDao_DiffNew", t, WithDao(func(d *Dao) {
+		file, err := d.DiffNew(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_DiffRetry(t *testing.T) {
+	Convey("TestDao_DiffRetry", t, WithDao(func(d *Dao) {
+		file, err := d.DiffRetry(ctx)
+		So(err, ShouldBeNil)
+		data, err2 := (json.Marshal(file))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_SaveFile(t *testing.T) {
+	Convey("TestDao_SaveFile", t, WithDao(func(d *Dao) {
+		err := d.SaveFile(ctx, 1, &model.FileInfo{
+			Name: "123",
+			Size: 123,
+			Type: "1",
+			Md5:  "1234",
+			URL:  "xxx",
+		})
+		So(err, ShouldBeNil)
+	}))
+}
+
+func TestDao_ParseResID(t *testing.T) {
+	Convey("TestDao_ParseResID", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		res, err := d.ParseResID(ctx, int(r.ID))
+		So(err, ShouldBeNil)
+		So(res, ShouldNotBeNil)
+		fmt.Println(r.ID)
+	}))
+}
+
+func TestDao_ParseResVer(t *testing.T) {
+	Convey("TestDao_ParseResVer", t, WithDao(func(d *Dao) {
+		dd, err := d.ParseResVer(ctx, 23, 1)
+		So(err, ShouldBeNil)
+		fmt.Println(err)
+		So(dd, ShouldNotBeNil)
+		data, err2 := (json.Marshal(dd))
+		So(err2, ShouldBeNil)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_ReadyFile(t *testing.T) {
+	Convey("TestDao_ReadyFile", t, WithDao(func(d *Dao) {
+		var r = &model.Resource{}
+		if err := d.db.QueryRow(ctx, _availableRes).Scan(&r.ID); err != nil {
+			return
+		}
+		dd, err := d.ReadyFile(ctx, int(r.ID), 0) // ftype = 0 full package
+		So(err, ShouldBeNil)
+		So(dd, ShouldNotBeNil)
+		data, _ := json.Marshal(dd)
+		fmt.Println(string(data))
+	}))
+}
+
+func TestDao_UpdateStatus(t *testing.T) {
+	Convey("TestDao_UpdateStatus", t, WithDao(func(d *Dao) {
+		err := d.UpdateStatus(ctx, 2, 304)
+		So(err, ShouldBeNil)
+	}))
+}

app/job/main/appstatic/dao/caldiff/dao.go

@@ -0,0 +1,24 @@
+package caldiff
+
+import (
+	"go-common/app/job/main/appstatic/conf"
+	xsql "go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao .
+type Dao struct {
+	c      *conf.Config
+	db     *xsql.DB
+	client *bm.Client
+}
+
+// New creates a dao instance.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     xsql.NewMySQL(c.MySQL),
+		client: bm.NewClient(c.HTTPClient),
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/dao_test.go

@@ -0,0 +1,53 @@
+package caldiff
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"go-common/app/job/main/appstatic/conf"
+
+	"gopkg.in/h2non/gock.v1"
+)
+
+var (
+	d   *Dao
+	ctx = context.Background()
+)
+
+func init() {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.appstatic-job")
+		flag.Set("conf_token", "bb62e0021e8d0c7996ccf1fb8d267cee")
+		flag.Set("tree_id", "40719")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		dir, _ := filepath.Abs("../cmd/appstatic-job-test.toml")
+		flag.Set("conf", dir)
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	d.client.SetTransport(gock.DefaultTransport)
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		f(d)
+	}
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	return r
+}

app/job/main/appstatic/dao/caldiff/download.go

@@ -0,0 +1,42 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"go-common/library/log"
+)
+
+const (
+	errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
+)
+
+// DownloadFile downloads one file from url to local
+func (d *Dao) DownloadFile(ctx context.Context, api string, fileName string) (bts int64, err error) {
+	var (
+		res []byte
+		f   *os.File
+		req *http.Request
+	)
+	if req, err = d.client.NewRequest(http.MethodGet, api, "", url.Values{}); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if res, err = d.client.Raw(ctx, req); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("httpGetURL-(%s)", api), err)
+		return
+	}
+	if f, err = os.Create(fileName); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("CreateFile(%s)", fileName), err)
+		return
+	}
+	if bts, err = io.Copy(f, bytes.NewReader(res)); err != nil {
+		log.Error(errFormat, "saveFile", fmt.Sprintf("SaveFile(%s)", fileName), err)
+	}
+	return
+}

app/job/main/appstatic/dao/caldiff/download_test.go

@@ -0,0 +1,29 @@
+package caldiff
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_DownloadFile(t *testing.T) {
+	var (
+		newPath = fmt.Sprintf("%s/%s", d.c.Cfg.Diff.Folder, "123.zip")
+		url     = "http://i0.hdslb.com/bfs/face/ca14680bcfe4a956d1b6c06fbc1f6a6529257746.jpg"
+	)
+	Convey("TestDao_DownloadFile", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		data, err := d.DownloadFile(ctx, url, newPath)
+		So(err, ShouldBeNil)
+		So(data, ShouldBeGreaterThan, 0)
+		fmt.Println(data)
+	}))
+	Convey("CreateFile Error", t, WithDao(func(d *Dao) {
+		httpMock("GET", url).Reply(200).Body(bytes.NewReader([]byte("test")))
+		_, err := d.DownloadFile(ctx, url, "/test/test.txt")
+		So(err, ShouldNotBeNil)
+		fmt.Println(err)
+	}))
+}

app/job/main/appstatic/dao/caldiff/upbfs.go

@@ -0,0 +1,89 @@
+package caldiff
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// bfs info
+const (
+	_uploadURL = "/bfs/%s/%s"
+	_template  = "%s\n%s\n%s\n%d\n"
+	_method    = "PUT"
+	_bucket    = "app-static"
+)
+
+// Upload upload picture or log file to bfs
+func (d *Dao) Upload(c context.Context, fileName string, fileType string, timing int64, data []byte, bfs *conf.Bfs) (location string, err error) {
+	var (
+		req    *http.Request
+		resp   *http.Response
+		code   int
+		client = &http.Client{Timeout: time.Duration(bfs.Timeout) * time.Millisecond}
+		url    = fmt.Sprintf(bfs.Host+_uploadURL, _bucket, fileName)
+	)
+	// prepare the data of the file and init the request
+	buf := new(bytes.Buffer)
+	_, err = buf.Write(data)
+	if err != nil {
+		log.Error("Upload.buf.Write.error(%v)", err)
+		err = ecode.RequestErr
+		return
+	}
+	if req, err = http.NewRequest(_method, url, buf); err != nil {
+		log.Error("http.NewRequest() Upload(%v) error(%v)", url, err)
+		return
+	}
+	// request setting
+	authorization := authorize(bfs.Key, bfs.Secret, _method, _bucket, fileName, timing)
+	req.Header.Set("Date", fmt.Sprint(timing))
+	req.Header.Set("Authorization", authorization)
+	req.Header.Set("Content-Type", fileType)
+	resp, err = client.Do(req)
+	// response treatment
+	if err != nil {
+		log.Error("Bfs client.Do(%s) error(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err = errors.Wrap(ecode.Int(resp.StatusCode), "Bfs Status Code Error")
+		return
+	}
+	code, err = strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = errors.Wrap(ecode.Int(code), "Bfs Header Code Error")
+		return
+	}
+	location = resp.Header.Get("Location")
+	return
+}
+
+// authorize returns authorization for upload file to bfs
+func authorize(key, secret, method, bucket, file string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf(_template, method, bucket, file, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}

app/job/main/appstatic/dao/caldiff/upbfs_test.go

@@ -0,0 +1,57 @@
+package caldiff
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoUpload(t *testing.T) {
+	var (
+		c        = context.Background()
+		fileName = "test.txt"
+		fileType = "txt"
+		timing   = time.Now().Unix()
+		bfs      = d.c.Bfs
+		data     = []byte("123")
+		url      = fmt.Sprintf(bfs.Host+_uploadURL, "app-static", fileName)
+	)
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		ctx.Convey("http code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(-400)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("business code error", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).JSON(`{"code":-400}`)
+			_, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldNotBeNil)
+		})
+		ctx.Convey("everything is fine", func(ctx convey.C) {
+			httpMock(_method, url).Reply(200).SetHeader("Location", "test").SetHeader("code", "200").JSON(`{"code":200}`)
+			location, err := d.Upload(c, fileName, fileType, timing, data, bfs)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(location, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestDaoauthorize(t *testing.T) {
+	var (
+		key    = "key"
+		secret = "secret"
+		method = "put"
+		bucket = "tv-cover"
+		file   = "file"
+		expire = int64(0)
+	)
+	convey.Convey("authorize", t, func(ctx convey.C) {
+		authorization := authorize(key, secret, method, bucket, file, expire)
+		ctx.Convey("Then authorization should not be nil.", func(ctx convey.C) {
+			ctx.So(authorization, convey.ShouldNotBeNil)
+		})
+	})
+}