Create & Init Project...

app/interface/main/upload/dao/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bfs_test.go",
+        "dao_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/upload/conf:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bfs.go",
+        "bucket.go",
+        "dao.go",
+    ],
+    importpath = "go-common/app/interface/main/upload/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/upload/conf:go_default_library",
+        "//app/interface/main/upload/model:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/upload/dao/bfs.go

@@ -0,0 +1,224 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/interface/main/upload/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// Bfs .
+type Bfs struct {
+	bfsURL         string
+	waterMarkURL   string
+	imageGenURL    string
+	client         *http.Client
+	wmClient       *http.Client
+	imageGenClient *http.Client
+}
+
+// NewBfs .
+func NewBfs(c *conf.Config) *Bfs {
+	return &Bfs{
+		bfsURL:       c.Bfs.BfsURL,
+		waterMarkURL: c.Bfs.WaterMarkURL,
+		imageGenURL:  c.Bfs.ImageGenURL,
+		client: &http.Client{
+			Timeout: time.Duration(c.Bfs.TimeOut),
+		},
+		wmClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.WmTimeOut),
+		},
+		imageGenClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.ImageGenTimeOut),
+		},
+	}
+}
+
+// GenImage gen a image.
+func (b *Bfs) GenImage(ctx context.Context, wmKey, wmText string, distance int, vertical bool) (res []byte, height, width int, hasher string, err error) {
+	var (
+		params = url.Values{}
+		resp   *http.Response
+	)
+	params.Set("wm_key", wmKey)
+	params.Set("wm_text", wmText)
+	params.Set("distance", strconv.Itoa(distance))
+	params.Set("vertical", strconv.FormatBool(vertical))
+	if resp, err = b.imageGenClient.PostForm(b.imageGenURL, params); err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Error("bfs.genImage.status(%v)", resp.StatusCode)
+	}
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.RequestErr
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.ServerErr
+		return
+	}
+	if height, err = strconv.Atoi(resp.Header.Get("O-Height")); err != nil {
+		log.Error("bfs.genImage.O-Height,err(%v)", err)
+		return
+	}
+	if width, err = strconv.Atoi(resp.Header.Get("O-Width")); err != nil {
+		log.Error("bfs.genImage.O-Width,err(%v)", err)
+		return
+	}
+	hasher = resp.Header.Get("Md5")
+	res, err = ioutil.ReadAll(resp.Body)
+	return
+}
+
+// Watermark generate watermark image by key and text.
+func (b *Bfs) Watermark(ctx context.Context, data []byte, contentType, wmKey, wmText string, paddingX, paddingY int, wmScale float64) (res []byte, err error) {
+	var (
+		resp *http.Response
+		bw   io.Writer
+		req  *http.Request
+		ext  string
+	)
+	buf := new(bytes.Buffer)
+	w := multipart.NewWriter(buf)
+	if bw, err = w.CreateFormFile("file", "1.jpg"); err != nil {
+		return
+	}
+	if _, err = bw.Write(data); err != nil {
+		return
+	}
+	w.WriteField("wm_key", wmKey)
+	w.WriteField("wm_text", wmText)
+	w.WriteField("padding_x", strconv.Itoa(paddingX))
+	w.WriteField("padding_y", strconv.Itoa(paddingY))
+	w.WriteField("wm_scale", strconv.FormatFloat(wmScale, 'f', 2, 64))
+	if ext = path.Base(contentType); ext == "jpeg" {
+		ext = "jpg"
+	}
+	w.WriteField("ext", fmt.Sprintf(".%s", ext))
+	if err = w.Close(); err != nil {
+		return
+	}
+	if req, err = http.NewRequest(http.MethodPost, b.waterMarkURL, buf); err != nil {
+		return
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if resp, err = b.wmClient.Do(req); err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Error("bfs.waterMark.status(%v)", resp.StatusCode)
+		err = ecode.ServerErr
+		return
+	}
+	res, err = ioutil.ReadAll(resp.Body)
+	return
+}
+
+// Upload upload file to bfs.
+func (b *Bfs) Upload(ctx context.Context, key, secret, contentType, bucket, dir, filename string, data []byte) (location, etag string, err error) {
+	dir = strings.Trim(dir, "/")
+	var (
+		buf   = new(bytes.Buffer)
+		fname = path.Join(dir, filename)
+	)
+	// 多级目录上传，request url必须以 / 结尾
+	if filename == "" && dir != "" {
+		fname = fname + "/"
+	}
+	reqURL := fmt.Sprintf("http://%s/bfs/%s/%s", b.bfsURL, bucket, fname)
+	if _, err = buf.Write(data); err != nil {
+		log.Error("Upload.buf.Write.error(%v)", err)
+		err = ecode.ServerErr
+		return
+	}
+	req, err := http.NewRequest(http.MethodPut, reqURL, buf)
+	if err != nil {
+		err = ecode.ServerErr
+		return
+	}
+	auth := b.authorize(key, secret, http.MethodPut, bucket, fname, time.Now().Unix())
+	req.Header.Add("Content-Type", contentType)
+	req.Header.Add("Authorization", auth)
+	resp, err := b.client.Do(req)
+	if err != nil {
+		log.Error("Upload.client.Do.error(%v)", err)
+		err = ecode.ServerErr
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Error("bucket:%s,filename:%s,response code:%+v", bucket, filename, resp.StatusCode)
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.RequestErr
+		return
+	case http.StatusUnauthorized:
+		// 验证不通过
+		err = ecode.BfsUploadAuthErr
+		return
+	case http.StatusRequestEntityTooLarge:
+		err = ecode.FileTooLarge
+		return
+	case http.StatusNotFound:
+		err = ecode.NothingFound
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusServiceUnavailable:
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.BfsUploadStatusErr
+		return
+	}
+	code, err := strconv.Atoi(resp.Header.Get("Code"))
+	if err != nil || code != 200 {
+		log.Error("bucket:%s,dir:%s,filename:%s response.Header.Code(%s) error(%v)", bucket, dir, filename, resp.Header.Get("Code"), err)
+		err = ecode.BfsUploadCodeErr
+		return
+	}
+	location = resp.Header.Get("location")
+	etag = resp.Header.Get("etag")
+	return
+}
+
+// authorize generate authorization
+func (b *Bfs) authorize(key, secret, method, bucket, fileName string, expire int64) string {
+	content := fmt.Sprintf("%s\n%s\n%s\n%d\n", method, bucket, fileName, expire)
+	mac := hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	return fmt.Sprintf("%s:%s:%d", key, signature, expire)
+}

app/interface/main/upload/dao/bfs_test.go

@@ -0,0 +1,84 @@
+package dao
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go-common/app/interface/main/upload/conf"
+	xtime "go-common/library/time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestNewBfs(t *testing.T) {
+	Convey("new bfs instance", t, func() {
+		b := NewBfs(&conf.Config{
+			Bfs: &conf.Bfs{
+				BfsURL:       "uat-bfs.bilibili.co",
+				WaterMarkURL: "http://i0.hdslb.com/imageserver/watermark/gen",
+				TimeOut:      xtime.Duration(time.Second * 5),
+				WmTimeOut:    xtime.Duration(time.Second * 5),
+			},
+		})
+		So(b, ShouldNotBeNil)
+	})
+}
+
+func TestGenImage(t *testing.T) {
+	Convey("create watermark image", t, func() {
+		image, height, width, hasher, err := b.GenImage(context.TODO(), "comic", "hello world", 2, true)
+		So(err, ShouldBeNil)
+		So(image, ShouldNotBeEmpty)
+		So(height, ShouldNotEqual, 0)
+		So(width, ShouldNotEqual, 0)
+		So(hasher, ShouldNotEqual, "")
+	})
+}
+
+func TestWatermark(t *testing.T) {
+	Convey("do watermark action", t, func() {
+		image, err := b.Watermark(context.TODO(), testData, "image/png", "comic", "hello", 0, 0, 0)
+		So(err, ShouldBeNil)
+		So(image, ShouldNotBeEmpty)
+	})
+}
+
+func TestUpload(t *testing.T) {
+	Convey("upload", t, func() {
+		var (
+			dir      = "dir1/"
+			filename = "1111.jpg"
+		)
+		location, _, err := b.Upload(context.Background(), "1b24a3d8560d2213", "415aaa6ff53659dabf8a2de394025a", "image/jpg", "static", dir, filename, testData)
+		So(err, ShouldBeNil)
+		So(location, ShouldNotBeEmpty)
+	})
+	Convey("upload", t, func() {
+		var (
+			dir      = "dir1/"
+			filename = ""
+		)
+		location, _, err := b.Upload(context.Background(), "1b24a3d8560d2213", "415aaa6ff53659dabf8a2de394025a", "image/jpg", "static", dir, filename, testData)
+		So(err, ShouldBeNil)
+		So(location, ShouldNotBeEmpty)
+	})
+	Convey("upload", t, func() {
+		var (
+			dir      = ""
+			filename = "1111.jpg"
+		)
+		location, _, err := b.Upload(context.Background(), "1b24a3d8560d2213", "415aaa6ff53659dabf8a2de394025a", "image/jpg", "static", dir, filename, testData)
+		So(err, ShouldBeNil)
+		So(location, ShouldNotBeEmpty)
+	})
+	Convey("upload", t, func() {
+		var (
+			dir      = ""
+			filename = ""
+		)
+		location, _, err := b.Upload(context.Background(), "1b24a3d8560d2213", "415aaa6ff53659dabf8a2de394025a", "image/jpg", "static", dir, filename, testData)
+		So(err, ShouldBeNil)
+		So(location, ShouldNotBeEmpty)
+	})
+}

app/interface/main/upload/dao/bucket.go

@@ -0,0 +1,68 @@
+package dao
+
+import (
+	"encoding/json"
+	"strings"
+
+	"go-common/app/interface/main/upload/model"
+	"go-common/library/log"
+)
+
+// Buckets all bucket info from database.
+func (d *Dao) Buckets() (bucketMap map[string]*model.Bucket, err error) {
+	var (
+		buckets  []*model.Bucket
+		limitMap map[string]map[string]*model.DirConfig
+	)
+	if err = d.orm.Table("bucket").Find(&buckets).Error; err != nil {
+		log.Error("orm.Table(bucket) error(%v)", err)
+		return
+	}
+	if limitMap, err = d.dirLimits(); err != nil {
+		return
+	}
+	bucketMap = make(map[string]*model.Bucket)
+	for _, b := range buckets {
+		v, ok := limitMap[b.Name]
+		if ok {
+			b.DirLimit = v
+		}
+		bucketMap[b.Name] = b
+	}
+	return
+}
+
+// dirLimits directory limit from database.
+func (d *Dao) dirLimits() (limitMap map[string]map[string]*model.DirConfig, err error) {
+	limits := make([]*model.DirLimit, 0)
+	if err = d.orm.Table("dir_limit").Find(&limits).Error; err != nil {
+		return
+	}
+	limitMap = make(map[string]map[string]*model.DirConfig)
+	for _, l := range limits {
+		var (
+			pic  model.DirPicConfig
+			rate model.DirRateConfig
+		)
+		if err = json.Unmarshal([]byte(l.DirPicConfig), &pic); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", l.DirPicConfig, err)
+			err = nil
+			continue
+		}
+		if pic.AllowType != "" {
+			pic.AllowTypeSlice = strings.Split(pic.AllowType, ",")
+		}
+		if err = json.Unmarshal([]byte(l.DirRateConfig), &rate); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", l.DirRateConfig, err)
+			err = nil
+			continue
+		}
+		if _, ok := limitMap[l.BucketName]; !ok {
+			limitMap[l.BucketName] = make(map[string]*model.DirConfig)
+		}
+		// NOTE empty dir is also in limit map
+		l.Dir = strings.Trim(l.Dir, "/")
+		limitMap[l.BucketName][l.Dir] = &model.DirConfig{Pic: pic, Rate: rate}
+	}
+	return
+}

app/interface/main/upload/dao/dao.go

@@ -0,0 +1,27 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/interface/main/upload/conf"
+	"go-common/library/database/orm"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Dao dao struct
+type Dao struct {
+	orm *gorm.DB
+}
+
+// NewDao new a dao instance.
+func NewDao(c *conf.Config) *Dao {
+	return &Dao{
+		orm: orm.NewMySQL(c.ORM),
+	}
+}
+
+// Ping ping database.
+func (d *Dao) Ping(c context.Context) error {
+	return nil
+}

app/interface/main/upload/dao/dao_test.go

@@ -0,0 +1,50 @@
+package dao
+
+import (
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"go-common/app/interface/main/upload/conf"
+	xtime "go-common/library/time"
+)
+
+var (
+	b        *Bfs
+	testData []byte
+	once     sync.Once
+)
+
+func TestMain(m *testing.M) {
+	initData()
+	once.Do(initBFS)
+	os.Exit(m.Run())
+}
+
+func initBFS() {
+	b = NewBfs(&conf.Config{
+		Bfs: &conf.Bfs{
+			BfsURL:          "uat-bfs.bilibili.co",
+			WaterMarkURL:    "http://172.18.33.121:8090/imageserver/watermark/gen",
+			ImageGenURL:     "http://172.18.33.121:8090/imageserver/image/gen",
+			TimeOut:         xtime.Duration(time.Second * 5),
+			WmTimeOut:       xtime.Duration(time.Second * 5),
+			ImageGenTimeOut: xtime.Duration(time.Second * 5),
+		},
+	})
+}
+
+func initData() {
+	client := &http.Client{}
+	resp, err := client.Get("http://uat-i0.hdslb.com/bfs/archive/fc7cd08beb29f93c596426557cf1aa11a08e9730.jpg")
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	if testData, err = ioutil.ReadAll(resp.Body); err != nil {
+		panic(err)
+	}
+}