Create & Init Project...

app/admin/main/upload/dao/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bfs_test.go",
+        "dao_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bfs.go",
+        "dao.go",
+        "hbase.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/tsuna/gohbase:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/dao/bfs.go

@@ -0,0 +1,194 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// Bfs .
+type Bfs struct {
+	bfsURL         string
+	waterMarkURL   string
+	imageGenURL    string
+	client         *http.Client
+	wmClient       *http.Client
+	imageGenClient *http.Client
+}
+
+// NewBfs .
+func NewBfs(c *conf.Config) *Bfs {
+	return &Bfs{
+		bfsURL:       c.Bfs.BfsURL,
+		waterMarkURL: c.Bfs.WaterMarkURL,
+		imageGenURL:  c.Bfs.ImageGenURL,
+		client: &http.Client{
+			Timeout: time.Duration(c.Bfs.TimeOut),
+		},
+		wmClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.WmTimeOut),
+		},
+		imageGenClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.ImageGenTimeOut),
+		},
+	}
+}
+
+func (b *Bfs) waterMark(ctx context.Context, data []byte, contentType, wmKey, wmText string, paddingX, paddingY int, wmScale float64) (res []byte, err error) {
+	var (
+		resp *http.Response
+		bw   io.Writer
+		req  *http.Request
+		ext  string
+	)
+	buf := new(bytes.Buffer)
+	w := multipart.NewWriter(buf)
+	if bw, err = w.CreateFormFile("file", "1.jpg"); err != nil {
+		return
+	}
+	if _, err = bw.Write(data); err != nil {
+		return
+	}
+	w.WriteField("wm_key", wmKey)
+	w.WriteField("wm_text", wmText)
+	w.WriteField("padding_x", strconv.Itoa(paddingX))
+	w.WriteField("padding_y", strconv.Itoa(paddingY))
+	w.WriteField("wm_scale", strconv.FormatFloat(wmScale, 'f', 2, 64))
+	if ext = path.Base(contentType); ext == "jpeg" {
+		ext = "jpg"
+	}
+	w.WriteField("ext", fmt.Sprintf(".%s", ext))
+	if err = w.Close(); err != nil {
+		return
+	}
+	if req, err = http.NewRequest(http.MethodPost, b.waterMarkURL, buf); err != nil {
+		return
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if resp, err = b.wmClient.Do(req); err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Error("bfs.waterMark.status(%v)", resp.StatusCode)
+		if resp.StatusCode == http.StatusRequestEntityTooLarge {
+			err = ecode.BfsUploadFileTooLarge
+		} else {
+			err = ecode.ServerErr
+		}
+		return
+	}
+	res, err = ioutil.ReadAll(resp.Body)
+	return
+}
+
+// Upload with watermark
+func (b *Bfs) Upload(ctx context.Context, up *model.UploadParam, data []byte) (location, etag string, err error) {
+	var (
+		res []byte
+	)
+	if up.WmKey != "" || up.WmText != "" {
+		if res, err = b.waterMark(ctx, data, up.ContentType, up.WmKey, up.WmText, up.WmPaddingX, up.WmPaddingY, up.WmScale); err != nil {
+			log.Error("b.waterMark(%+v) error(%v)", up, err)
+			return
+		}
+		data = res
+	}
+	location, etag, err = b.upload(ctx, up.ContentType, up.Auth, up.Bucket, up.FileName, data)
+	return
+}
+
+// SpecificUpload .
+func (b *Bfs) SpecificUpload(ctx context.Context, contentType, auth, bucket, fileName string, data []byte) (location, etag string, err error) {
+	location, etag, err = b.upload(ctx, contentType, auth, bucket, fileName, data)
+	return
+}
+
+func (b *Bfs) upload(ctx context.Context, contentType, auth, bucket, fileName string, data []byte) (location, etag string, err error) {
+	reqURL := fmt.Sprintf("http://%s/bfs/%s/%s", b.bfsURL, bucket, fileName)
+	buf := new(bytes.Buffer)
+	if _, err = buf.Write(data); err != nil {
+		log.Error("buf.Write error(%v)", err)
+		return
+	}
+	req, err := http.NewRequest("PUT", reqURL, buf)
+	if err != nil {
+		log.Error("http.NewRequest() error(%v)", err)
+		return
+	}
+	req.Header.Add("Content-Type", contentType)
+	req.Header.Add("Authorization", auth)
+	resp, err := b.client.Do(req)
+	if err != nil {
+		log.Error("client.Do error(%v)", err)
+		return
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.BfsRequestErr
+		return
+	case http.StatusUnauthorized:
+		// 验证不通过
+		err = ecode.BfsUploadAuthErr
+		return
+	case http.StatusRequestEntityTooLarge:
+		err = ecode.FileTooLarge
+		return
+	case http.StatusNotFound:
+		err = ecode.NothingFound
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusServiceUnavailable:
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.BfsUploadStatusErr
+		return
+	}
+	code, err := strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = ecode.BfsUploadCodeErr
+		return
+	}
+	location = resp.Header.Get("location")
+	etag = resp.Header.Get("etag")
+	return
+}
+
+// Authorize .
+func (b *Bfs) Authorize(key, secret, method, bucket, fileName string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf("%s\n%s\n%s\n%d\n", method, bucket, fileName, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}

app/admin/main/upload/dao/bfs_test.go

@@ -0,0 +1,126 @@
+package dao
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoNewBfs(t *testing.T) {
+	convey.Convey("NewBfs", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ctx.Convey("d.Bfs should not be nil", func(ctx convey.C) {
+				ctx.So(d.Bfs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+//func TestDaowaterMark(t *testing.T) {
+//	convey.Convey("waterMark", t, func(ctx convey.C) {
+//		bts, err := ioutil.ReadFile("/Users/sunsuke/Desktop/hahaha.png")
+//		fmt.Println(err)
+//		//for _, byt := range bts {
+//		//fmt.Print(byt, ",")
+//		//}
+//
+//		var (
+//			c = context.Background()
+//			// png file
+//			data        = bts
+//			contentType = http.DetectContentType(data)
+//			wmKey       = ""
+//			wmText      = "test"
+//			paddingX    = int(0)
+//			paddingY    = int(0)
+//			wmScale     = float64(0)
+//		)
+//		fmt.Println(contentType)
+//		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+//			res, err := d.Bfs.waterMark(c, data, contentType, wmKey, wmText, paddingX, paddingY, wmScale)
+//			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+//				ctx.So(err, convey.ShouldBeNil)
+//				ctx.So(res, convey.ShouldNotBeNil)
+//			})
+//		})
+//	})
+//}
+
+func TestDaoUpload(t *testing.T) {
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			data = []byte("ut test")
+			up   = &model.UploadParam{
+				Bucket:      "test",
+				Auth:        d.Bfs.Authorize("221bce6492eba70f", "6eb80603e85842542f9736eb13b7e3", http.MethodPut, "test", "", time.Now().Unix()),
+				ContentType: http.DetectContentType(data),
+			}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			location, etag, err := d.Bfs.Upload(c, up, data)
+			ctx.Convey("Then err should be nil.location,etag should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(etag, convey.ShouldNotBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSpecificUpload(t *testing.T) {
+	convey.Convey("SpecificUpload", t, func(ctx convey.C) {
+		var (
+			c           = context.Background()
+			data        = []byte("ut specific upload")
+			contentType = http.DetectContentType(data)
+			auth        = d.Bfs.Authorize("221bce6492eba70f", "6eb80603e85842542f9736eb13b7e3", http.MethodPut, "test", "", time.Now().Unix())
+			bucket      = "test"
+			fileName    = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			location, etag, err := d.Bfs.SpecificUpload(c, contentType, auth, bucket, fileName, data)
+			ctx.Convey("Then err should be nil.location,etag should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(etag, convey.ShouldNotBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAuthorize(t *testing.T) {
+	convey.Convey("Authorize", t, func(ctx convey.C) {
+		var (
+			key      = "221bce6492eba70f"
+			secret   = "6eb80603e85842542f9736eb13b7e3"
+			method   = http.MethodPut
+			bucket   = "test"
+			fileName = ""
+			expire   = time.Now().Unix()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			authorization := d.Bfs.Authorize(key, secret, method, bucket, fileName, expire)
+			ctx.Convey("Then authorization should not be nil.", func(ctx convey.C) {
+				ctx.So(authorization, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoClose(t *testing.T) {
+	convey.Convey("close", t, func(ctx convey.C) {
+		d.Close()
+	})
+}
+
+func TestDaoPing(t *testing.T) {
+	convey.Convey("ping", t, func(ctx convey.C) {
+		d.Ping(context.Background())
+	})
+}

app/admin/main/upload/dao/dao.go

@@ -0,0 +1,36 @@
+package dao
+
+import (
+	"context"
+	"strings"
+
+	"go-common/app/admin/main/upload/conf"
+
+	"github.com/tsuna/gohbase"
+)
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	hbase gohbase.AdminClient
+	Bfs   *Bfs
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		hbase: gohbase.NewAdminClient(strings.Join(c.Hbase.Zookeeper.Addrs, ",")),
+		Bfs:   NewBfs(c),
+	}
+	return dao
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return nil
+}

app/admin/main/upload/dao/dao_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/main/upload/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.manager.upload-admin")
+		flag.Set("conf_token", "29334a5b68dbf2ba2aa820a036fede52")
+		flag.Set("tree_id", "33970")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/upload-admin.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/admin/main/upload/dao/hbase.go

@@ -0,0 +1,41 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/log"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+var (
+	prefix = "bucket_"
+)
+
+// CreateTable .
+// TODO check namespace
+func (d *Dao) CreateTable(c context.Context, table string) error {
+	families := make(map[string]map[string]string)
+	families["bfsfile"] = map[string]string{
+		"BLOOMFILTER":         "ROW",
+		"VERSIONS":            "1",
+		"IN_MEMORY":           "false",
+		"KEEP_DELETED_CELLS":  "false",
+		"DATA_BLOCK_ENCODING": "NONE",
+		"TTL":               "2147483647", // NOTE: 2147483647 is forever
+		"COMPRESSION":       "NONE",
+		"MIN_VERSIONS":      "0",
+		"BLOCKCACHE":        "true",
+		"BLOCKSIZE":         "65536",
+		"REPLICATION_SCOPE": "0",
+	}
+	b := [][]byte{[]byte("1"), []byte("2"), []byte("3"), []byte("4"), []byte("5"), []byte("6"), []byte("7"),
+		[]byte("8"), []byte("9"), []byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), []byte("f")}
+	ct := hrpc.NewCreateTable(c, []byte(prefix+table), families, hrpc.SplitKeys(b))
+	err := d.hbase.CreateTable(ct)
+	if err != nil {
+		log.Error("CreateTable(),err:%+v", err)
+		return err
+	}
+	return nil
+}