Create & Init Project...

app/admin/main/upload/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/admin/main/upload/cmd:all-srcs",
+        "//app/admin/main/upload/conf:all-srcs",
+        "//app/admin/main/upload/dao:all-srcs",
+        "//app/admin/main/upload/http:all-srcs",
+        "//app/admin/main/upload/model:all-srcs",
+        "//app/admin/main/upload/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/CHANGELOG.md

@@ -0,0 +1,28 @@
+### upload-admin
+
+### v1.2.5
+> 1. 优先使用用户指定的content-type
+> 1.支持dir验证mime类型
+
+### v1.2.4
+> 1. fix EOF
+
+### v1.2.3
+> 1. fix multipart
+
+### v1.1.1
+> 1.create table for hbase
+
+### v1.1.0
+> 1.鉴黄和涉政分数保存到数据库
+
+### v1.0.0
+> 1.增加dir级别配置
+> 2.修复创建bucket
+
+### v0.1.0
+> 1.初始化项目
+> 2.实现黄图从原bucket迁移到facepri bucket
+> 3.通过bazel构建
+> 4.add接口接收formpost参数
+> 5.优化路由

app/admin/main/upload/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+haoguanwei
+zhapuyu
+liangkai
+
+# Author
+zhuangzhewei
+
+# Reviewer
+zhapuyu
+liangkai

app/admin/main/upload/OWNERS

@@ -0,0 +1,17 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- liangkai
+- zhapuyu
+- zhuangzhewei
+labels:
+- admin
+- admin/main/upload
+- main
+options:
+  no_parent_owners: true
+reviewers:
+- liangkai
+- zhapuyu
+- zhuangzhewei

app/admin/main/upload/README.md

@@ -0,0 +1,10 @@
+# upload
+
+# 项目简介
+> 1.bfs图片上传鉴黄的文件管理服务
+
+# 编译环境
+> 请只用golang v1.7.x以上版本编译执行。 
+
+# 依赖包
+> 1.公共包go-common  

app/admin/main/upload/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["upload-admin.toml"],
+    importpath = "go-common/app/admin/main/upload/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/http:go_default_library",
+        "//app/admin/main/upload/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/cmd/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/http"
+	"go-common/app/admin/main/upload/service"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("upload start, listening: %s \n", conf.Conf.BM.Addr)
+	// service init
+	svc := service.New(conf.Conf)
+	http.Init(conf.Conf, svc)
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("upload get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("upload exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/admin/main/upload/cmd/upload-admin.toml

@@ -0,0 +1,160 @@
+# This is a TOML document. Boom
+
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/upload.pid"
+dir = "./"
+
+bfsDownloadHost = "uat-i0.hdslb.com"
+bfsUpdateHost = "uat-bfs.bilibili.co"
+bfsDeleteHost = "uat-bfs.bilibili.co"
+
+[bm]
+	addr = "0.0.0.0:9001"
+	maxListen = 10
+	timeout = "1s"
+
+[multiHTTP]
+	[multiHTTP.outer]
+	addrs = ["0.0.0.0:6121"]
+	maxListen = 10
+	[multiHTTP.inner]
+	addrs = ["0.0.0.0:6122"]
+	maxListen = 10
+	[multiHTTP.local]
+	addrs = ["0.0.0.0:6123"]
+	maxListen = 10
+
+[identify]
+	whiteAccessKey = ""
+	whiteMid = 0
+	[identify.app]
+	key = "6a29f8ed87407c11"
+	secret = "d3c5a85f5b895a03735b5d20a273bc57"
+	[identify.memcache]
+	name = "go-business/identify"
+	proto = "tcp"
+	addr = "172.16.33.54:11211"
+	active = 10
+	idle = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "80s"
+	[identify.host]
+	auth = "http://uat-passport.bilibili.co"
+	secret = "http://uat-open.bilibili.co"
+	[identify.httpClient]
+	key = "f022126a8a365e20"
+	secret = "b7b86838145d634b487e67b811b8fab2"
+	dial = "30ms"
+	timeout = "100ms"
+	keepAlive = "60s"
+	[identify.httpClient.breaker]
+	window  = "10s"
+	sleep   = "100ms"
+	bucket  = 10
+	ratio   = 0.5
+	request = 100
+	[identify.httpClient.url]
+	"http://passport.bilibili.co/intranet/auth/tokenInfo" = {timeout = "100ms"}
+	"http://passport.bilibili.co/intranet/auth/cookieInfo" = {timeout = "100ms"}
+	"http://open.bilibili.co/api/getsecret" = {timeout = "500ms"}
+
+[orm]
+    dsn = "test:test@tcp(172.16.33.205:3308)/bfs?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 10
+    idle = 10
+    idleTimeout = "4h"
+
+[httpClient]
+   [httpClient.read]
+    key = "6aa4286456d16b97"
+    secret = "351cf022e1ae8296109c3c524faafcc8"
+    dial = "50ms"
+    timeout = "500ms"
+    keepAlive = "60s"
+    timer = 16
+    [httpClient.read.breaker]
+    window  = "10s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [httpClient.write]
+    key = "6aa4286456d16b97"
+    secret = "351cf022e1ae8296109c3c524faafcc8"
+    dial = "50ms"
+    timeout = "800ms"
+    keepAlive = "60s"
+    timer = 16
+    [httpClient.write.breaker]
+    window  = "10s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+
+[auth]
+    managerHost = "http://uat-manager.bilibili.co"
+    dashboardHost = "http://dashboard-mng.bilibili.co"
+    dashboardCaller = "manager-go"
+    [auth.DsHTTPClient]
+    key = "manager-go"
+    secret = "949bbb2dd3178252638c2407578bc7ad"
+    dial = "1s"
+    timeout = "1s"
+    keepAlive = "60s"
+    [auth.DsHTTPClient.breaker]
+    window  = "3s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [auth.MaHTTPClient]
+    key = "f6433799dbd88751"
+    secret = "36f8ddb1806207fe07013ab6a77a3935"
+    dial = "1s"
+    timeout = "1s"
+    keepAlive = "60s"
+    [auth.MaHTTPClient.breaker]
+    window  = "3s"
+    sleep   = "100ms"
+    bucket  = 10
+    ratio   = 0.5
+    request = 100
+    [auth.session]
+    sessionIDLength = 32
+    cookieLifeTime = 1800
+    cookieName = "mng-go"
+    domain = ".bilibili.co"
+    [auth.session.Memcache]
+    name = "go-business/auth"
+    proto = "tcp"
+    addr = "172.16.33.54:11211"
+    active = 10
+    idle = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "80s"
+
+[bfs]
+BfsUrl = "uat-bfs.bilibili.co"
+WaterMarkUrl = "http://172.18.33.121:8090/imageserver/watermark/gen"
+ImageGenURL = "http://172.18.33.121:8090/imageserver/image/gen"
+TimeOut = "10s"
+
+
+[hbase]
+master = ""
+meta = ""
+dialTimeout = "1s"
+readTimeout = "10s"
+readsTimeout = "10s"
+writeTimeout = "10s"
+writesTimeout = "10s"
+    [hbase.zookeeper]
+    root = ""
+     addrs = ["172.18.33.131:2181","172.18.33.168:2181","172.18.33.169:2181"]
+    timeout = "30s"

app/admin/main/upload/conf/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/admin/main/upload/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/permit:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/conf/conf.go

@@ -0,0 +1,131 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/conf"
+	"go-common/library/database/orm"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/permit"
+	xtime "go-common/library/time"
+
+	"go-common/library/database/hbase.v2"
+
+	"github.com/BurntSushi/toml"
+)
+
+// global var
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// HTTPClient http client
+type HTTPClient struct {
+	Read  *bm.ClientConfig
+	Write *bm.ClientConfig
+}
+
+// Config config set
+type Config struct {
+	// base
+	Log *log.Config
+	// http
+	BM *bm.ServerConfig
+	// auth
+	Auth *permit.Config
+	// MySQL
+	MySQL *sql.Config
+	//httpClient
+	HTTPClient *HTTPClient
+	//ORM
+	ORM *orm.Config
+	//bfs hosts
+	BfsDownloadHost string
+	BfsUpdateHost   string
+	BfsDeleteHost   string
+	// bfs
+	Bfs *Bfs
+	// Hbase
+	Hbase *HBaseConfig
+}
+
+// Bfs .
+type Bfs struct {
+	BfsURL          string
+	WaterMarkURL    string
+	ImageGenURL     string
+	TimeOut         xtime.Duration
+	WmTimeOut       xtime.Duration
+	ImageGenTimeOut xtime.Duration
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}
+
+// Item describe bfs bucket accessKey and accessSecret
+type Item struct {
+	Name      string // bucket name
+	KeyID     string // accessKey
+	KeySecret string // accessSecret
+}
+
+// HBaseConfig ...
+type HBaseConfig struct {
+	*hbase.Config
+	WriteTimeout xtime.Duration
+	ReadTimeout  xtime.Duration
+}

app/admin/main/upload/dao/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bfs_test.go",
+        "dao_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bfs.go",
+        "dao.go",
+        "hbase.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/tsuna/gohbase:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/dao/bfs.go

@@ -0,0 +1,194 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// Bfs .
+type Bfs struct {
+	bfsURL         string
+	waterMarkURL   string
+	imageGenURL    string
+	client         *http.Client
+	wmClient       *http.Client
+	imageGenClient *http.Client
+}
+
+// NewBfs .
+func NewBfs(c *conf.Config) *Bfs {
+	return &Bfs{
+		bfsURL:       c.Bfs.BfsURL,
+		waterMarkURL: c.Bfs.WaterMarkURL,
+		imageGenURL:  c.Bfs.ImageGenURL,
+		client: &http.Client{
+			Timeout: time.Duration(c.Bfs.TimeOut),
+		},
+		wmClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.WmTimeOut),
+		},
+		imageGenClient: &http.Client{
+			Timeout: time.Duration(c.Bfs.ImageGenTimeOut),
+		},
+	}
+}
+
+func (b *Bfs) waterMark(ctx context.Context, data []byte, contentType, wmKey, wmText string, paddingX, paddingY int, wmScale float64) (res []byte, err error) {
+	var (
+		resp *http.Response
+		bw   io.Writer
+		req  *http.Request
+		ext  string
+	)
+	buf := new(bytes.Buffer)
+	w := multipart.NewWriter(buf)
+	if bw, err = w.CreateFormFile("file", "1.jpg"); err != nil {
+		return
+	}
+	if _, err = bw.Write(data); err != nil {
+		return
+	}
+	w.WriteField("wm_key", wmKey)
+	w.WriteField("wm_text", wmText)
+	w.WriteField("padding_x", strconv.Itoa(paddingX))
+	w.WriteField("padding_y", strconv.Itoa(paddingY))
+	w.WriteField("wm_scale", strconv.FormatFloat(wmScale, 'f', 2, 64))
+	if ext = path.Base(contentType); ext == "jpeg" {
+		ext = "jpg"
+	}
+	w.WriteField("ext", fmt.Sprintf(".%s", ext))
+	if err = w.Close(); err != nil {
+		return
+	}
+	if req, err = http.NewRequest(http.MethodPost, b.waterMarkURL, buf); err != nil {
+		return
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if resp, err = b.wmClient.Do(req); err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Error("bfs.waterMark.status(%v)", resp.StatusCode)
+		if resp.StatusCode == http.StatusRequestEntityTooLarge {
+			err = ecode.BfsUploadFileTooLarge
+		} else {
+			err = ecode.ServerErr
+		}
+		return
+	}
+	res, err = ioutil.ReadAll(resp.Body)
+	return
+}
+
+// Upload with watermark
+func (b *Bfs) Upload(ctx context.Context, up *model.UploadParam, data []byte) (location, etag string, err error) {
+	var (
+		res []byte
+	)
+	if up.WmKey != "" || up.WmText != "" {
+		if res, err = b.waterMark(ctx, data, up.ContentType, up.WmKey, up.WmText, up.WmPaddingX, up.WmPaddingY, up.WmScale); err != nil {
+			log.Error("b.waterMark(%+v) error(%v)", up, err)
+			return
+		}
+		data = res
+	}
+	location, etag, err = b.upload(ctx, up.ContentType, up.Auth, up.Bucket, up.FileName, data)
+	return
+}
+
+// SpecificUpload .
+func (b *Bfs) SpecificUpload(ctx context.Context, contentType, auth, bucket, fileName string, data []byte) (location, etag string, err error) {
+	location, etag, err = b.upload(ctx, contentType, auth, bucket, fileName, data)
+	return
+}
+
+func (b *Bfs) upload(ctx context.Context, contentType, auth, bucket, fileName string, data []byte) (location, etag string, err error) {
+	reqURL := fmt.Sprintf("http://%s/bfs/%s/%s", b.bfsURL, bucket, fileName)
+	buf := new(bytes.Buffer)
+	if _, err = buf.Write(data); err != nil {
+		log.Error("buf.Write error(%v)", err)
+		return
+	}
+	req, err := http.NewRequest("PUT", reqURL, buf)
+	if err != nil {
+		log.Error("http.NewRequest() error(%v)", err)
+		return
+	}
+	req.Header.Add("Content-Type", contentType)
+	req.Header.Add("Authorization", auth)
+	resp, err := b.client.Do(req)
+	if err != nil {
+		log.Error("client.Do error(%v)", err)
+		return
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.BfsRequestErr
+		return
+	case http.StatusUnauthorized:
+		// 验证不通过
+		err = ecode.BfsUploadAuthErr
+		return
+	case http.StatusRequestEntityTooLarge:
+		err = ecode.FileTooLarge
+		return
+	case http.StatusNotFound:
+		err = ecode.NothingFound
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusServiceUnavailable:
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.BfsUploadStatusErr
+		return
+	}
+	code, err := strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = ecode.BfsUploadCodeErr
+		return
+	}
+	location = resp.Header.Get("location")
+	etag = resp.Header.Get("etag")
+	return
+}
+
+// Authorize .
+func (b *Bfs) Authorize(key, secret, method, bucket, fileName string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf("%s\n%s\n%s\n%d\n", method, bucket, fileName, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}

app/admin/main/upload/dao/bfs_test.go

@@ -0,0 +1,126 @@
+package dao
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoNewBfs(t *testing.T) {
+	convey.Convey("NewBfs", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			ctx.Convey("d.Bfs should not be nil", func(ctx convey.C) {
+				ctx.So(d.Bfs, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+//func TestDaowaterMark(t *testing.T) {
+//	convey.Convey("waterMark", t, func(ctx convey.C) {
+//		bts, err := ioutil.ReadFile("/Users/sunsuke/Desktop/hahaha.png")
+//		fmt.Println(err)
+//		//for _, byt := range bts {
+//		//fmt.Print(byt, ",")
+//		//}
+//
+//		var (
+//			c = context.Background()
+//			// png file
+//			data        = bts
+//			contentType = http.DetectContentType(data)
+//			wmKey       = ""
+//			wmText      = "test"
+//			paddingX    = int(0)
+//			paddingY    = int(0)
+//			wmScale     = float64(0)
+//		)
+//		fmt.Println(contentType)
+//		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+//			res, err := d.Bfs.waterMark(c, data, contentType, wmKey, wmText, paddingX, paddingY, wmScale)
+//			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+//				ctx.So(err, convey.ShouldBeNil)
+//				ctx.So(res, convey.ShouldNotBeNil)
+//			})
+//		})
+//	})
+//}
+
+func TestDaoUpload(t *testing.T) {
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			data = []byte("ut test")
+			up   = &model.UploadParam{
+				Bucket:      "test",
+				Auth:        d.Bfs.Authorize("221bce6492eba70f", "6eb80603e85842542f9736eb13b7e3", http.MethodPut, "test", "", time.Now().Unix()),
+				ContentType: http.DetectContentType(data),
+			}
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			location, etag, err := d.Bfs.Upload(c, up, data)
+			ctx.Convey("Then err should be nil.location,etag should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(etag, convey.ShouldNotBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSpecificUpload(t *testing.T) {
+	convey.Convey("SpecificUpload", t, func(ctx convey.C) {
+		var (
+			c           = context.Background()
+			data        = []byte("ut specific upload")
+			contentType = http.DetectContentType(data)
+			auth        = d.Bfs.Authorize("221bce6492eba70f", "6eb80603e85842542f9736eb13b7e3", http.MethodPut, "test", "", time.Now().Unix())
+			bucket      = "test"
+			fileName    = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			location, etag, err := d.Bfs.SpecificUpload(c, contentType, auth, bucket, fileName, data)
+			ctx.Convey("Then err should be nil.location,etag should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(etag, convey.ShouldNotBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoAuthorize(t *testing.T) {
+	convey.Convey("Authorize", t, func(ctx convey.C) {
+		var (
+			key      = "221bce6492eba70f"
+			secret   = "6eb80603e85842542f9736eb13b7e3"
+			method   = http.MethodPut
+			bucket   = "test"
+			fileName = ""
+			expire   = time.Now().Unix()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			authorization := d.Bfs.Authorize(key, secret, method, bucket, fileName, expire)
+			ctx.Convey("Then authorization should not be nil.", func(ctx convey.C) {
+				ctx.So(authorization, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoClose(t *testing.T) {
+	convey.Convey("close", t, func(ctx convey.C) {
+		d.Close()
+	})
+}
+
+func TestDaoPing(t *testing.T) {
+	convey.Convey("ping", t, func(ctx convey.C) {
+		d.Ping(context.Background())
+	})
+}

app/admin/main/upload/dao/dao.go

@@ -0,0 +1,36 @@
+package dao
+
+import (
+	"context"
+	"strings"
+
+	"go-common/app/admin/main/upload/conf"
+
+	"github.com/tsuna/gohbase"
+)
+
+// Dao dao
+type Dao struct {
+	c     *conf.Config
+	hbase gohbase.AdminClient
+	Bfs   *Bfs
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:     c,
+		hbase: gohbase.NewAdminClient(strings.Join(c.Hbase.Zookeeper.Addrs, ",")),
+		Bfs:   NewBfs(c),
+	}
+	return dao
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return nil
+}

app/admin/main/upload/dao/dao_test.go

@@ -0,0 +1,35 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/main/upload/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.manager.upload-admin")
+		flag.Set("conf_token", "29334a5b68dbf2ba2aa820a036fede52")
+		flag.Set("tree_id", "33970")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/upload-admin.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/admin/main/upload/dao/hbase.go

@@ -0,0 +1,41 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/library/log"
+
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+var (
+	prefix = "bucket_"
+)
+
+// CreateTable .
+// TODO check namespace
+func (d *Dao) CreateTable(c context.Context, table string) error {
+	families := make(map[string]map[string]string)
+	families["bfsfile"] = map[string]string{
+		"BLOOMFILTER":         "ROW",
+		"VERSIONS":            "1",
+		"IN_MEMORY":           "false",
+		"KEEP_DELETED_CELLS":  "false",
+		"DATA_BLOCK_ENCODING": "NONE",
+		"TTL":               "2147483647", // NOTE: 2147483647 is forever
+		"COMPRESSION":       "NONE",
+		"MIN_VERSIONS":      "0",
+		"BLOCKCACHE":        "true",
+		"BLOCKSIZE":         "65536",
+		"REPLICATION_SCOPE": "0",
+	}
+	b := [][]byte{[]byte("1"), []byte("2"), []byte("3"), []byte("4"), []byte("5"), []byte("6"), []byte("7"),
+		[]byte("8"), []byte("9"), []byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), []byte("f")}
+	ct := hrpc.NewCreateTable(c, []byte(prefix+table), families, hrpc.SplitKeys(b))
+	err := d.hbase.CreateTable(ct)
+	if err != nil {
+		log.Error("CreateTable(),err:%+v", err)
+		return err
+	}
+	return nil
+}

app/admin/main/upload/http/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucekt.go",
+        "dir.go",
+        "file.go",
+        "http.go",
+        "sniff.go",
+        "upload.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//app/admin/main/upload/service:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/binding:go_default_library",
+        "//library/net/http/blademaster/middleware/permit:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/http/blademaster/render:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/http/bucekt.go

@@ -0,0 +1,68 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+	"go-common/library/net/http/blademaster/render"
+)
+
+func addBucket(c *bm.Context) {
+	var err error
+	abp := new(model.AddBucketParam)
+	if err = c.BindWith(abp, binding.FormPost); err != nil {
+		return
+	}
+	if len(abp.KeyID) != 16 {
+		c.Render(http.StatusOK, render.JSON{
+			Code:    ecode.RequestErr.Code(),
+			Message: "key_id should length 16",
+			Data:    nil,
+		})
+		c.Abort()
+		return
+	}
+	if len(abp.KeySecret) != 30 {
+		c.Render(http.StatusOK, render.JSON{
+			Code:    ecode.RequestErr.Code(),
+			Message: "key_secret should length 30",
+			Data:    nil,
+		})
+		c.Abort()
+		return
+	}
+
+	c.JSON(nil, uaSvc.AddBucket(c, abp))
+}
+
+func listBucket(c *bm.Context) {
+	var err error
+	lbp := new(model.ListBucketParam)
+	if err = c.Bind(lbp); err != nil {
+		return
+	}
+	c.JSON(uaSvc.ListBucket(c, lbp))
+}
+
+func listPublicBucket(c *bm.Context) {
+	var err error
+	lbp := new(model.ListBucketParam)
+	if err = c.Bind(lbp); err != nil {
+		return
+	}
+	c.JSON(uaSvc.ListPublicBucket(c, lbp))
+}
+
+func detailBucket(c *bm.Context) {
+	var err error
+	dbp := new(struct {
+		Bucket string `json:"bucket" form:"bucket" validate:"required"`
+	})
+	if err = c.Bind(dbp); err != nil {
+		return
+	}
+	c.JSON(uaSvc.DetailBucket(c, dbp.Bucket))
+}

app/admin/main/upload/http/dir.go

@@ -0,0 +1,17 @@
+package http
+
+import (
+	"go-common/app/admin/main/upload/model"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+)
+
+func addDir(c *bm.Context) {
+	var err error
+	adp := &model.AddDirParam{}
+	if err = c.BindWith(adp, binding.FormPost); err != nil {
+		return
+	}
+
+	c.JSON(nil, uaSvc.AddDir(c, adp))
+}

app/admin/main/upload/http/file.go

@@ -0,0 +1,36 @@
+package http
+
+import (
+	"bytes"
+	"io"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+)
+
+// InternalUploadAdminImage .
+func InternalUploadAdminImage(c *bm.Context) {
+	var err error
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	up.WMInit()
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(uaSvc.UploadAdminRecord(c, "internal_admin_upload", up, buf.Bytes()))
+}

app/admin/main/upload/http/http.go

@@ -0,0 +1,73 @@
+package http
+
+import (
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/permit"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	authSvc   *permit.Permit
+	uaSvc     *service.Service
+	verifySvc *verify.Verify
+)
+
+// Init init
+func Init(c *conf.Config, s *service.Service) {
+	initService(c, s)
+	// init router
+	engine := bm.DefaultServer(c.BM)
+	innerRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start error(%v)", err)
+		panic(err)
+	}
+}
+
+// initService init services.
+func initService(c *conf.Config, s *service.Service) {
+	authSvc = permit.New(c.Auth)
+	verifySvc = verify.New(nil)
+	uaSvc = s
+}
+
+// innerRouter init outer router api path.
+func innerRouter(e *bm.Engine) {
+	//init api
+	e.Ping(ping)
+
+	uploadAdmin := e.Group("/x/admin/upload")
+	{
+		uploadAdmin.POST("/add", add)
+		uploadAdmin.GET("/list", authSvc.Permit(""), list)
+		uploadAdmin.DELETE("/delete", authSvc.Permit(""), deleteFile)
+
+		file := uploadAdmin.Group("/file")
+		{
+			file.POST("/upload", authSvc.Permit(""), InternalUploadAdminImage)
+			file.DELETE("/delete", authSvc.Permit(""), deleteRawFile)
+		}
+	}
+
+	uploadAdminV2 := e.Group("/x/admin/upload/v2")
+	{
+		uploadAdminV2.GET("/list", authSvc.Permit(""), multiList)
+		uploadAdminV2.DELETE("/delete", authSvc.Permit(""), deleteFileV2)
+	}
+
+	bucket := uploadAdmin.Group("/bucket")
+	{
+		bucket.POST("/add", verifySvc.Verify, addBucket)
+		bucket.GET("/list", listBucket)
+		bucket.GET("/list/public", listPublicBucket)
+		bucket.GET("/detail", detailBucket)
+
+		dir := bucket.Group("/dir")
+		{
+			dir.POST("/add", verifySvc.Verify, addDir)
+		}
+	}
+}

app/admin/main/upload/http/sniff.go

@@ -0,0 +1,261 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+	"bytes"
+	"encoding/binary"
+)
+
+// The algorithm uses at most sniffLen bytes to make its decision.
+const sniffLen = 512
+
+// DetectContentType implements the algorithm described
+// at http://mimesniff.spec.whatwg.org/ to determine the
+// Content-Type of the given data. It considers at most the
+// first 512 bytes of data. DetectContentType always returns
+// a valid MIME type: if it cannot determine a more specific one, it
+// returns "application/octet-stream".
+func DetectContentType(data []byte) string {
+	if len(data) > sniffLen {
+		data = data[:sniffLen]
+	}
+
+	// Index of the first non-whitespace byte in data.
+	firstNonWS := 0
+	for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ {
+	}
+
+	for _, sig := range sniffSignatures {
+		if ct := sig.match(data, firstNonWS); ct != "" {
+			return ct
+		}
+	}
+
+	return "application/octet-stream" // fallback
+}
+
+func isWS(b byte) bool {
+	switch b {
+	case '\t', '\n', '\x0c', '\r', ' ':
+		return true
+	}
+	return false
+}
+
+type sniffSig interface {
+	// match returns the MIME type of the data, or "" if unknown.
+	match(data []byte, firstNonWS int) string
+}
+
+// Data matching the table in section 6.
+var sniffSignatures = []sniffSig{
+	htmlSig("<!DOCTYPE HTML"),
+	htmlSig("<HTML"),
+	htmlSig("<HEAD"),
+	htmlSig("<SCRIPT"),
+	htmlSig("<IFRAME"),
+	htmlSig("<H1"),
+	htmlSig("<DIV"),
+	htmlSig("<FONT"),
+	htmlSig("<TABLE"),
+	htmlSig("<A"),
+	htmlSig("<STYLE"),
+	htmlSig("<TITLE"),
+	htmlSig("<B"),
+	htmlSig("<BODY"),
+	htmlSig("<BR"),
+	htmlSig("<P"),
+	htmlSig("<!--"),
+
+	&maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"},
+
+	&exactSig{[]byte("%PDF-"), "application/pdf"},
+	&exactSig{[]byte("%!PS-Adobe-"), "application/postscript"},
+
+	// UTF BOMs.
+	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"},
+	&maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"},
+	&maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"},
+
+	&exactSig{[]byte("GIF87a"), "image/gif"},
+	&exactSig{[]byte("GIF89a"), "image/gif"},
+	&exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"},
+	&exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"},
+	&exactSig{[]byte("BM"), "image/bmp"},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"),
+		pat:  []byte("RIFF\x00\x00\x00\x00WEBPVP"),
+		ct:   "image/webp",
+	},
+	&exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"},
+
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
+		pat:  []byte("RIFF\x00\x00\x00\x00WAVE"),
+		ct:   "audio/wave",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
+		pat:  []byte("FORM\x00\x00\x00\x00AIFF"),
+		ct:   "audio/aiff",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF"),
+		pat:  []byte(".snd"),
+		ct:   "audio/basic",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\xFF"),
+		pat:  []byte("OggS\x00"),
+		ct:   "application/ogg",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"),
+		pat:  []byte("MThd\x00\x00\x00\x06"),
+		ct:   "audio/midi",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF"),
+		pat:  []byte("ID3"),
+		ct:   "audio/mpeg",
+	},
+	&maskedSig{
+		mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"),
+		pat:  []byte("RIFF\x00\x00\x00\x00AVI "),
+		ct:   "video/avi",
+	},
+
+	// Fonts
+	&maskedSig{
+		// 34 NULL bytes followed by the string "LP"
+		pat: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x4C\x50"),
+		// 34 NULL bytes followed by \xF\xF
+		mask: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF"),
+		ct:   "application/vnd.ms-fontobject",
+	},
+	&exactSig{[]byte("\x00\x01\x00\x00"), "application/font-ttf"},
+	&exactSig{[]byte("OTTO"), "application/font-off"},
+	&exactSig{[]byte("ttcf"), "application/font-cff"},
+	&exactSig{[]byte("wOFF"), "application/font-woff"},
+
+	&exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"},
+	&exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"},
+	&exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"},
+	&exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"},
+
+	mp4Sig{},
+
+	textSig{}, // should be last
+}
+
+type exactSig struct {
+	sig []byte
+	ct  string
+}
+
+func (e *exactSig) match(data []byte, firstNonWS int) string {
+	if bytes.HasPrefix(data, e.sig) {
+		return e.ct
+	}
+	return ""
+}
+
+type maskedSig struct {
+	mask, pat []byte
+	skipWS    bool
+	ct        string
+}
+
+func (m *maskedSig) match(data []byte, firstNonWS int) string {
+	// pattern matching algorithm section 6
+	// https://mimesniff.spec.whatwg.org/#pattern-matching-algorithm
+
+	if m.skipWS {
+		data = data[firstNonWS:]
+	}
+	if len(m.pat) != len(m.mask) {
+		return ""
+	}
+	if len(data) < len(m.mask) {
+		return ""
+	}
+	for i, mask := range m.mask {
+		db := data[i] & mask
+		if db != m.pat[i] {
+			return ""
+		}
+	}
+	return m.ct
+}
+
+type htmlSig []byte
+
+func (h htmlSig) match(data []byte, firstNonWS int) string {
+	data = data[firstNonWS:]
+	if len(data) < len(h)+1 {
+		return ""
+	}
+	for i, b := range h {
+		db := data[i]
+		if 'A' <= b && b <= 'Z' {
+			db &= 0xDF
+		}
+		if b != db {
+			return ""
+		}
+	}
+	// Next byte must be space or right angle bracket.
+	if db := data[len(h)]; db != ' ' && db != '>' {
+		return ""
+	}
+	return "text/html; charset=utf-8"
+}
+
+var mp4ftype = []byte("ftyp")
+var mp4 = []byte("mp4")
+
+type mp4Sig struct{}
+
+func (mp4Sig) match(data []byte, firstNonWS int) string {
+	// https://mimesniff.spec.whatwg.org/#signature-for-mp4
+	// c.f. section 6.2.1
+	if len(data) < 12 {
+		return ""
+	}
+	boxSize := int(binary.BigEndian.Uint32(data[:4]))
+	if boxSize%4 != 0 || len(data) < boxSize {
+		return ""
+	}
+	if !bytes.Equal(data[4:8], mp4ftype) {
+		return ""
+	}
+	for st := 8; st < boxSize; st += 4 {
+		if st == 12 {
+			// minor version number
+			continue
+		}
+		if bytes.Equal(data[st:st+3], mp4) {
+			return "video/mp4"
+		}
+	}
+	return ""
+}
+
+type textSig struct{}
+
+func (textSig) match(data []byte, firstNonWS int) string {
+	// c.f. section 5, step 4.
+	for _, b := range data[firstNonWS:] {
+		switch {
+		case b <= 0x08,
+			b == 0x0B,
+			0x0E <= b && b <= 0x1A,
+			0x1C <= b && b <= 0x1F:
+			return ""
+		}
+	}
+	return "text/plain; charset=utf-8"
+}

app/admin/main/upload/http/upload.go

@@ -0,0 +1,99 @@
+package http
+
+import (
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+)
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := uaSvc.Ping(c); err != nil {
+		c.Error = err
+		c.AbortWithStatus(503)
+	}
+}
+
+func add(c *bm.Context) {
+	var err error
+	ap := &model.AddParam{}
+	if err = c.BindWith(ap, binding.FormPost); err != nil {
+		return
+	}
+
+	c.JSON(nil, uaSvc.Add(c, ap))
+}
+
+func list(c *bm.Context) {
+	var (
+		err error
+	)
+	lp := &model.ListParam{}
+	if err = c.Bind(lp); err != nil {
+		return
+	}
+	c.JSON(uaSvc.List(c, lp))
+}
+
+func deleteFile(c *bm.Context) {
+	var (
+		err     error
+		ok      bool
+		adminID interface{}
+	)
+	dp := new(model.DeleteParam)
+	if adminID, ok = c.Get("uid"); !ok {
+		c.JSON(nil, ecode.UserNotExist)
+		return
+	}
+	dp.AdminID = adminID.(int64)
+	if err = c.Bind(dp); err != nil {
+		return
+	}
+	c.JSON(nil, uaSvc.Delete(c, dp))
+}
+
+func deleteRawFile(c *bm.Context) {
+	var (
+		err error
+	)
+	dp := new(model.DeleteRawParam)
+	if err = c.Bind(dp); err != nil {
+		return
+	}
+	c.JSON(nil, uaSvc.DeleteRaw(c, dp))
+}
+
+func deleteFileV2(c *bm.Context) {
+	var (
+		adminID interface{}
+		err     error
+		ok      bool
+	)
+	dp := new(model.DeleteV2Param)
+	if adminID, ok = c.Get("uid"); !ok {
+		c.JSON(nil, ecode.UserNotExist)
+		return
+	}
+	dp.AdminID = adminID.(int64)
+	if err = c.Bind(dp); err != nil {
+		return
+	}
+	if err = uaSvc.DeleteV2(c, dp); err != nil {
+		log.Error("deleteFileV2 error(%v)", err)
+	}
+	c.JSON(nil, err)
+}
+
+func multiList(c *bm.Context) {
+	var (
+		err error
+	)
+	lp := &model.MultiListParam{}
+	if err = c.Bind(lp); err != nil {
+		return
+	}
+	c.JSON(uaSvc.MultiList(c, lp))
+}

app/admin/main/upload/model/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucket.go",
+        "dir.go",
+        "param.go",
+        "record.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/model/bucket.go

@@ -0,0 +1,55 @@
+package model
+
+import (
+	xtime "go-common/library/time"
+)
+
+const (
+	// PrivateReadBit 私有读位
+	PrivateReadBit = 0
+	// PrivateWriteBit 私有写位
+	PrivateWriteBit = 1
+	//status
+
+	// Public = 0
+	Public = int(0)
+	// PrivateRead = 1
+	PrivateRead = int(1 << PrivateReadBit)
+	// PrivateWrite = 2
+	PrivateWrite = int(1 << PrivateWriteBit)
+	// PrivateReadWrite = 3
+	PrivateReadWrite = int(PrivateRead | PrivateWrite)
+)
+
+// Bucket bucekt table orm
+type Bucket struct {
+	ID           int         `json:"id" gorm:"column:id"`
+	BucketName   string      `json:"bucket_name" gorm:"column:bucket_name"`
+	Property     int         `json:"property" gorm:"column:property"`
+	KeyID        string      `json:"key_id" gorm:"column:key_id"`
+	KeySecret    string      `json:"key_secret" gorm:"column:key_secret"`
+	PurgeCDN     bool        `json:"purge_cdn" gorm:"column:purge_cdn"`
+	CacheControl int         `json:"cache_control" gorm:"column:cache_control"`
+	Domain       string      `json:"domain" gorm:"column:domain"`
+	CTime        xtime.Time  `json:"ctime" gorm:"column:ctime"`
+	MTime        xtime.Time  `json:"mtime" gorm:"column:mtime"`
+	DirLimit     []*DirLimit `json:"dir_limit" gorm:"-"`
+}
+
+// TableName bucket
+func (b Bucket) TableName() string {
+	return "bucket"
+}
+
+// Page common page response
+type Page struct {
+	PS    int `json:"ps"`
+	PN    int `json:"pn"`
+	Total int `json:"total"`
+}
+
+// BucketListPage bucket/list result
+type BucketListPage struct {
+	Items []*Bucket `json:"items"`
+	Page  *Page     `json:"page"`
+}

app/admin/main/upload/model/dir.go

@@ -0,0 +1,44 @@
+package model
+
+import xtime "go-common/library/time"
+
+// DirConfig dir config
+type DirConfig struct {
+	Pic  DirPicConfig  `json:"dir_pic_config"`
+	Rate DirRateConfig `json:"dir_rate_config"`
+}
+
+// DirPicConfig pic config
+type DirPicConfig struct {
+	FileSize           uint    `json:"file_size"`             //文件大小上限 单位 Byte
+	MaxPixelWidthSize  uint    `json:"max_pixel_width_size"`  //像素宽上限
+	MinPixelWidthSize  uint    `json:"min_pixel_width_size"`  //像素高下限
+	MaxPixelHeightSize uint    `json:"max_pixel_height_size"` //像素高上限
+	MinPixelHeightSize uint    `json:"min_pixel_height_size"` //像素宽下限
+	MaxAspectRatio     float64 `json:"max_aspect_ratio"`      //最大宽高比
+	MinAspectRatio     float64 `json:"min_aspect_ratio"`      //最小宽高比
+	AllowType          string  `json:"allow_type"`            //允许的MIME类型
+}
+
+// DirRateConfig rate config
+type DirRateConfig struct {
+	// SecondQPS 接受 CountQPS 个请求
+	SecondQPS uint `json:"second_qps"`
+	CountQPS  uint `json:"count_qps"`
+}
+
+// DirLimit table dir_limit ORM
+type DirLimit struct {
+	ID         int        `json:"id" gorm:"column:id"`
+	BucketName string     `json:"bucket_name" gorm:"column:bucket_name"`
+	Dir        string     `json:"dir" gorm:"column:dir"`
+	ConfigPic  string     `json:"config_pic" gorm:"column:config_pic"`
+	ConfigRate string     `json:"config_rate" gorm:"column:config_rate"`
+	CTime      xtime.Time `json:"ctime" gorm:"column:ctime"`
+	MTime      xtime.Time `json:"mtime" gorm:"column:mtime"`
+}
+
+// TableName dir_limit
+func (dl DirLimit) TableName() string {
+	return "dir_limit"
+}

app/admin/main/upload/model/param.go

@@ -0,0 +1,129 @@
+package model
+
+const (
+	_defaultWmPaddingX = 10
+	_defaultWmPaddingY = 10
+	_defaultWmScale    = float64(1) / 24
+
+	// delete status .please read document.
+	// http://info.bilibili.co/pages/viewpage.action?pageId=8718262#bfs%E7%AE%A1%E7%90%86%E5%90%8E%E5%8F%B0%E7%9B%B8%E5%85%B3%E6%8E%A5%E5%8F%A3%E6%96%87%E6%A1%A3-db%E8%AE%BE%E8%AE%A1
+
+	// PassStatus express pass status.
+	PassStatus = 2
+	// DeleteStatus .
+	DeleteStatus = 3
+)
+
+// AddParam describe add api param
+type AddParam struct {
+	Bucket   string `json:"bucket" form:"bucket" validate:"required"`
+	FileName string `json:"filename" form:"filename" validate:"required"`
+	URL      string `json:"url" form:"url"`
+	Sex      int    `json:"sex" form:"sex"`
+	Politics int    `json:"politics" form:"politics"`
+}
+
+// ListParam describe list api param
+type ListParam struct {
+	Bucket string `json:"bucket" form:"bucket" validate:"required"`
+	State  int    `json:"state" form:"state" validate:"required,min=0"`
+	PN     int    `json:"pn" form:"pn" validate:"min=1"`
+	PS     int    `json:"ps" form:"ps" validate:"min=1"`
+}
+
+// MultiListParam describe list api param
+type MultiListParam struct {
+	Bucket []string `json:"bucket" form:"bucket"`
+	State  int      `json:"state" form:"state" validate:"min=0"`
+	PN     int      `json:"pn" form:"pn" validate:"min=1" default:"1"`
+	PS     int      `json:"ps" form:"ps" validate:"min=1" default:"50"`
+}
+
+// DeleteParam describe list api param
+type DeleteParam struct {
+	Rid      int    `json:"rid" form:"rid" validate:"required"`
+	Bucket   string `json:"bucket"`
+	FileName string `json:"filename"`
+	AdminID  int64  `json:"admin_id"`
+}
+
+// DeleteV2Param describe list api param
+type DeleteV2Param struct {
+	Rid      int    `json:"rid" form:"rid" validate:"required"`
+	Status   int    `json:"status" form:"status" validate:"required"`
+	Bucket   string `json:"bucket"`
+	FileName string `json:"filename"`
+	AdminID  int64  `json:"admin_id"`
+}
+
+// DeleteRawParam describe list api param
+type DeleteRawParam struct {
+	Bucket   string `json:"bucket" form:"bucket" validate:"required"`
+	FileName string `json:"filename" form:"filename" validate:"required"`
+}
+
+// AddBucketParam .
+type AddBucketParam struct {
+	Name         string `form:"name" json:"name" validate:"required"`
+	Property     int    `form:"property" json:"property" validate:"min=0,max=3"`
+	KeyID        string `form:"key_id" json:"key_id" validate:"required"`
+	KeySecret    string `form:"key_secret" json:"key_secret" validate:"required"`
+	PurgeCDN     bool   `form:"purge_cdn" json:"purge_cdn"`
+	CacheControl int    `form:"cache_control" json:"cache_control"`
+	Domain       string `form:"domain" json:"domain"`
+}
+
+// AddDirParam .
+type AddDirParam struct {
+	BucketName string `form:"bucket_name" validate:"required"`
+	DirName    string `form:"dir_name" validate:"required"`
+	Pic        string `form:"pic"`
+	Rate       string `form:"rate"`
+}
+
+// ListBucketParam .
+type ListBucketParam struct {
+	PN int `form:"pn" validate:"min=1"`
+	PS int `form:"ps" validate:"min=1"`
+}
+
+// UploadParam .
+type UploadParam struct {
+	Bucket      string  `form:"bucket" json:"bucket" validate:"required" `
+	ContentType string  `form:"content_type" json:"content_type"`
+	Auth        string  `form:"auth" json:"-"`
+	Dir         string  `form:"dir" json:"dir"`
+	FileName    string  `form:"file_name" json:"file_name"`
+	WmKey       string  `form:"wm_key" json:"wm_key"`
+	WmText      string  `form:"wm_text" json:"wm_text"`
+	WmPaddingX  int     `form:"wm_padding_x" json:"wm_padding_x"`
+	WmPaddingY  int     `form:"wm_padding_y" json:"wm_padding_y"`
+	WmScale     float64 `form:"wm_scale" json:"wm_scale"`
+}
+
+// WMInit init UploadParam
+func (up *UploadParam) WMInit() {
+	if up.WmKey != "" || up.WmText != "" {
+		if up.WmPaddingX < 0 {
+			up.WmPaddingX = _defaultWmPaddingX
+		}
+		if up.WmPaddingY < 0 {
+			up.WmPaddingY = _defaultWmPaddingY
+		}
+		if up.WmScale <= 0 {
+			up.WmScale = _defaultWmScale
+		}
+	}
+}
+
+// UploadResult .
+type UploadResult struct {
+	Location string `json:"location"`
+	Etag     string `json:"etag"`
+}
+
+// MultiListResult .
+type MultiListResult struct {
+	Bucket string    `json:"bucket"`
+	Imgs   []*Record `json:"imgs"`
+}

app/admin/main/upload/model/record.go

@@ -0,0 +1,32 @@
+package model
+
+import xtime "go-common/library/time"
+
+// Record .
+type Record struct {
+	ID       int        `json:"id" gorm:"column:id"`
+	Bucket   string     `json:"bucket" gorm:"column:bucket"`
+	FileName string     `json:"filename" gorm:"column:filename"`
+	AdminID  int        `json:"admin_id" gorm:"column:adminid"`
+	State    int        `json:"state" gorm:"column:state"`
+	CTime    xtime.Time `json:"ctime" gorm:"column:ctime"`
+	MTime    xtime.Time `json:"mtime" gorm:"column:mtime"`
+	URL      string     `json:"url" gorm:"url"`
+	Sex      int        `json:"sex" gorm:"sex"`
+	Politics int        `json:"politics" gorm:"politics"`
+}
+
+// TableName .
+func (Record) TableName() string {
+	return "upload_yellowing"
+}
+
+// TinyRecord .
+type TinyRecord struct {
+	Rid int `gorm:"column:id"`
+}
+
+// TableName .
+func (TinyRecord) TableName() string {
+	return "upload_yellowing"
+}

app/admin/main/upload/service/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucket.go",
+        "dir.go",
+        "file.go",
+        "service.go",
+        "upload.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/dao:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/service/bucket.go

@@ -0,0 +1,95 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/log"
+)
+
+// ListBucket .
+func (s *Service) ListBucket(c context.Context, lbp *model.ListBucketParam) (bucketPage *model.BucketListPage, err error) {
+	var (
+		buckets []*model.Bucket
+		count   int
+	)
+	if err = s.orm.Table("bucket").Order("id desc").Limit(lbp.PS).Offset((lbp.PN - 1) * lbp.PS).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	if err = s.orm.Table("bucket").Count(&count).Error; err != nil {
+		log.Error("bucket count error(%v)", err)
+		err = nil
+	}
+	bucketPage = &model.BucketListPage{
+		Items: buckets,
+		Page: &model.Page{
+			PS:    lbp.PS,
+			PN:    lbp.PN,
+			Total: count,
+		},
+	}
+	return
+}
+
+// ListPublicBucket .
+func (s *Service) ListPublicBucket(c context.Context, lbp *model.ListBucketParam) (bucketPage *model.BucketListPage, err error) {
+	var (
+		buckets []*model.Bucket
+		count   int
+	)
+	if err = s.orm.Table("bucket").Order("id desc").Limit(lbp.PS).Offset((lbp.PN-1)*lbp.PS).Where("property != ? AND property != ?", model.PrivateRead, model.PrivateReadWrite).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	if err = s.orm.Table("bucket").Count(&count).Error; err != nil {
+		log.Error("bucket count error(%v)", err)
+		err = nil
+	}
+	bucketPage = &model.BucketListPage{
+		Items: buckets,
+		Page: &model.Page{
+			PS:    lbp.PS,
+			PN:    lbp.PN,
+			Total: count,
+		},
+	}
+	return
+}
+
+// AddBucket .
+func (s *Service) AddBucket(c context.Context, abp *model.AddBucketParam) (err error) {
+	b := &model.Bucket{BucketName: abp.Name}
+	if err = s.orm.Table("bucket").Where("bucket_name=?", abp.Name).
+		Assign(map[string]interface{}{
+			"key_id":        abp.KeyID,
+			"key_secret":    abp.KeySecret,
+			"purge_cdn":     abp.PurgeCDN,
+			"property":      abp.Property,
+			"cache_control": abp.CacheControl,
+			"domain":        abp.Domain,
+		}).FirstOrCreate(b).Error; err != nil {
+		log.Error("Failed to add bucket (%+v): %v", b, err)
+		return
+	}
+	return s.dao.CreateTable(context.Background(), abp.Name)
+}
+
+// DetailBucket .
+func (s *Service) DetailBucket(c context.Context, bucketName string) (bucket *model.Bucket, err error) {
+	var (
+		limits []*model.DirLimit
+	)
+	bucket = &model.Bucket{}
+	if err = s.orm.Table("bucket").Where("bucket_name = ?", bucketName).Find(bucket).Error; err != nil {
+		return
+	}
+	if err = s.orm.Table("dir_limit").Where("bucket_name = ?", bucket.BucketName).Find(&limits).Error; err != nil {
+		return
+	}
+	if len(limits) == 0 {
+		return
+	}
+	bucket.DirLimit = limits
+	return
+}

app/admin/main/upload/service/dir.go

@@ -0,0 +1,26 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/log"
+)
+
+// AddDir .
+func (s *Service) AddDir(c context.Context, adp *model.AddDirParam) (err error) {
+	d := &model.DirLimit{}
+	if err = s.orm.Model(d).
+		Where(&model.DirLimit{BucketName: adp.BucketName, Dir: adp.DirName}).
+		Assign(&model.DirLimit{
+			BucketName: adp.BucketName,
+			Dir:        adp.DirName,
+			ConfigPic:  adp.Pic,
+			ConfigRate: adp.Rate,
+		}).
+		FirstOrCreate(d).Error; err != nil {
+		log.Error("Failed to add dir (%+v): %v", d, err)
+		return
+	}
+	return
+}

app/admin/main/upload/service/file.go

@@ -0,0 +1,37 @@
+package service
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+
+	"github.com/pkg/errors"
+)
+
+// UploadAdminRecord upload file to bfs
+func (s *Service) UploadAdminRecord(ctx context.Context, action string, up *model.UploadParam, data []byte) (result *model.UploadResult, err error) {
+	var (
+		location, etag string
+		b              *model.Bucket
+		ok             bool
+	)
+	if b, ok = s.bucketCache[up.Bucket]; !ok {
+		err = errors.Errorf("read bucket items failed: (%s)", up.Bucket)
+		return
+	}
+	// auth calc.
+	up.Auth = s.dao.Bfs.Authorize(b.KeyID, b.KeySecret, http.MethodPut, up.Bucket, up.FileName, time.Now().Unix())
+	if up.ContentType == "" {
+		up.ContentType = http.DetectContentType(data)
+	}
+	if location, etag, err = s.dao.Bfs.Upload(ctx, up, data); err != nil {
+		return
+	}
+	result = &model.UploadResult{
+		Location: location,
+		Etag:     etag,
+	}
+	return
+}

app/admin/main/upload/service/service.go

@@ -0,0 +1,64 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/dao"
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/database/orm"
+	"go-common/library/log"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Service struct
+type Service struct {
+	c           *conf.Config
+	orm         *gorm.DB
+	bucketCache map[string]*model.Bucket
+	dao         *dao.Dao
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		orm: orm.NewMySQL(c.ORM),
+		dao: dao.New(c),
+	}
+	s.bucketCache = make(map[string]*model.Bucket)
+	go s.cacheproc()
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return nil
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.orm.Close()
+}
+
+func (s *Service) cacheproc() {
+	for {
+		s.loadcache()
+		time.Sleep(5 * time.Minute)
+	}
+}
+
+func (s *Service) loadcache() {
+	var buckets []*model.Bucket
+	if err := s.orm.Table("bucket").Order("id desc").Limit(1000).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	b := make(map[string]*model.Bucket)
+	for _, v := range buckets {
+		b[v.BucketName] = v
+	}
+	s.bucketCache = b
+}

app/admin/main/upload/service/service_test.go

@@ -0,0 +1,35 @@
+package service
+
+import (
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var svr *Service
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/push-admin-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	svr = New(conf.Conf)
+	time.Sleep(time.Second)
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		Reset(func() {})
+		f(svr)
+	}
+}
+
+func Test_Service(t *testing.T) {
+	Convey("service test", t, WithService(func(s *Service) {
+		s.Close()
+	}))
+}

app/admin/main/upload/service/upload.go

@@ -0,0 +1,285 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_downloadURL            = "http://%s/bfs/%s/%s"
+	_uploadURL              = "http://%s/bfs/%s/%s"
+	_deleteURL              = "http://%s/bfs/%s/%s"
+	_privateBucket          = "facepri" // bucket to save yellow pic
+	_privateBucketAppKey    = "8923aff2e1124bb2"
+	_privateBucketAppSecret = "b237e8927823cc2984aee980123cb0"
+)
+
+// Add will add a record into bfs_upload_admin table
+func (s *Service) Add(c context.Context, ap *model.AddParam) (err error) {
+	record := new(model.Record)
+	record.Bucket = ap.Bucket
+	record.FileName = ap.FileName
+	record.URL = ap.URL
+	record.Sex = ap.Sex
+	record.Politics = ap.Politics
+
+	err = s.orm.Create(&record).Error
+	return
+}
+
+// List lists records
+func (s *Service) List(c context.Context, lp *model.ListParam) (listResSlice []*model.Record, err error) {
+	listResSlice = make([]*model.Record, 0)
+	err = s.orm.Limit(10).Order("id desc").Where("state=?", lp.State).Where("bucket=?", lp.Bucket).Find(&listResSlice).Error
+	return
+}
+
+// MultiList lists records from multi bucket
+func (s *Service) MultiList(c context.Context, lp *model.MultiListParam) (result []*model.MultiListResult, err error) {
+	result = make([]*model.MultiListResult, 0)
+	if len(lp.Bucket) == 0 {
+		var buckets []*model.Bucket
+		if err = s.orm.Table("bucket").Order("id desc").Find(&buckets).Error; err != nil {
+			log.Error("read bucket error(%v)", err)
+			return
+		}
+		for _, v := range buckets {
+			lp.Bucket = append(lp.Bucket, v.BucketName)
+		}
+	}
+	for _, bucket := range lp.Bucket {
+		tmpResult := &model.MultiListResult{}
+		tmpResult.Bucket = bucket
+		tmpRecord := make([]*model.Record, 0)
+		if err = s.orm.Limit(10).Order("id desc").Where("state=?", lp.State).Where("bucket=?", bucket).Find(&tmpRecord).Error; err != nil {
+			return
+		}
+		tmpResult.Imgs = tmpRecord
+		result = append(result, tmpResult)
+	}
+	return
+}
+
+// Delete deletes a record and delete file in bfs
+func (s *Service) Delete(c context.Context, dp *model.DeleteParam) (err error) {
+	var (
+		downloadBytes []byte
+		contentType   string
+	)
+	record := new(model.Record)
+	if err = s.orm.Where("id=?", dp.Rid).Find(&record).Error; err != nil {
+		err = errors.Wrapf(err, "Query(%d)", dp.Rid)
+		return
+	}
+	dp.Bucket = record.Bucket
+	dp.FileName = record.FileName
+	if downloadBytes, contentType, err = s.download(dp); err != nil {
+		return
+	}
+	if err = s.upload(dp, contentType, downloadBytes); err != nil {
+		return
+	}
+	if err = s.delete(dp); err != nil {
+		return
+	}
+	if err = s.orm.Where("id=?", dp.Rid).Update("state", 1).Update("adminid", dp.AdminID).Error; err != nil {
+		err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, 1, dp.AdminID)
+		return
+	}
+	return
+}
+
+// DeleteRaw delete file in bfs
+func (s *Service) DeleteRaw(c context.Context, dp *model.DeleteRawParam) (err error) {
+	d := &model.DeleteParam{
+		Bucket:   dp.Bucket,
+		FileName: dp.FileName,
+	}
+	return s.delete(d)
+}
+
+// download from bfs
+func (s *Service) download(dp *model.DeleteParam) (downloadBytes []byte, contentType string, err error) {
+	var (
+		downloadReq    *http.Request
+		resp           *http.Response
+		bfsDownloadURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsDownloadURL = fmt.Sprintf(_downloadURL, s.c.BfsDownloadHost, dp.Bucket, dp.FileName)
+	if downloadReq, err = http.NewRequest(http.MethodGet, bfsDownloadURL, nil); err != nil {
+		log.Error("client.NewRequest(%s) error(%v)", bfsDownloadURL, err)
+		return
+	}
+	if resp, err = client.Do(downloadReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", downloadReq, err)
+		return
+	}
+	contentType = resp.Header.Get("Content-Type")
+	if downloadBytes, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("ioutil.ReadAll(%v) error(%v)", resp.Body, err)
+		return
+	}
+	return
+}
+
+// upload file to facepri bucket
+func (s *Service) upload(dp *model.DeleteParam, contentType string, body []byte) (err error) {
+	var (
+		uploadReq    *http.Request
+		resp         *http.Response
+		bfsUploadURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsUploadURL = fmt.Sprintf(_uploadURL, s.c.BfsUpdateHost, dp.Bucket, dp.FileName)
+	if uploadReq, err = http.NewRequest(http.MethodPut, bfsUploadURL, bytes.NewReader(body)); err != nil {
+		return
+	}
+	auth := s.authorize(_privateBucketAppKey, _privateBucketAppSecret, http.MethodPut, _privateBucket, dp.FileName, time.Now().Unix())
+	uploadReq.Header.Add("Host", "bfs.bilibili.co")
+	uploadReq.Header.Add("Date", time.Now().String())
+	uploadReq.Header.Add("Authorization", auth)
+	uploadReq.Header.Add("Content-Type", contentType)
+	uploadReq.Header.Add("Date", fmt.Sprint(time.Now().Unix()))
+	if resp, err = client.Do(uploadReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", uploadReq, err)
+		return
+	}
+	// judge response code
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.RequestErr
+		return
+	case http.StatusUnauthorized:
+		// 验证不通过
+		err = ecode.BfsUploadAuthErr
+		return
+	case http.StatusRequestEntityTooLarge:
+		err = ecode.FileTooLarge
+		return
+	case http.StatusNotFound:
+		err = ecode.NothingFound
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusServiceUnavailable:
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.BfsUploadStatusErr
+		return
+	}
+	code, err := strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = ecode.BfsUploadCodeErr
+		return
+	}
+	return
+}
+
+// delete file in old bucket
+func (s *Service) delete(dp *model.DeleteParam) (err error) {
+	var (
+		deleteReq    *http.Request
+		resp         *http.Response
+		bfsDeleteURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsDeleteURL = fmt.Sprintf(_deleteURL, s.c.BfsDeleteHost, dp.Bucket, dp.FileName)
+	if deleteReq, err = http.NewRequest("DELETE", bfsDeleteURL, nil); err != nil {
+		log.Error("client.NewRequest(%s) error(%v)", bfsDeleteURL, err)
+		return
+	}
+	item, ok := s.bucketCache[dp.Bucket]
+	if !ok {
+		err = errors.Wrapf(ecode.NothingFound, "bucket not exist: %s", dp.Bucket)
+		log.Error("bucket not exist: %s", dp.Bucket)
+		return
+	}
+	deleteReq.Header.Add("Host", "bfs.bilibili.co")
+	deleteReq.Header.Add("Date", fmt.Sprint(time.Now().Unix()))
+	deleteReq.Header.Add("Authorization", s.authorize(item.KeyID, item.KeySecret, http.MethodDelete, dp.Bucket, dp.FileName, time.Now().Unix()))
+	if resp, err = client.Do(deleteReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", deleteReq, err)
+		return
+	}
+	if resp.StatusCode != 200 {
+		log.Error("bfs delete error code: %d", resp.StatusCode)
+		return
+	}
+	return
+}
+
+// authorize return token
+func (s *Service) authorize(key, secret, method, bucket, fileName string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf("%s\n%s\n%s\n%d\n", method, bucket, fileName, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}
+
+// DeleteV2 deletes a record and delete file in bfs
+func (s *Service) DeleteV2(c context.Context, dp *model.DeleteV2Param) (err error) {
+	switch dp.Status {
+	case model.PassStatus:
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Update("state", model.PassStatus).Update("adminid", dp.AdminID).Error; err != nil {
+			err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, model.PassStatus, dp.AdminID)
+			return
+		}
+	case model.DeleteStatus:
+		record := new(model.Record)
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Find(&record).Error; err != nil {
+			err = errors.Wrapf(err, "Query(%d)", dp.Rid)
+			return
+		}
+		dp.Bucket = record.Bucket
+		dp.FileName = record.FileName
+		if err = s.delete(&model.DeleteParam{
+			Bucket:   dp.Bucket,
+			FileName: dp.FileName,
+		}); err != nil {
+			return
+		}
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Update("state", model.DeleteStatus).Update("adminid", dp.AdminID).Error; err != nil {
+			err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, model.DeleteStatus, dp.AdminID)
+			return
+		}
+	default:
+		err = errors.Wrapf(err, "illegal Status(%d,%d,%d)", dp.Rid, model.DeleteStatus, dp.AdminID)
+		return
+	}
+	return
+}