Create & Init Project...

app/admin/main/upload/service/BUILD

@@ -0,0 +1,57 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["service_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucket.go",
+        "dir.go",
+        "file.go",
+        "service.go",
+        "upload.go",
+    ],
+    importpath = "go-common/app/admin/main/upload/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/upload/conf:go_default_library",
+        "//app/admin/main/upload/dao:go_default_library",
+        "//app/admin/main/upload/model:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/main/upload/service/bucket.go

@@ -0,0 +1,95 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/log"
+)
+
+// ListBucket .
+func (s *Service) ListBucket(c context.Context, lbp *model.ListBucketParam) (bucketPage *model.BucketListPage, err error) {
+	var (
+		buckets []*model.Bucket
+		count   int
+	)
+	if err = s.orm.Table("bucket").Order("id desc").Limit(lbp.PS).Offset((lbp.PN - 1) * lbp.PS).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	if err = s.orm.Table("bucket").Count(&count).Error; err != nil {
+		log.Error("bucket count error(%v)", err)
+		err = nil
+	}
+	bucketPage = &model.BucketListPage{
+		Items: buckets,
+		Page: &model.Page{
+			PS:    lbp.PS,
+			PN:    lbp.PN,
+			Total: count,
+		},
+	}
+	return
+}
+
+// ListPublicBucket .
+func (s *Service) ListPublicBucket(c context.Context, lbp *model.ListBucketParam) (bucketPage *model.BucketListPage, err error) {
+	var (
+		buckets []*model.Bucket
+		count   int
+	)
+	if err = s.orm.Table("bucket").Order("id desc").Limit(lbp.PS).Offset((lbp.PN-1)*lbp.PS).Where("property != ? AND property != ?", model.PrivateRead, model.PrivateReadWrite).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	if err = s.orm.Table("bucket").Count(&count).Error; err != nil {
+		log.Error("bucket count error(%v)", err)
+		err = nil
+	}
+	bucketPage = &model.BucketListPage{
+		Items: buckets,
+		Page: &model.Page{
+			PS:    lbp.PS,
+			PN:    lbp.PN,
+			Total: count,
+		},
+	}
+	return
+}
+
+// AddBucket .
+func (s *Service) AddBucket(c context.Context, abp *model.AddBucketParam) (err error) {
+	b := &model.Bucket{BucketName: abp.Name}
+	if err = s.orm.Table("bucket").Where("bucket_name=?", abp.Name).
+		Assign(map[string]interface{}{
+			"key_id":        abp.KeyID,
+			"key_secret":    abp.KeySecret,
+			"purge_cdn":     abp.PurgeCDN,
+			"property":      abp.Property,
+			"cache_control": abp.CacheControl,
+			"domain":        abp.Domain,
+		}).FirstOrCreate(b).Error; err != nil {
+		log.Error("Failed to add bucket (%+v): %v", b, err)
+		return
+	}
+	return s.dao.CreateTable(context.Background(), abp.Name)
+}
+
+// DetailBucket .
+func (s *Service) DetailBucket(c context.Context, bucketName string) (bucket *model.Bucket, err error) {
+	var (
+		limits []*model.DirLimit
+	)
+	bucket = &model.Bucket{}
+	if err = s.orm.Table("bucket").Where("bucket_name = ?", bucketName).Find(bucket).Error; err != nil {
+		return
+	}
+	if err = s.orm.Table("dir_limit").Where("bucket_name = ?", bucket.BucketName).Find(&limits).Error; err != nil {
+		return
+	}
+	if len(limits) == 0 {
+		return
+	}
+	bucket.DirLimit = limits
+	return
+}

app/admin/main/upload/service/dir.go

@@ -0,0 +1,26 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/log"
+)
+
+// AddDir .
+func (s *Service) AddDir(c context.Context, adp *model.AddDirParam) (err error) {
+	d := &model.DirLimit{}
+	if err = s.orm.Model(d).
+		Where(&model.DirLimit{BucketName: adp.BucketName, Dir: adp.DirName}).
+		Assign(&model.DirLimit{
+			BucketName: adp.BucketName,
+			Dir:        adp.DirName,
+			ConfigPic:  adp.Pic,
+			ConfigRate: adp.Rate,
+		}).
+		FirstOrCreate(d).Error; err != nil {
+		log.Error("Failed to add dir (%+v): %v", d, err)
+		return
+	}
+	return
+}

app/admin/main/upload/service/file.go

@@ -0,0 +1,37 @@
+package service
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+
+	"github.com/pkg/errors"
+)
+
+// UploadAdminRecord upload file to bfs
+func (s *Service) UploadAdminRecord(ctx context.Context, action string, up *model.UploadParam, data []byte) (result *model.UploadResult, err error) {
+	var (
+		location, etag string
+		b              *model.Bucket
+		ok             bool
+	)
+	if b, ok = s.bucketCache[up.Bucket]; !ok {
+		err = errors.Errorf("read bucket items failed: (%s)", up.Bucket)
+		return
+	}
+	// auth calc.
+	up.Auth = s.dao.Bfs.Authorize(b.KeyID, b.KeySecret, http.MethodPut, up.Bucket, up.FileName, time.Now().Unix())
+	if up.ContentType == "" {
+		up.ContentType = http.DetectContentType(data)
+	}
+	if location, etag, err = s.dao.Bfs.Upload(ctx, up, data); err != nil {
+		return
+	}
+	result = &model.UploadResult{
+		Location: location,
+		Etag:     etag,
+	}
+	return
+}

app/admin/main/upload/service/service.go

@@ -0,0 +1,64 @@
+package service
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+	"go-common/app/admin/main/upload/dao"
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/database/orm"
+	"go-common/library/log"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Service struct
+type Service struct {
+	c           *conf.Config
+	orm         *gorm.DB
+	bucketCache map[string]*model.Bucket
+	dao         *dao.Dao
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:   c,
+		orm: orm.NewMySQL(c.ORM),
+		dao: dao.New(c),
+	}
+	s.bucketCache = make(map[string]*model.Bucket)
+	go s.cacheproc()
+	return s
+}
+
+// Ping Service
+func (s *Service) Ping(c context.Context) (err error) {
+	return nil
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.orm.Close()
+}
+
+func (s *Service) cacheproc() {
+	for {
+		s.loadcache()
+		time.Sleep(5 * time.Minute)
+	}
+}
+
+func (s *Service) loadcache() {
+	var buckets []*model.Bucket
+	if err := s.orm.Table("bucket").Order("id desc").Limit(1000).Find(&buckets).Error; err != nil {
+		log.Error("read bucket error(%v)", err)
+		return
+	}
+	b := make(map[string]*model.Bucket)
+	for _, v := range buckets {
+		b[v.BucketName] = v
+	}
+	s.bucketCache = b
+}

app/admin/main/upload/service/service_test.go

@@ -0,0 +1,35 @@
+package service
+
+import (
+	"flag"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go-common/app/admin/main/upload/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var svr *Service
+
+func init() {
+	dir, _ := filepath.Abs("../cmd/push-admin-test.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	svr = New(conf.Conf)
+	time.Sleep(time.Second)
+}
+
+func WithService(f func(s *Service)) func() {
+	return func() {
+		Reset(func() {})
+		f(svr)
+	}
+}
+
+func Test_Service(t *testing.T) {
+	Convey("service test", t, WithService(func(s *Service) {
+		s.Close()
+	}))
+}

app/admin/main/upload/service/upload.go

@@ -0,0 +1,285 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_downloadURL            = "http://%s/bfs/%s/%s"
+	_uploadURL              = "http://%s/bfs/%s/%s"
+	_deleteURL              = "http://%s/bfs/%s/%s"
+	_privateBucket          = "facepri" // bucket to save yellow pic
+	_privateBucketAppKey    = "8923aff2e1124bb2"
+	_privateBucketAppSecret = "b237e8927823cc2984aee980123cb0"
+)
+
+// Add will add a record into bfs_upload_admin table
+func (s *Service) Add(c context.Context, ap *model.AddParam) (err error) {
+	record := new(model.Record)
+	record.Bucket = ap.Bucket
+	record.FileName = ap.FileName
+	record.URL = ap.URL
+	record.Sex = ap.Sex
+	record.Politics = ap.Politics
+
+	err = s.orm.Create(&record).Error
+	return
+}
+
+// List lists records
+func (s *Service) List(c context.Context, lp *model.ListParam) (listResSlice []*model.Record, err error) {
+	listResSlice = make([]*model.Record, 0)
+	err = s.orm.Limit(10).Order("id desc").Where("state=?", lp.State).Where("bucket=?", lp.Bucket).Find(&listResSlice).Error
+	return
+}
+
+// MultiList lists records from multi bucket
+func (s *Service) MultiList(c context.Context, lp *model.MultiListParam) (result []*model.MultiListResult, err error) {
+	result = make([]*model.MultiListResult, 0)
+	if len(lp.Bucket) == 0 {
+		var buckets []*model.Bucket
+		if err = s.orm.Table("bucket").Order("id desc").Find(&buckets).Error; err != nil {
+			log.Error("read bucket error(%v)", err)
+			return
+		}
+		for _, v := range buckets {
+			lp.Bucket = append(lp.Bucket, v.BucketName)
+		}
+	}
+	for _, bucket := range lp.Bucket {
+		tmpResult := &model.MultiListResult{}
+		tmpResult.Bucket = bucket
+		tmpRecord := make([]*model.Record, 0)
+		if err = s.orm.Limit(10).Order("id desc").Where("state=?", lp.State).Where("bucket=?", bucket).Find(&tmpRecord).Error; err != nil {
+			return
+		}
+		tmpResult.Imgs = tmpRecord
+		result = append(result, tmpResult)
+	}
+	return
+}
+
+// Delete deletes a record and delete file in bfs
+func (s *Service) Delete(c context.Context, dp *model.DeleteParam) (err error) {
+	var (
+		downloadBytes []byte
+		contentType   string
+	)
+	record := new(model.Record)
+	if err = s.orm.Where("id=?", dp.Rid).Find(&record).Error; err != nil {
+		err = errors.Wrapf(err, "Query(%d)", dp.Rid)
+		return
+	}
+	dp.Bucket = record.Bucket
+	dp.FileName = record.FileName
+	if downloadBytes, contentType, err = s.download(dp); err != nil {
+		return
+	}
+	if err = s.upload(dp, contentType, downloadBytes); err != nil {
+		return
+	}
+	if err = s.delete(dp); err != nil {
+		return
+	}
+	if err = s.orm.Where("id=?", dp.Rid).Update("state", 1).Update("adminid", dp.AdminID).Error; err != nil {
+		err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, 1, dp.AdminID)
+		return
+	}
+	return
+}
+
+// DeleteRaw delete file in bfs
+func (s *Service) DeleteRaw(c context.Context, dp *model.DeleteRawParam) (err error) {
+	d := &model.DeleteParam{
+		Bucket:   dp.Bucket,
+		FileName: dp.FileName,
+	}
+	return s.delete(d)
+}
+
+// download from bfs
+func (s *Service) download(dp *model.DeleteParam) (downloadBytes []byte, contentType string, err error) {
+	var (
+		downloadReq    *http.Request
+		resp           *http.Response
+		bfsDownloadURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsDownloadURL = fmt.Sprintf(_downloadURL, s.c.BfsDownloadHost, dp.Bucket, dp.FileName)
+	if downloadReq, err = http.NewRequest(http.MethodGet, bfsDownloadURL, nil); err != nil {
+		log.Error("client.NewRequest(%s) error(%v)", bfsDownloadURL, err)
+		return
+	}
+	if resp, err = client.Do(downloadReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", downloadReq, err)
+		return
+	}
+	contentType = resp.Header.Get("Content-Type")
+	if downloadBytes, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("ioutil.ReadAll(%v) error(%v)", resp.Body, err)
+		return
+	}
+	return
+}
+
+// upload file to facepri bucket
+func (s *Service) upload(dp *model.DeleteParam, contentType string, body []byte) (err error) {
+	var (
+		uploadReq    *http.Request
+		resp         *http.Response
+		bfsUploadURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsUploadURL = fmt.Sprintf(_uploadURL, s.c.BfsUpdateHost, dp.Bucket, dp.FileName)
+	if uploadReq, err = http.NewRequest(http.MethodPut, bfsUploadURL, bytes.NewReader(body)); err != nil {
+		return
+	}
+	auth := s.authorize(_privateBucketAppKey, _privateBucketAppSecret, http.MethodPut, _privateBucket, dp.FileName, time.Now().Unix())
+	uploadReq.Header.Add("Host", "bfs.bilibili.co")
+	uploadReq.Header.Add("Date", time.Now().String())
+	uploadReq.Header.Add("Authorization", auth)
+	uploadReq.Header.Add("Content-Type", contentType)
+	uploadReq.Header.Add("Date", fmt.Sprint(time.Now().Unix()))
+	if resp, err = client.Do(uploadReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", uploadReq, err)
+		return
+	}
+	// judge response code
+	switch resp.StatusCode {
+	case http.StatusOK:
+	case http.StatusBadRequest:
+		err = ecode.RequestErr
+		return
+	case http.StatusUnauthorized:
+		// 验证不通过
+		err = ecode.BfsUploadAuthErr
+		return
+	case http.StatusRequestEntityTooLarge:
+		err = ecode.FileTooLarge
+		return
+	case http.StatusNotFound:
+		err = ecode.NothingFound
+		return
+	case http.StatusMethodNotAllowed:
+		err = ecode.MethodNotAllowed
+		return
+	case http.StatusServiceUnavailable:
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	case http.StatusInternalServerError:
+		err = ecode.ServerErr
+		return
+	default:
+		err = ecode.BfsUploadStatusErr
+		return
+	}
+	code, err := strconv.Atoi(resp.Header.Get("code"))
+	if err != nil || code != 200 {
+		err = ecode.BfsUploadCodeErr
+		return
+	}
+	return
+}
+
+// delete file in old bucket
+func (s *Service) delete(dp *model.DeleteParam) (err error) {
+	var (
+		deleteReq    *http.Request
+		resp         *http.Response
+		bfsDeleteURL string
+	)
+	client := &http.Client{
+		Timeout: time.Duration(s.c.HTTPClient.Read.Timeout),
+	}
+	bfsDeleteURL = fmt.Sprintf(_deleteURL, s.c.BfsDeleteHost, dp.Bucket, dp.FileName)
+	if deleteReq, err = http.NewRequest("DELETE", bfsDeleteURL, nil); err != nil {
+		log.Error("client.NewRequest(%s) error(%v)", bfsDeleteURL, err)
+		return
+	}
+	item, ok := s.bucketCache[dp.Bucket]
+	if !ok {
+		err = errors.Wrapf(ecode.NothingFound, "bucket not exist: %s", dp.Bucket)
+		log.Error("bucket not exist: %s", dp.Bucket)
+		return
+	}
+	deleteReq.Header.Add("Host", "bfs.bilibili.co")
+	deleteReq.Header.Add("Date", fmt.Sprint(time.Now().Unix()))
+	deleteReq.Header.Add("Authorization", s.authorize(item.KeyID, item.KeySecret, http.MethodDelete, dp.Bucket, dp.FileName, time.Now().Unix()))
+	if resp, err = client.Do(deleteReq); err != nil {
+		log.Error("client.Do(%v) error(%v)", deleteReq, err)
+		return
+	}
+	if resp.StatusCode != 200 {
+		log.Error("bfs delete error code: %d", resp.StatusCode)
+		return
+	}
+	return
+}
+
+// authorize return token
+func (s *Service) authorize(key, secret, method, bucket, fileName string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf("%s\n%s\n%s\n%d\n", method, bucket, fileName, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}
+
+// DeleteV2 deletes a record and delete file in bfs
+func (s *Service) DeleteV2(c context.Context, dp *model.DeleteV2Param) (err error) {
+	switch dp.Status {
+	case model.PassStatus:
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Update("state", model.PassStatus).Update("adminid", dp.AdminID).Error; err != nil {
+			err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, model.PassStatus, dp.AdminID)
+			return
+		}
+	case model.DeleteStatus:
+		record := new(model.Record)
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Find(&record).Error; err != nil {
+			err = errors.Wrapf(err, "Query(%d)", dp.Rid)
+			return
+		}
+		dp.Bucket = record.Bucket
+		dp.FileName = record.FileName
+		if err = s.delete(&model.DeleteParam{
+			Bucket:   dp.Bucket,
+			FileName: dp.FileName,
+		}); err != nil {
+			return
+		}
+		if err = s.orm.Table("upload_yellowing").Where("id=?", dp.Rid).Update("state", model.DeleteStatus).Update("adminid", dp.AdminID).Error; err != nil {
+			err = errors.Wrapf(err, "Update(%d,%d,%d)", dp.Rid, model.DeleteStatus, dp.AdminID)
+			return
+		}
+	default:
+		err = errors.Wrapf(err, "illegal Status(%d,%d,%d)", dp.Rid, model.DeleteStatus, dp.AdminID)
+		return
+	}
+	return
+}