Create & Init Project...

app/interface/main/creative/dao/bfs/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cover.go",
+        "dao.go",
+    ],
+    importpath = "go-common/app/interface/main/creative/dao/bfs",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/creative/conf:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cover_test.go",
+        "dao_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/creative/conf:go_default_library",
+        "//vendor/github.com/bouk/monkey:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/gopkg.in/h2non/gock.v1:go_default_library",
+    ],
+)

app/interface/main/creative/dao/bfs/cover.go

@@ -0,0 +1,59 @@
+package bfs
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+
+	"fmt"
+	"go-common/library/log"
+)
+
+// UpVideoCovers upload video covers.
+func (d *Dao) UpVideoCovers(c context.Context, covers []string) (cvs []string, err error) {
+	var (
+		nfsURL string
+	)
+	for _, cv := range covers {
+		// get nfs file
+		bs, err := d.bvcCover(cv)
+		if err != nil || len(bs) == 0 {
+			log.Error("d.UpVideoCovers(%s) error(%v) or bs==0", nfsURL, err)
+			continue
+		}
+		// up to bfs
+		bfsPath, err := d.UploadArc(c, http.DetectContentType(bs), bytes.NewReader(bs))
+		if err != nil {
+			log.Error("d.UpVideoCovers raw url(%s) error(%v)", cv, err)
+			continue
+		}
+		// parse bfs return path
+		if err != nil {
+			log.Error("url.Parse(%v) error(%v)", bfsPath, err)
+			continue
+		}
+		cvs = append(cvs, bfsPath)
+		log.Info("UpVideoCovers cover(%s) bfs (%s)", cv, bfsPath)
+	}
+	return
+}
+
+// bvcCover http get bvc cover bytes.
+func (d *Dao) bvcCover(url string) (bs []byte, err error) {
+	resp, err := d.client.Get(url)
+	if err != nil {
+		log.Error("s.client.Get(%v) error(%v)", url, err)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("get NFS file faild, url(%s) http status: %d", url, resp.StatusCode)
+		return
+	}
+	// read bytes
+	if bs, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("ioutil.ReadAll error(%v)", err)
+	}
+	return
+}

app/interface/main/creative/dao/bfs/cover_test.go

@@ -0,0 +1,42 @@
+package bfs
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var mockHeader = map[string]string{"location": "mockLocation", "code": "200"}
+
+func TestBfsUpVideoCovers(t *testing.T) {
+	var (
+		c      = context.TODO()
+		covers = []string{
+			"http://static.hdslb.com/images/transparent.gif",
+		}
+	)
+	convey.Convey("UpVideoCovers", t, func(ctx convey.C) {
+		httpMock(_method, _url).Reply(200).SetHeaders(mockHeader)
+		httpMock("GET", covers[0]).Reply(200).JSON("mock byte")
+		cvs, err := d.UpVideoCovers(c, covers)
+		ctx.Convey("Then err should be nil.cvs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(cvs, convey.ShouldResemble, []string{"mockLocation"})
+		})
+	})
+}
+
+func TestBfsbvcCover(t *testing.T) {
+	var (
+		url = "http://static.hdslb.com/images/transparent.gif"
+	)
+	convey.Convey("bvcCover", t, func(ctx convey.C) {
+		httpMock("GET", url).Reply(200).JSON("mock byte")
+		bs, err := d.bvcCover(url)
+		ctx.Convey("Then err should be nil.bs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(bs, convey.ShouldNotBeNil)
+		})
+	})
+}

app/interface/main/creative/dao/bfs/dao.go

@@ -0,0 +1,316 @@
+package bfs
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	nurl "net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/interface/main/creative/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_bucket = "archive"
+	_url    = "http://bfs.bilibili.co/bfs/archive/"
+	_method = "PUT"
+	_key    = "8d4e593ba7555502"
+	_secret = "0bdbd4c7caeeddf587c3c4daec0475"
+)
+
+var (
+	errUpload   = errors.New("Upload failed")
+	errDownload = errors.New("Download out image link failed")
+)
+
+// Dao is bfs dao.
+type Dao struct {
+	c          *conf.Config
+	client     *http.Client
+	captureCli *http.Client
+}
+
+// New new a bfs dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c: c,
+		client: &http.Client{
+			Timeout: time.Duration(c.BFS.Timeout),
+		},
+		captureCli: &http.Client{
+			Timeout: time.Duration(time.Second),
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return ecode.RequestErr
+			},
+		},
+	}
+	return d
+}
+
+// Upload upload bfs.
+func (d *Dao) Upload(c context.Context, fileType string, bs []byte) (location string, err error) {
+	req, err := http.NewRequest(d.c.BFS.Method, d.c.BFS.URL, bytes.NewBuffer(bs))
+	if err != nil {
+		log.Error("http.NewRequest error (%v) | fileType(%s)", err, fileType)
+		return
+	}
+	expire := time.Now().Unix()
+	authorization := authorize(d.c.BFS.Key, d.c.BFS.Secret, d.c.BFS.Method, d.c.BFS.Bucket, expire)
+	req.Header.Set("Host", d.c.BFS.URL)
+	req.Header.Add("Date", fmt.Sprint(expire))
+	req.Header.Add("Authorization", authorization)
+	req.Header.Add("Content-Type", fileType)
+	// timeout
+	ctx, cancel := context.WithTimeout(c, time.Duration(d.c.BFS.Timeout))
+	req = req.WithContext(ctx)
+	defer cancel()
+	resp, err := d.client.Do(req)
+	if err != nil {
+		log.Error("d.Client.Do error(%v) | url(%s)", err, d.c.BFS.URL)
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Error("Upload http.StatusCode nq http.StatusOK (%d) | url(%s)", resp.StatusCode, d.c.BFS.URL)
+		err = errUpload
+		return
+	}
+	header := resp.Header
+	code := header.Get("Code")
+	if code != strconv.Itoa(http.StatusOK) {
+		log.Error("strconv.Itoa err, code(%s) | url(%s)", code, d.c.BFS.URL)
+		err = errUpload
+		return
+	}
+	location = header.Get("Location")
+	return
+}
+
+// UploadArc upload bfs to archive bucket.
+func (d *Dao) UploadArc(c context.Context, fileType string, body io.Reader) (location string, err error) {
+	req, err := http.NewRequest(_method, _url, body)
+	if err != nil {
+		log.Error("http.NewRequest error (%v) | fileType(%s)", err, fileType)
+		return
+	}
+	expire := time.Now().Unix()
+	authorization := authorize(_key, _secret, _method, _bucket, expire)
+	req.Header.Set("Host", _url)
+	req.Header.Add("Date", fmt.Sprint(expire))
+	req.Header.Add("Authorization", authorization)
+	req.Header.Add("Content-Type", fileType)
+	// timeout
+	c, cancel := context.WithTimeout(c, time.Duration(d.c.BFS.Timeout))
+	req = req.WithContext(c)
+	defer cancel()
+	resp, err := d.client.Do(req)
+	if err != nil {
+		log.Error("d.Client.Do error(%v) | url(%s)", err, _url)
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Error("Upload http.StatusCode nq http.StatusOK (%d) | url(%s)", resp.StatusCode, _url)
+		err = errUpload
+		return
+	}
+	header := resp.Header
+	code := header.Get("Code")
+	if code != strconv.Itoa(http.StatusOK) {
+		log.Error("strconv.Itoa err, code(%s) | url(%s)", code, _url)
+		err = errUpload
+		return
+	}
+	location = header.Get("Location")
+	return
+}
+
+// authorize returns authorization for upload file to bfs
+func authorize(key, secret, method, bucket string, expire int64) (authorization string) {
+	var (
+		content   string
+		mac       hash.Hash
+		signature string
+	)
+	content = fmt.Sprintf("%s\n%s\n\n%d\n", method, bucket, expire)
+	mac = hmac.New(sha1.New, []byte(secret))
+	mac.Write([]byte(content))
+	signature = base64.StdEncoding.EncodeToString(mac.Sum(nil))
+	authorization = fmt.Sprintf("%s:%s:%d", key, signature, expire)
+	return
+}
+
+// UploadByFile upload local img file.
+func (d *Dao) UploadByFile(c context.Context, imgpath string) (location string, err error) {
+	data, err := ioutil.ReadFile(imgpath)
+	if err != nil {
+		log.Error("UploadByFile ioutil.ReadFile error (%v) | imgpath(%s)", err, imgpath)
+		return
+	}
+	fileType := http.DetectContentType(data)
+	if fileType != "image/jpeg" && fileType != "image/png" {
+		log.Error("file type not allow file type(%s)", fileType)
+		err = ecode.CreativeArticleImageTypeErr
+	}
+	body := new(bytes.Buffer)
+	_, err = body.Write(data)
+	if err != nil {
+		log.Error("body.Write error (%v)", err)
+		return
+	}
+	req, err := http.NewRequest(_method, _url, body)
+	if err != nil {
+		log.Error("http.NewRequest error (%v) | fileType(%s)", err, fileType)
+		return
+	}
+	expire := time.Now().Unix()
+	authorization := authorize(_key, _secret, _method, _bucket, expire)
+	req.Header.Set("Host", _url)
+	req.Header.Add("Date", fmt.Sprint(expire))
+	req.Header.Add("Authorization", authorization)
+	req.Header.Add("Content-Type", fileType)
+	// timeout
+	c, cancel := context.WithTimeout(c, time.Duration(d.c.BFS.Timeout))
+	req = req.WithContext(c)
+	defer cancel()
+	resp, err := d.client.Do(req)
+	if err != nil {
+		log.Error("d.Client.Do error(%v) | url(%s)", err, _url)
+		err = ecode.BfsUploadServiceUnavailable
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Error("Upload http.StatusCode nq http.StatusOK (%d) | url(%s)", resp.StatusCode, _url)
+		err = errUpload
+		return
+	}
+	header := resp.Header
+	code := header.Get("Code")
+	if code != strconv.Itoa(http.StatusOK) {
+		log.Error("strconv.Itoa err, code(%s) | url(%s)", code, _url)
+		err = errUpload
+		return
+	}
+	location = header.Get("Location")
+	return
+}
+
+//Capture performs a HTTP Get request for the image url and upload bfs.
+func (d *Dao) Capture(c context.Context, url string) (loc string, size int, err error) {
+	if err = checkURL(url); err != nil {
+		return
+	}
+	bs, ct, err := d.download(c, url)
+	if err != nil {
+		return
+	}
+	size = len(bs)
+	if size == 0 {
+		log.Error("capture image size(%d)|url(%s)", size, url)
+		return
+	}
+	if ct != "image/jpeg" && ct != "image/jpg" && ct != "image/png" && ct != "image/gif" {
+		log.Error("capture not allow image file type(%s)", ct)
+		err = ecode.CreativeArticleImageTypeErr
+		return
+	}
+	loc, err = d.Upload(c, ct, bs)
+	return loc, size, err
+}
+
+func (d *Dao) download(c context.Context, url string) (bs []byte, ct string, err error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		log.Error("capture http.NewRequest error(%v)|url (%s)", err, url)
+		return
+	}
+	// timeout
+	ctx, cancel := context.WithTimeout(c, 800*time.Millisecond)
+	req = req.WithContext(ctx)
+	defer cancel()
+	resp, err := d.captureCli.Do(req)
+	if err != nil {
+		log.Error("capture d.client.Do error(%v)|url(%s)", err, url)
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Error("capture http.StatusCode nq http.StatusOK(%d)|url(%s)", resp.StatusCode, url)
+		err = errDownload
+		return
+	}
+	if bs, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("capture ioutil.ReadAll error(%v)", err)
+		err = errDownload
+		return
+	}
+	ct = http.DetectContentType(bs)
+	return
+}
+
+func checkURL(url string) (err error) {
+	// http || https
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		log.Error("capture url invalid(%s)", url)
+		err = ecode.RequestErr
+		return
+	}
+	u, err := nurl.Parse(url)
+	if err != nil {
+		log.Error("capture url.Parse error(%v)", err)
+		err = ecode.RequestErr
+		return
+	}
+	// make sure ip is public. avoid ssrf
+	ips, err := net.LookupIP(u.Host) // take from 1st argument
+	if err != nil {
+		log.Error("capture url(%s) LookupIP failed", url)
+		err = ecode.RequestErr
+		return
+	}
+	if len(ips) == 0 {
+		log.Error("capture url(%s) LookupIP length 0", url)
+		err = ecode.RequestErr
+		return
+	}
+	for _, v := range ips {
+		if !isPublicIP(v) {
+			log.Error("capture url(%s) is not public ip(%v)", url, v)
+			err = ecode.RequestErr
+			return
+		}
+	}
+	return
+}
+
+func isPublicIP(IP net.IP) bool {
+	if IP.IsLoopback() || IP.IsLinkLocalMulticast() || IP.IsLinkLocalUnicast() {
+		return false
+	}
+	if ip4 := IP.To4(); ip4 != nil {
+		switch true {
+		case ip4[0] == 10:
+			return false
+		case ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31:
+			return false
+		case ip4[0] == 192 && ip4[1] == 168:
+			return false
+		default:
+			return true
+		}
+	}
+	return false
+}

app/interface/main/creative/dao/bfs/dao_test.go

@@ -0,0 +1,132 @@
+package bfs
+
+import (
+	"context"
+	"flag"
+	"github.com/bouk/monkey"
+	"github.com/smartystreets/goconvey/convey"
+	"go-common/app/interface/main/creative/conf"
+	"gopkg.in/h2non/gock.v1"
+	"io/ioutil"
+	"net"
+	"os"
+	"strings"
+	"testing"
+)
+
+var (
+	d          *Dao
+	defaultImg = "https://avatars3.githubusercontent.com/u/12002442"
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.archive.creative")
+		flag.Set("conf_token", "96b6a6c10bb311e894c14a552f48fef8")
+		flag.Set("tree_id", "2305")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/creative.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}
+
+func httpMock(method, url string) *gock.Request {
+	r := gock.New(url)
+	r.Method = strings.ToUpper(method)
+	d.client.Transport = gock.DefaultTransport
+	return r
+}
+
+func TestBfsUpload(t *testing.T) {
+	convey.Convey("Upload", t, func(ctx convey.C) {
+		var (
+			c        = context.Background()
+			fileType = ""
+			bs       = []byte("")
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			httpMock(d.c.BFS.Method, d.c.BFS.URL).Reply(200).SetHeaders(mockHeader).JSON("mockByte")
+			location, err := d.Upload(c, fileType, bs)
+			ctx.Convey("Then err should be nil.location should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestBfsUploadByFile(t *testing.T) {
+	convey.Convey("UploadByFile", t, func(ctx convey.C) {
+		var (
+			c       = context.Background()
+			imgpath = defaultImg
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			monkey.Patch(ioutil.ReadFile, func(_ string) ([]byte, error) {
+				return []byte{}, nil
+			})
+			httpMock(_method, _url).Reply(200).SetHeaders(mockHeader)
+			location, err := d.UploadByFile(c, imgpath)
+			ctx.Convey("Then err should be nil.location should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(location, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestBfsCapture(t *testing.T) {
+	convey.Convey("Capture", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			url = defaultImg
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			httpMock("GET", url).Reply(200).SetHeaders(mockHeader)
+			loc, size, err := d.Capture(c, url)
+			ctx.Convey("Then err should be nil.loc,size should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(size, convey.ShouldNotBeNil)
+				ctx.So(loc, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestBfscheckURL(t *testing.T) {
+	convey.Convey("checkURL", t, func(ctx convey.C) {
+		var (
+			url = defaultImg
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			err := checkURL(url)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestBfsisPublicIP(t *testing.T) {
+	convey.Convey("isPublicIP", t, func(ctx convey.C) {
+		var IP = net.ParseIP("127.0.0.1")
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := isPublicIP(IP)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}