Create & Init Project...

app/job/main/growup/dao/dataplatform/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "dataplatform_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "dataplatform.go",
+    ],
+    importpath = "go-common/app/job/main/growup/dao/dataplatform",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/growup/conf:go_default_library",
+        "//app/job/main/growup/model:go_default_library",
+        "//app/job/main/growup/model/income:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/growup/dao/dataplatform/dao.go

@@ -0,0 +1,34 @@
+package dataplatform
+
+import (
+	"go-common/app/job/main/growup/conf"
+	"go-common/library/database/sql"
+	xhttp "go-common/library/net/http/blademaster"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c        *conf.Config
+	db       *sql.DB
+	url      string
+	spyURL   string
+	bgmURL   string
+	basicURL string
+	client   *xhttp.Client
+}
+
+// New is new redis dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.Mysql.Growup),
+		// client
+		client: xhttp.NewClient(c.DPClient),
+		url:    c.Host.DataPlatform + "/avenger/api/38/query",
+		spyURL: c.Host.DataPlatform + "/avenger/api/51/query",
+		//		bgmURL: c.Host.DataPlatform + "/avenger/api/81/query",
+		bgmURL:   c.Host.DataPlatform + "/avenger/api/95/query",
+		basicURL: c.Host.DataPlatform + "/avenger/api/200/query",
+	}
+	return
+}

app/job/main/growup/dao/dataplatform/dao_test.go

@@ -0,0 +1,34 @@
+package dataplatform
+
+import (
+	"flag"
+	"go-common/app/job/main/growup/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "mobile.studio.growup-job")
+		flag.Set("conf_token", "8781e02680f40996bc01eb1248ac2ac9")
+		flag.Set("tree_id", "14716")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/growup-job.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/growup/dao/dataplatform/dataplatform.go

@@ -0,0 +1,245 @@
+package dataplatform
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	xhttp "net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/growup/model"
+	income "go-common/app/job/main/growup/model/income"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+	_userAgent = "User-Agent"
+)
+
+func (d *Dao) setParams() url.Values {
+	params := url.Values{}
+	params.Set("appKey", d.c.DPClient.Key)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	return params
+}
+
+// GetArchiveByMID get archive id by mid
+func (d *Dao) GetArchiveByMID(c context.Context, query string) (ids []int64, err error) {
+	ids = make([]int64, 0)
+	params := d.setParams()
+	params.Set("query", query)
+	var res struct {
+		Code   int                `json:"code"`
+		Result []*model.ArchiveID `json:"result"`
+	}
+	if err = d.NewRequest(c, d.url, "", params, &res); err != nil {
+		log.Error("dataplatform.send NewRequest url(%s) error(%v)", d.url+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 200 {
+		log.Error("dateplatform.send NewRequest error code:%d ; url(%s) ", res.Code, d.url+"?"+params.Encode())
+		return
+	}
+	if res.Code == 200 && len(res.Result) > 0 {
+		for _, archive := range res.Result {
+			ids = append(ids, archive.ID)
+		}
+	}
+	return
+}
+
+// Send ...
+func (d *Dao) Send(c context.Context, query string) (infos []*model.ArchiveInfo, err error) {
+	log.Info("dateplatform Send start")
+	params := url.Values{}
+	params.Set("appKey", d.c.DPClient.Key)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("query", query)
+	var res struct {
+		Code   int                  `json:"code"`
+		Result []*model.ArchiveInfo `json:"result"`
+	}
+	if err = d.NewRequest(c, d.url, "", params, &res); err != nil {
+		log.Error("dataplatform.send NewRequest url(%s) error(%v)", d.url+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 200 {
+		log.Error("dateplatform.send NewRequest error code:%d ; url(%s) ", res.Code, d.url+"?"+params.Encode())
+		return
+	}
+	if res.Code == 200 && len(res.Result) > 0 {
+		infos = res.Result
+	}
+	return
+}
+
+// SendSpyRequest send.
+func (d *Dao) SendSpyRequest(c context.Context, query string) (infos []*model.Spy, err error) {
+	log.Info("dateplatform Send start")
+	params := url.Values{}
+	params.Set("appKey", d.c.DPClient.Key)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("query", query)
+	var res struct {
+		Code   int          `json:"code"`
+		Result []*model.Spy `json:"result"`
+	}
+	if err = d.NewRequest(c, d.spyURL, "", params, &res); err != nil {
+		log.Error("dataplatform.SendSpyRequest NewRequest url(%s) error(%v)", d.spyURL+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 200 {
+		log.Error("dateplatform.SendSpyRequest NewRequest error code:%d ; url(%s) ", res.Code, d.spyURL+"?"+params.Encode())
+		return
+	}
+	if res.Code == 200 && len(res.Result) > 0 {
+		infos = res.Result
+	}
+	return
+}
+
+// SendBGMRequest get bgm infos
+func (d *Dao) SendBGMRequest(c context.Context, query string) (infos []*income.BGM, err error) {
+	params := url.Values{}
+	params.Set("appKey", d.c.DPClient.Key)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("query", query)
+	var res struct {
+		Code   int           `json:"code"`
+		Result []*income.BGM `json:"result"`
+	}
+	if err = d.NewRequest(c, d.bgmURL, "", params, &res); err != nil {
+		log.Error("dataplatform.SendBGMRequest NewRequest url(%s) error(%v)", d.bgmURL+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 200 {
+		log.Error("dateplatform.SendBGMRequest NewRequest error code:%d ; url(%s) ", res.Code, d.bgmURL+"?"+params.Encode())
+		return
+	}
+	if res.Code == 200 && len(res.Result) > 0 {
+		infos = res.Result
+	}
+	return
+}
+
+// SendBasicDataRequest get basic data status
+func (d *Dao) SendBasicDataRequest(c context.Context, query string) (ok bool, err error) {
+	params := url.Values{}
+	params.Set("appKey", d.c.DPClient.Key)
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("query", query)
+	var res struct {
+		Code   int `json:"code"`
+		Result []*struct {
+			Stat int `json:"stat"`
+		} `json:"result"`
+	}
+	url := d.basicURL
+	if err = d.NewRequest(c, url, "", params, &res); err != nil {
+		log.Error("SendBasicDataRequest NewRequest url(%s) error(%v)", url+"?"+params.Encode(), err)
+		return
+	}
+	if res.Code != 200 {
+		log.Error("SendBasicDataRequest NewRequest error code:%d ; url(%s) ", res.Code, url+"?"+params.Encode())
+		return
+	}
+	if res.Code == 200 && len(res.Result) > 0 {
+		info := res.Result[0]
+		if info.Stat == 1 {
+			ok = true
+		}
+	}
+	return
+}
+
+// NewRequest new http request with method, url, ip, values and headers.
+func (d *Dao) NewRequest(c context.Context, url, realIP string, params url.Values, res interface{}) (err error) {
+	enc, err := d.sign(params)
+	if err != nil {
+		log.Error("url:%s,params:%v", url, params)
+		return
+	}
+	if enc != "" {
+		url = url + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, url, nil)
+	if err != nil {
+		log.Error("method:%s,url:%s", xhttp.MethodGet, url)
+		return
+	}
+	req.Header.Set(_userAgent, "haoguanwei@bilibili.com  "+env.AppID)
+	if err != nil {
+		return
+	}
+	return d.client.Do(c, req, res)
+}
+
+// sign calc appkey and appsecret sign.
+func (d *Dao) sign(params url.Values) (query string, err error) {
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(d.c.DPClient.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(d.c.DPClient.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}

app/job/main/growup/dao/dataplatform/dataplatform_test.go

@@ -0,0 +1,130 @@
+package dataplatform
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDataplatformsetParams(t *testing.T) {
+	convey.Convey("setParams", t, func(ctx convey.C) {
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := d.setParams()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformGetArchiveByMID(t *testing.T) {
+	convey.Convey("GetArchiveByMID", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			query = ""
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			ids, err := d.GetArchiveByMID(c, query)
+			ctx.Convey("Then err should be nil.ids should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(ids, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformSend(t *testing.T) {
+	convey.Convey("Send", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			query = ""
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			infos, err := d.Send(c, query)
+			ctx.Convey("Then err should be nil.infos should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(infos, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformSendSpyRequest(t *testing.T) {
+	convey.Convey("SendSpyRequest", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			query = ""
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			infos, err := d.SendSpyRequest(c, query)
+			ctx.Convey("Then err should be nil.infos should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(infos, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformSendBGMRequest(t *testing.T) {
+	convey.Convey("SendBGMRequest", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			query = ""
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			infos, err := d.SendBGMRequest(c, query)
+			ctx.Convey("Then err should be nil.infos should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(infos, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformNewRequest(t *testing.T) {
+	convey.Convey("NewRequest", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			url    = ""
+			realIP = ""
+			res    = interface{}(0)
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			err := d.NewRequest(c, url, realIP, d.setParams(), res)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformsign(t *testing.T) {
+	convey.Convey("sign", t, func(ctx convey.C) {
+		var (
+			params url.Values
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			query, err := d.sign(params)
+			ctx.Convey("Then err should be nil.query should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(query, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDataplatformencode(t *testing.T) {
+	convey.Convey("encode", t, func(ctx convey.C) {
+		var (
+			v url.Values
+		)
+		ctx.Convey("When everything gose positive", func(ctx convey.C) {
+			p1 := d.encode(v)
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}