Create & Init Project...

app/job/main/tv/dao/report/BUILD

@@ -0,0 +1,62 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mc_test.go",
+        "report_test.go",
+        "style_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mc.go",
+        "report.go",
+        "style.go",
+    ],
+    importpath = "go-common/app/job/main/tv/dao/report",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/tv/conf:go_default_library",
+        "//app/job/main/tv/model/pgc:go_default_library",
+        "//app/job/main/tv/model/report:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/tv/dao/report/dao.go

@@ -0,0 +1,27 @@
+package report
+
+import (
+	"go-common/app/job/main/tv/conf"
+	"go-common/library/cache/memcache"
+	"go-common/library/database/sql"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Dao dao .
+type Dao struct {
+	conf  *conf.Config
+	httpR *bm.Client
+	mc    *memcache.Pool
+	DB    *sql.DB
+}
+
+// New create a instance of Dao and return .
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		conf:  c,
+		httpR: bm.NewClient(c.DpClient),
+		mc:    memcache.NewPool(c.Memcache.Config),
+		DB:    sql.NewMySQL(c.Mysql),
+	}
+	return
+}

app/job/main/tv/dao/report/dao_test.go

@@ -0,0 +1,35 @@
+package report
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/job/main/tv/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.web-svr.tv-job")
+		flag.Set("conf_token", "ab3e9801a77c076b997de0ac5cb21775")
+		flag.Set("tree_id", "15260")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../../cmd/tv-job-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/job/main/tv/dao/report/mc.go

@@ -0,0 +1,119 @@
+package report
+
+import (
+	"context"
+	"time"
+
+	mdlpgc "go-common/app/job/main/tv/model/pgc"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+)
+
+const (
+	_report = "_report"
+	_style  = "style_label"
+	_label  = "label_data"
+)
+
+// SetReportCache set report cache .
+func (d *Dao) SetReportCache(c context.Context, val map[string]interface{}) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = _report
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     val,
+		Flags:      memcache.FlagJSON,
+		Expiration: int32(time.Duration(d.conf.Report.Expire) / time.Second),
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+// GetReportCache get report all data .
+func (d *Dao) GetReportCache(c context.Context) (res map[string]interface{}, err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = _report
+		rp   *memcache.Item
+	)
+	res = make(map[string]interface{})
+	defer conn.Close()
+	if rp, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if err = conn.Scan(rp, &res); err != nil {
+		log.Error("conn.Scan error(%v)", err)
+	}
+	return
+}
+
+// SetStyleCache style show .
+func (d *Dao) SetStyleCache(c context.Context, val map[int][]*mdlpgc.ParamStyle) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = _style
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     val,
+		Flags:      memcache.FlagJSON,
+		Expiration: 0,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+// SetLabelCache label  show .
+func (d *Dao) SetLabelCache(c context.Context, val map[int]map[string]int) (err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = _label
+	)
+	defer conn.Close()
+	item := &memcache.Item{
+		Key:        key,
+		Object:     val,
+		Flags:      memcache.FlagJSON,
+		Expiration: 0,
+	}
+	if err = conn.Set(item); err != nil {
+		log.Error("conn.Set(%v) error(%v)", item, err)
+	}
+	return
+}
+
+//
+// GetLabelCache get label all data .
+func (d *Dao) GetLabelCache(c context.Context) (res map[int]map[string]int, err error) {
+	var (
+		conn = d.mc.Get(c)
+		key  = _label
+		rp   *memcache.Item
+	)
+	defer conn.Close()
+	if rp, err = conn.Get(key); err != nil {
+		if err == memcache.ErrNotFound {
+			err = nil
+		} else {
+			log.Error("mc.Get(%s) error(%v)", key, err)
+		}
+		return
+	}
+	if err = conn.Scan(rp, &res); err != nil {
+		log.Error("conn.Scan error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/report/mc_test.go

@@ -0,0 +1,38 @@
+package report
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestReportSetReportCache(t *testing.T) {
+	convey.Convey("SetReportCache", t, func(ctx convey.C) {
+		var (
+			c   = context.Background()
+			val = make(map[string]interface{})
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetReportCache(c, val)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestReportGetReportCache(t *testing.T) {
+	convey.Convey("GetReportCache", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.GetReportCache(c)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}

app/job/main/tv/dao/report/report.go

@@ -0,0 +1,227 @@
+package report
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	xhttp "net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	mdlrep "go-common/app/job/main/tv/model/report"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_httpOK     = "0"
+	_pid        = "73" // platform: 73 is tv
+	_plat       = 5    // platform: 5  is tv
+	_tv         = "android_tv_yst"
+	_actDur     = `select request_uri,time_iso,ip,version,buvid,fts,proid,chid,pid,brand,deviceid,model,osver,ctime,mid,ver,net,oid,eid,start_time,end_time,duration,openudid,idfa,mac,is_coldstart,session_id,buvid_ext from ods.app_active_duration where unix_timestamp(ctime, 'yyyyMMddHHmmss')>=%d and unix_timestamp(ctime, 'yyyyMMddHHmmss')<%d and log_date=%s and pid=%s`
+	_playDur    = `select stime,build,buvid,mobi_app,platform,session,mid,aid,cid,sid,epid,type,sub_type,quality,total_time,paused_time,played_time,video_duration,play_type,network_type,last_play_progress_time,max_play_progress_time,device,epid_status,play_status,user_status,actual_played_time,auto_play,detail_play_time,list_play_time from ods.app_play_duration where stime>=%d and stime<%d and log_date=%s and mobi_app=%s`
+	_visitEvent = `select request_uri,time_iso,ip,version,buvid,fts,proid,chid,pid,brand,deviceid,model,osver,mid,ctime,ver,net,oid,page_name,page_arg,ua,h5_chid,unix_timestamp(ctime, 'yyyyMMddHHmmss') from ods.app_visit_event where unix_timestamp(ctime, 'yyyyMMddHHmmss')>=%d and unix_timestamp(ctime, 'yyyyMMddHHmmss')<%d and log_date=%s and pid=%s`
+	_arcClick   = `select r_type,avid,cid,part,mid,stime,did,ip,ua,buvid,cookie_sid,refer,type,sub_type,sid,epid,platform,device from ods.ods_archive_click where stime>=%d and stime<%d and log_date=%s and plat=%d`
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+	_userAgent = "User-Agent"
+)
+
+// Report .
+func (d *Dao) Report(ctx context.Context, table string) (info string, err error) {
+	var (
+		v          = url.Values{}
+		ip         = metadata.String(ctx, metadata.RemoteIP)
+		logdata    = d.queryfmt()
+		start, end = d.dealDate()
+		query      string
+		res        struct {
+			Code      int    `json:"code"`
+			Msg       string `json:"msg"`
+			StatusURL string `json:"jobStatusUrl"`
+		}
+	)
+	logdata = "'" + logdata + "'"
+	switch table {
+	case mdlrep.ArchiveClick:
+		query = fmt.Sprintf(_arcClick, start, end, logdata, _plat)
+	case mdlrep.ActiveDuration:
+		query = fmt.Sprintf(_actDur, start, end, logdata, `"`+_pid+`"`)
+	case mdlrep.PlayDuration:
+		query = fmt.Sprintf(_playDur, start, end, logdata, "'"+_tv+"'")
+	case mdlrep.VisitEvent:
+		query = fmt.Sprintf(_visitEvent, start, end, logdata, `"`+_pid+`"`)
+	default:
+		err = errors.New("table is nill")
+		return
+	}
+	v.Set("appKey", d.conf.DpClient.Key)
+	v.Set("signMethod", "md5")
+	v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	v.Set("version", "1.0")
+	v.Set("query", query)
+	if err = d.newRequest(ctx, d.conf.Report.ReportURI, ip, v, &res); err != nil {
+		log.Error("newRequest url(%v), err(%v)", d.conf.Report.ReportURI+"?"+v.Encode(), err)
+		return
+	}
+	if res.Code == 200 && res.StatusURL != "" {
+		info = res.StatusURL
+	}
+	return
+}
+
+// CheckJob .
+func (d *Dao) CheckJob(ctx context.Context, urls string) (res *mdlrep.DpCheckJobResult, err error) {
+	var (
+		v  = url.Values{}
+		ip = metadata.String(ctx, metadata.RemoteIP)
+	)
+	res = &mdlrep.DpCheckJobResult{}
+	v.Set("appKey", d.conf.DpClient.Key)
+	v.Set("signMethod", "md5")
+	v.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	v.Set("version", "1.0")
+	if err = d.newRequest(ctx, urls, ip, v, &res); err != nil {
+		log.Error("d.newRequest error(%v)", err)
+		return
+	}
+	if res.Code != xhttp.StatusOK {
+		log.Error("d.CheckJob newRequest error code:%d ; url(%s) ", res.Code, urls+"?"+v.Encode())
+		err = fmt.Errorf("code(%d) msg(%s) statusID(%d) statusID(%s)", res.Code, res.Msg, res.StatusID, res.StatusMsg)
+	}
+	return
+}
+
+// PostRequest .
+func (d *Dao) PostRequest(ctx context.Context, body string) (err error) {
+	var (
+		req *xhttp.Request
+		res struct {
+			Code    string `json:"code"`
+			Message string `json:"message"`
+		}
+	)
+	if req, err = xhttp.NewRequest(xhttp.MethodPost, d.conf.Report.UpDataURI, strings.NewReader(body)); err != nil {
+		log.Error("xhttp.NewRequest url(%s) error (%v)", d.conf.Report.UpDataURI, err)
+		return
+	}
+	req.Header.Add("Content-Type", "text/plain; charset=utf-8")
+	if err = d.httpR.Do(ctx, req, &res); err != nil {
+		log.Error("d.httpReq.Do error(%v) url(%s)", err, d.conf.Report.UpDataURI)
+		return
+	}
+	if res.Code != _httpOK {
+		log.Error("PostRequest error code:%s ; url(%s) ", res.Code, d.conf.Report.UpDataURI)
+		err = fmt.Errorf("code(%s)", res.Code)
+	}
+	return
+}
+
+// NewRequest new http request with method, url, ip, values and headers .
+func (d *Dao) newRequest(c context.Context, url, realIP string, params url.Values, res interface{}) (err error) {
+	enc, err := d.sign(params)
+	if err != nil {
+		log.Error("url:%s,params:%v", url, params)
+		return
+	}
+	if enc != "" {
+		url = url + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, url, nil)
+	if err != nil {
+		log.Error("xhttp.NewRequest method:%s,url:%s", xhttp.MethodGet, url)
+		return
+	}
+	req.Header.Set(_userAgent, "haoguanwei@bilibili.com  "+env.AppID)
+	if err != nil {
+		return
+	}
+	return d.httpR.Do(c, req, res)
+}
+
+// Sign calc appkey and appsecret sign .
+func (d *Dao) sign(params url.Values) (query string, err error) {
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(d.conf.DpClient.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(d.conf.DpClient.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form .
+// ("bar=baz&foo=quux") sorted by key .
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+// delDate create start and end time .
+func (d *Dao) dealDate() (start, end int64) {
+	var (
+		timeDelay = "-" + d.conf.Report.TimeDelay
+		timeSpan  = "-" + time.Duration(d.conf.Report.SeTimeSpan).String()
+	)
+	endTime := time.Now()
+	et, _ := time.ParseDuration(timeDelay)
+	endTmp := endTime.Add(et)
+	end = endTime.Add(et).Unix()
+	st, _ := time.ParseDuration(timeSpan)
+	start = endTmp.Add(st).Unix()
+	return
+}
+
+func (d *Dao) queryfmt() (logdata string) {
+	var (
+		dtime    = time.Now()
+		timeData = "-" + d.conf.Report.TimeDelay
+	)
+	et, _ := time.ParseDuration(timeData)
+	logdata = dtime.Add(et).Format("20060102")
+	return
+}

app/job/main/tv/dao/report/report_test.go

@@ -0,0 +1,32 @@
+package report
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestReportReport(t *testing.T) {
+	convey.Convey("Report", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			_, err := d.Report(c, "archive_click")
+			println(err)
+		})
+	})
+}
+
+func TestReportCheckJob(t *testing.T) {
+	convey.Convey("CheckJob", t, func(ctx convey.C) {
+		var (
+			c    = context.Background()
+			urls = ""
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			d.CheckJob(c, urls)
+		})
+	})
+}

app/job/main/tv/dao/report/style.go

@@ -0,0 +1,69 @@
+package report
+
+import (
+	"context"
+
+	mdlpgc "go-common/app/job/main/tv/model/pgc"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_styleSQL = "SELECT id,style,category FROM tv_ep_season WHERE is_deleted=0 AND `check`=1 AND valid=1"
+	_labelSQL = `SELECT name,value,category FROM tv_label WHERE deleted=0 AND param="style_id" AND valid=1`
+)
+
+// FindStyle style all .
+func (d *Dao) FindStyle(ctx context.Context) (res []*mdlpgc.StyleRes, err error) {
+	var (
+		rows *sql.Rows
+	)
+	if rows, err = d.DB.Query(ctx, _styleSQL); err != nil {
+		log.Error("d.DB.Query sql(%s) error(%v)", _styleSQL, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &mdlpgc.StyleRes{}
+		if err = rows.Scan(&r.ID, &r.Style, &r.Category); err != nil {
+			log.Error("d.DB.QueryRow error(%v)", err)
+			return
+		}
+		res = append(res, r)
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}
+
+// FindLabelID label all .
+func (d *Dao) FindLabelID(ctx context.Context) (res map[int]map[string]int, err error) {
+	var (
+		rows *sql.Rows
+		m    map[string]int
+	)
+	res = make(map[int]map[string]int)
+	if rows, err = d.DB.Query(ctx, _labelSQL); err != nil {
+		log.Error("d.DB.Query error(%v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		r := &mdlpgc.LabelRes{}
+		if err = rows.Scan(&r.Name, &r.Value, &r.Category); err != nil {
+			log.Error("d.DB.Query Scan error(%v)", err)
+			return
+		}
+		if _, ok := res[r.Category]; ok {
+			m[r.Name] = r.Value
+		} else {
+			m = make(map[string]int)
+		}
+		res[r.Category] = m
+	}
+	if err = rows.Err(); err != nil {
+		log.Error("rows.Err() error(%v)", err)
+	}
+	return
+}

app/job/main/tv/dao/report/style_test.go

@@ -0,0 +1,38 @@
+package report
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestReportFindStyle(t *testing.T) {
+	convey.Convey("FindStyle", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.FindStyle(c)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestReportFindLabelID(t *testing.T) {
+	convey.Convey("FindLabelID", t, func(ctx convey.C) {
+		var (
+			c = context.Background()
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			res, err := d.FindLabelID(c)
+			ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(res, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}