Create & Init Project...

app/interface/main/report-click/http/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "click.go",
+        "err_report.go",
+        "http.go",
+        "player.go",
+    ],
+    importpath = "go-common/app/interface/main/report-click/http",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/report-click/conf:go_default_library",
+        "//app/interface/main/report-click/model:go_default_library",
+        "//app/interface/main/report-click/service:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/auth:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/metadata:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/report-click/http/click.go

@@ -0,0 +1,271 @@
+package http
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/interface/main/report-click/conf"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_platWeb       = "0"
+	_platH5        = "1"
+	_platOuter     = "2"
+	_platIos       = "3"
+	_platAndroid   = "4"
+	_platAndroidTV = "5"
+)
+
+var _expireCookie = time.Date(2022, time.November, 10, 23, 0, 0, 0, time.UTC)
+
+// webClick write the archive data.
+func webClick(c *bm.Context) {
+	err := flashPlay(c, _platWeb, conf.Conf.Click.WebSecret)
+	c.JSON(nil, err)
+}
+
+// outerClick write the archive data.
+func outerClick(c *bm.Context) {
+	err := flashPlay(c, _platOuter, conf.Conf.Click.OutSecret)
+	c.JSON(nil, err)
+}
+
+// iosClick write the archive data.
+func iosClick(c *bm.Context) {
+	err := mobilePlay(c, conf.Conf.Click.AesKey, conf.Conf.Click.AesIv, conf.Conf.Click.AesSalt, _platIos)
+	c.JSON(nil, err)
+}
+
+// androidClick write the archive data.
+func androidClick(c *bm.Context) {
+	err := mobilePlay(c, conf.Conf.Click.AesKey, conf.Conf.Click.AesIv, conf.Conf.Click.AesSalt, _platAndroid)
+	c.JSON(nil, err)
+}
+
+// android2Click write the archive data.
+func android2Click(c *bm.Context) {
+	err := mobilePlay(c, conf.Conf.Click.AesKey2, conf.Conf.Click.AesIv2, conf.Conf.Click.AesSalt2, _platAndroid)
+	c.JSON(nil, err)
+}
+
+// androidTV == android2Click write the archive data.
+func androidTV(c *bm.Context) {
+	err := mobilePlay(c, conf.Conf.Click.AesKey2, conf.Conf.Click.AesIv2, conf.Conf.Click.AesSalt2, _platAndroidTV)
+	c.JSON(nil, err)
+}
+
+// outerClickH5  h5 outer click same to flash plat .
+func outerClickH5(c *bm.Context) {
+	c.JSON(nil, h5Play(c, _platOuter))
+}
+
+// h5Click write the archive data.
+func h5Click(c *bm.Context) {
+	c.JSON(nil, h5Play(c, _platH5))
+}
+
+// webH5Click write the archive data.
+func webH5Click(c *bm.Context) {
+	c.JSON(nil, h5Play(c, _platWeb))
+}
+
+// flashPlay.
+func flashPlay(c *bm.Context, plat, secret string) (err error) {
+	var (
+		buvid  string
+		mid    int64
+		ck     *http.Cookie
+		params = c.Request.Form
+		unix   = time.Now()
+	)
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	if ck, err = c.Request.Cookie("buvid3"); err == nil {
+		buvid = ck.Value
+	}
+	userAgent := c.Request.Header.Get("User-Agent")
+	refer := c.Request.Header.Get("Referer")
+	sign := params.Get("sign")
+	if err = clickSvr.FlashSigned(params, secret, unix); err != nil {
+		log.Error("clickSvr.FlashSigned() error(%v)", err)
+		if err == ecode.ClickQuerySignErr {
+			log.Warn("click sign error(%s,%s,%s,%s)", sign, refer, userAgent, c.Request.Header.Get("Origin"))
+		}
+		return
+	}
+	midStr := params.Get("mid")
+	if mid != 0 && midStr != strconv.FormatInt(mid, 10) {
+		log.Warn("flashPlay stat mid(%d) not equal stat mid(%s)", mid, midStr)
+		return
+	}
+	midStr = strconv.FormatInt(mid, 10)
+	aid := params.Get("aid")
+	var cookieSid string
+	if ck, err := c.Request.Cookie("sid"); err == nil {
+		cookieSid = ck.Value
+	}
+	typeID := params.Get("type")
+	subType := params.Get("sub_type")
+	sid := params.Get("sid")
+	epid := params.Get("epid")
+	// service.
+	ip := metadata.String(c, metadata.RemoteIP)
+	clickSvr.Play(c, plat, aid, params.Get("cid"), params.Get("part"),
+		midStr, params.Get("lv"), params.Get("ftime"), params.Get("stime"),
+		params.Get("did"), ip, userAgent, buvid, cookieSid, refer, typeID, subType, sid, epid, "", "", "", "", "", "")
+	return
+}
+
+// mobilePlay.
+func mobilePlay(c *bm.Context, aesKey, aesIv, aesSalt, plat string) (err error) {
+	// check params.
+	req := c.Request
+	unix := time.Now()
+	bs, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		log.Error("ioutil.ReadAll error(%v)", err)
+		err = ecode.ServerErr
+		return
+	}
+	req.Body.Close()
+	bs, err = clickSvr.Decrypt(bs, aesKey, aesIv)
+	if err != nil {
+		log.Error("clickSvr.Decrypt(%s) error(%d)", bs, err)
+		return
+	}
+	p, err := clickSvr.Verify(bs, aesSalt, unix)
+	if err != nil {
+		log.Error("clickSvr.Verify(%s) error(%d)", bs, err)
+		return
+	}
+	req.Form = p // for log
+	typeID := p.Get("type")
+	subType := p.Get("sub_type")
+	sid := p.Get("sid")
+	epid := p.Get("epid")
+	playMode := p.Get("play_mode")
+	platform := p.Get("platform")
+	device := p.Get("device")
+	mobiAapp := p.Get("mobi_app")
+	autoPlay := p.Get("auto_play")
+	ap, _ := strconv.ParseInt(autoPlay, 10, 64)
+	// service.
+	aidStr := p.Get("aid")
+	var (
+		accessKey string
+		midStr    = p.Get("mid")
+		noAccess  bool
+	)
+	paasMid, _ := strconv.ParseInt(midStr, 10, 64)
+	accessKey = p.Get("access_key")
+	if paasMid > 0 && accessKey == "" {
+		noAccess = true
+	}
+	if accessKey != "" {
+		c.Request.Form.Set("access_key", accessKey)
+		authSvc.User(c)
+		mid, ok := c.Get("mid")
+		if !ok {
+			log.Warn("idfSvc.Access() access_key", accessKey)
+			if paasMid > 0 {
+				noAccess = true
+			}
+		}
+		if mid != nil {
+			midStr = strconv.FormatInt(mid.(int64), 10)
+		}
+	}
+	refer := c.Request.Header.Get("Referer")
+	userAgent := c.Request.Header.Get("User-Agent")
+	if noAccess {
+		userAgent = userAgent + " (no_accesskey)"
+	}
+	if ap == 1 || ap == 2 { // abandon the logic that transforms the plat to 6/7/8/9, keep the plat and modify the UA
+		userAgent += " (inline_play_begin)"
+	}
+	buvid := req.Header.Get("buvid")
+	var cookieSid string
+	if ck, err := c.Request.Cookie("sid"); err == nil {
+		cookieSid = ck.Value
+	}
+	ip := metadata.String(c, metadata.RemoteIP)
+	clickSvr.Play(c, plat, aidStr, p.Get("cid"), p.Get("part"), midStr, p.Get("lv"),
+		p.Get("ftime"), p.Get("stime"), p.Get("did"), ip, userAgent, buvid,
+		cookieSid, refer, typeID, subType, sid, epid, playMode, platform, device, mobiAapp, autoPlay, "")
+	return
+}
+
+func h5Play(c *bm.Context, plat string) (err error) {
+	var (
+		buvid  string
+		mid    int64
+		ck     *http.Cookie
+		params = c.Request.Form
+		unix   = time.Now()
+	)
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	if ck, err = c.Request.Cookie("buvid3"); err == nil {
+		buvid = ck.Value
+	}
+	// check params.
+	st := params.Get("stime")
+	stime, err := strconv.ParseInt(st, 10, 64)
+	if err != nil {
+		err = ecode.ClickQueryFormatErr
+		return
+	}
+	if unix.Unix()-stime > 60 {
+		err = ecode.ClickServerTimeout
+		return
+	}
+	typeID := params.Get("type")
+	subType := params.Get("sub_type")
+	sid := params.Get("sid")
+	epid := params.Get("epid")
+	var ft string
+	// check cookie did
+	var (
+		did string
+		ip  = metadata.String(c, metadata.RemoteIP)
+	)
+	cookie, err := c.Request.Cookie("rpdid")
+	if err != nil || cookie.Value == "" {
+		did = clickSvr.GenDid(ip, unix)
+		http.SetCookie(c.Writer, &http.Cookie{Name: "rpdid", Value: did, Path: "/", Domain: ".bilibili.com", Expires: _expireCookie})
+		err = nil
+	} else {
+		did = cookie.Value
+	}
+	_, ft = clickSvr.CheckDid(did)
+	if ft == "" {
+		log.Error("ft null ft:%s,did:%s", ft, did)
+		return
+	}
+	midStr := params.Get("mid")
+	if mid != 0 && midStr != strconv.FormatInt(mid, 10) {
+		log.Warn("h5 stat mid(%d) not equal stat mid(%s)", mid, midStr)
+		return
+	}
+	midStr = strconv.FormatInt(mid, 10)
+	aid := params.Get("aid")
+	userAgent := c.Request.Header.Get("User-Agent")
+	var cookieSid string
+	if ck, err := c.Request.Cookie("sid"); err == nil {
+		cookieSid = ck.Value
+	}
+	refer := c.Request.Header.Get("Referer")
+	// service.
+	clickSvr.Play(c, plat, aid, params.Get("cid"), params.Get("part"),
+		midStr, params.Get("lv"), ft, params.Get("stime"), did,
+		ip, userAgent, buvid, cookieSid, refer, typeID, subType, sid, epid, "", "", "", "", "", "")
+	return
+}

app/interface/main/report-click/http/err_report.go

@@ -0,0 +1,15 @@
+package http
+
+import (
+	"go-common/app/interface/main/report-click/model"
+	bm "go-common/library/net/http/blademaster"
+)
+
+func errReport(c *bm.Context) {
+	v := new(model.ErrReport)
+	if err := c.Bind(v); err != nil {
+		return
+	}
+	clickSvr.ErrReport(c, v)
+	c.JSON(nil, nil)
+}

app/interface/main/report-click/http/http.go

@@ -0,0 +1,90 @@
+package http
+
+import (
+	"time"
+
+	"go-common/app/interface/main/report-click/conf"
+	"go-common/app/interface/main/report-click/service"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/auth"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	clickSvr        *service.Service
+	authSvc         *auth.Auth
+	verifySvc       *verify.Verify
+	infocRealTime   *infoc.Infoc
+	infocStatistics *infoc.Infoc
+	fromMap         = make(map[int64]bool)
+	fromInlineMap   = make(map[int64]bool)
+	inlineDuration  int64
+)
+
+// New http init.
+func New(c *conf.Config) (engine *bm.Engine) {
+	clickSvr = service.New(c)
+	authSvc = auth.New(c.Auth)
+	verifySvc = verify.New(c.Verify)
+	infocRealTime = infoc.New(c.Infoc2.RealTime)
+	infocStatistics = infoc.New(c.Infoc2.Statistics)
+	for _, v := range c.Click.From {
+		fromMap[v] = true
+	}
+	for _, v := range c.Click.FromInline { // init inline play "from"
+		fromInlineMap[v] = true
+	}
+	inlineDuration = c.Click.InlineDuration // inline play duration line
+	engine = bm.DefaultServer(c.BM)
+	engine.Use(bm.Recovery(), bm.Logger())
+	outerRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start() error(%v)", err)
+		panic(err)
+	}
+	return
+}
+
+func outerRouter(e *bm.Engine) {
+	e.GET("/monitor/ping", ping)
+	e.POST("/x/report/click/web", authSvc.GuestWeb, webClick)
+	e.POST("/x/report/click/outer", authSvc.GuestWeb, outerClick)
+	e.POST("/x/stat/web", authSvc.GuestWeb, webClick)
+	e.POST("/x/stat/outer", authSvc.GuestWeb, outerClick)
+	click := e.Group("/x/report/click")
+	{
+		click.GET("/now", serverNow)
+		click.POST("/h5", authSvc.Guest, h5Click)
+		click.POST("/h5/outer", authSvc.Guest, outerClickH5) // nocsrf
+		click.POST("/ios", authSvc.Guest, iosClick)
+		click.POST("/android", authSvc.Guest, androidClick)
+		click.POST("/android2", authSvc.Guest, android2Click)
+		click.POST("/web/h5", authSvc.Guest, webH5Click)
+		click.POST("/android/tv", authSvc.Guest, androidTV)
+	}
+	report := e.Group("/x/report/")
+	{
+		report.POST("/player", verifySvc.Verify, reportPlayer)       // old 30s heart
+		report.POST("/heartbeat", verifySvc.Verify, reportHeartbeat) // new app 30s heart
+		report.POST("/heartbeat/mobile", verifySvc.Verify, heartbeatMobile)
+		report.POST("/web/heartbeat", authSvc.Guest, webHeartbeat) // web 30s heart
+
+	}
+	stat := e.Group("/x/stat")
+	{
+		stat.GET("/now", serverNow)
+		stat.POST("/err_report", errReport)
+		stat.POST("/h5", authSvc.Guest, h5Click)
+		stat.POST("/ios", authSvc.Guest, iosClick)
+		stat.POST("/android", authSvc.Guest, androidClick)
+		stat.POST("/android2", authSvc.Guest, android2Click)
+	}
+}
+
+func ping(c *bm.Context) {}
+func serverNow(c *bm.Context) {
+	data := map[string]int64{"now": time.Now().Unix()}
+	c.JSON(data, nil)
+}

app/interface/main/report-click/http/player.go

@@ -0,0 +1,195 @@
+package http
+
+import (
+	"strconv"
+	"time"
+
+	"go-common/app/interface/main/report-click/model"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/metadata"
+)
+
+// reportPlayer write the archive data.
+func reportPlayer(c *bm.Context) {
+	params := c.Request.Form
+	header := c.Request.Header
+	buvid := header.Get("Buvid")
+	displayid := header.Get("Display-ID")
+	ts := params.Get("ts")
+	aid := params.Get("aid")
+	cid := params.Get("cid")
+	playedTime := params.Get("played_time")
+	mid := params.Get("mid")
+	moAp := params.Get("mobi_app")
+	typeID := params.Get("type")
+	subType := params.Get("sub_type")
+	sid := params.Get("sid")
+	epid := params.Get("epid")
+	infocRealTime.Info(ts, buvid, displayid, mid, aid, cid, playedTime, strconv.FormatInt(time.Now().Unix(), 10), "1", moAp, "", typeID, subType, sid, epid, "")
+	c.JSON(nil, nil)
+}
+
+// reportHeartbeat write the archive data.
+func reportHeartbeat(c *bm.Context) {
+	params := c.Request.Form
+	header := c.Request.Header
+	buvid := header.Get("Buvid")
+	displayid := header.Get("Display-ID")
+	sts := params.Get("start_ts")
+	aid := params.Get("aid")
+	if aid == "" {
+		aid = params.Get("avid")
+	}
+	cid := params.Get("cid")
+	playedTime := params.Get("played_time")
+	mid := params.Get("mid")
+	moAp := params.Get("mobi_app")
+	typeID := params.Get("type")
+	subType := params.Get("sub_type")
+	sid := params.Get("sid")
+	epid := params.Get("epid")
+	playType := params.Get("play_type")
+	if playType == "" {
+		playType = params.Get("playtype")
+	}
+	infocRealTime.Info(sts, buvid, displayid, mid, aid, cid, playedTime, strconv.FormatInt(time.Now().Unix(), 10), "2", moAp, "", typeID, subType, sid, epid, playType)
+	c.JSON(nil, nil)
+}
+
+func heartbeatMobile(c *bm.Context) {
+	params := c.Request.Form
+	header := c.Request.Header
+	sts := params.Get("start_ts")
+	build := params.Get("build")
+	buvid := header.Get("Buvid")
+	mobileApp := params.Get("mobi_app")
+	device := params.Get("device")
+	platform := params.Get("platform")
+	session := params.Get("session")
+	mid := params.Get("mid")
+	aid := params.Get("aid")
+	cid := params.Get("cid")
+	sid := params.Get("sid")
+	epid := params.Get("epid")
+	tp := params.Get("type")
+	subType := params.Get("sub_type")
+	quality := params.Get("quality")
+	totalTime := params.Get("total_time")
+	pausedTime := params.Get("paused_time")
+	playedTime := params.Get("played_time")
+	videoDuration := params.Get("video_duration")
+	playType := params.Get("play_type")
+	networkType := params.Get("network_type")
+	playProgressTimeLast := params.Get("last_play_progress_time")
+	playProgressTimeMax := params.Get("max_play_progress_time")
+	playMode := params.Get("play_mode")
+	from := params.Get("from")
+	epidStatus := params.Get("epid_status")
+	playStatus := params.Get("play_status")
+	userStatus := params.Get("user_status")
+	actualPlayedTime := params.Get("actual_played_time")
+	autoPlay := params.Get("auto_play")
+	detailPlayTime := params.Get("detail_play_time")
+	listPlayTime := params.Get("list_play_time")
+	userAgent := c.Request.Header.Get("User-Agent")
+	ts, err := strconv.ParseInt(sts, 10, 64)
+	if err != nil || ts <= 0 {
+		ts = time.Now().Unix()
+		sts = strconv.FormatInt(ts, 10)
+	}
+	// NOTE /x/report//heartbeat/mobile auto_play = 2 ===> /x/report/click/android2 & ios
+	// (自动播放的上报>> 2:天马feed流inline) 播放时长转成播放点击
+	autoPlayInt, _ := strconv.ParseInt(autoPlay, 10, 64)
+	fromInt, _ := strconv.ParseInt(from, 10, 64)
+	videoDurInt, _ := strconv.ParseInt(videoDuration, 10, 64)
+	playedTimeInt, _ := strconv.ParseInt(playedTime, 10, 64)
+	var needCompens bool
+	if (autoPlayInt == 2 || autoPlayInt == 1) && fromMap[fromInt] {
+		userAgent = userAgent + " (inline_play_to_view)" // change from auto_play to inline_play_heartbeat, then to inline_play_to_view
+		needCompens = true
+	}
+	if (autoPlayInt == 2 || autoPlayInt == 1) && fromInlineMap[fromInt] && playedTimeInt >= inlineDuration && (videoDurInt >= playedTimeInt) {
+		userAgent += " (played_time_enough)" // new logic, if inline play more than 10s, count it also
+		needCompens = true
+	}
+	if needCompens {
+		var cookieSid, plat string
+		if ck, err := c.Request.Cookie("sid"); err == nil {
+			cookieSid = ck.Value
+		}
+		ip := metadata.String(c, metadata.RemoteIP)
+		switch platform {
+		case "android":
+			plat = _platAndroid
+		case "ios":
+			plat = _platIos
+		}
+		clickSvr.Play(c, plat, aid, cid, params.Get("part"), mid, params.Get("lv"),
+			"0", sts, buvid, ip, userAgent, buvid,
+			cookieSid, c.Request.Header.Get("Referer"), tp,
+			subType, sid, epid, playMode, platform, device, mobileApp, autoPlay, session)
+
+		log.Warn("plat:%s,aid:%s,cid:%s,part:%s,mid:%s,lv:%s,0:%s,sts:%s,buvid:%s,ip:%s,userAgent:%s,"+
+			"buvid:%s,cookieSid:%s,Referer:%s,tp:%s,subType:%s,sid:%s,epid:%s,playMode:%s,"+
+			"platform:%s,device:%s,mobileApp:%s,autoPlay:%s,session:%s",
+			plat, aid, cid, params.Get("part"), mid, params.Get("lv"),
+			"0", sts, buvid, ip, userAgent, buvid,
+			cookieSid, c.Request.Header.Get("Referer"), tp,
+			subType, sid, epid, playMode, platform, device, mobileApp, autoPlay, session)
+	}
+	buildInt, _ := strconv.ParseInt(build, 10, 64)
+	clickSvr.SuccReport(c, &model.SuccReport{ // record the success by different build
+		MobiApp: mobileApp,
+		Build:   buildInt,
+	})
+	infocStatistics.Info(sts, build, buvid, mobileApp, platform, session, mid, aid, cid, sid,
+		epid, tp, subType, quality, totalTime, pausedTime, playedTime, videoDuration,
+		playType, networkType, playProgressTimeLast, playProgressTimeMax, playMode, device, from, epidStatus, playStatus, userStatus, actualPlayedTime, autoPlay, detailPlayTime, listPlayTime)
+	data := make(map[string]interface{}, 1)
+	data["ts"] = ts
+	c.JSON(data, nil)
+}
+
+// webHeartbeat write the archive data.
+func webHeartbeat(c *bm.Context) {
+	var (
+		buvid, mid, term string
+		params           = c.Request.Form
+	)
+	if ck, err := c.Request.Cookie("buvid3"); err == nil {
+		buvid = ck.Value
+	}
+	sts := params.Get("start_ts")
+	aid := params.Get("aid")
+	cid := params.Get("cid")
+	pause := params.Get("pause")
+	playType := params.Get("play_type")
+	if playType == "" {
+		playType = params.Get("playtype")
+	}
+	playedTime := params.Get("played_time")
+	if midI, ok := c.Get("mid"); ok {
+		mid = strconv.FormatInt(midI.(int64), 10)
+	}
+	tp := params.Get("type")
+	subType := params.Get("sub_type")
+	sid := params.Get("sid")
+	if sid == "" {
+		sid = params.Get("seasonID")
+	}
+	epid := params.Get("epid")
+	dt := params.Get("dt")
+	if dt == "7" {
+		// count m.bilibili.com visits times.
+		term = "h5"
+	} else {
+		dt = "2"
+		term = "web"
+	}
+	realtime := params.Get("realtime")
+	ts := strconv.FormatInt(time.Now().Unix(), 10)
+	infocRealTime.Info(sts, buvid, "", mid, aid, cid, playedTime, ts, dt, term, pause, tp, subType, sid, epid, playType)
+	clickSvr.Report(c, playedTime, cid, tp, subType, realtime, aid, mid, sid, epid, dt, ts)
+	c.JSON(nil, nil)
+}