Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,67 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"api_test.go",
"dao_test.go",
"databus_test.go",
"memcache_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/interface/main/app-card/model:go_default_library",
"//app/interface/main/app-card/model/card/ai:go_default_library",
"//app/interface/main/app-feed/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"api.go",
"dao.go",
"databus.go",
"memcache.go",
"redis.go",
],
importpath = "go-common/app/interface/main/app-feed/dao/recommend",
tags = ["automanaged"],
deps = [
"//app/interface/main/app-card/model:go_default_library",
"//app/interface/main/app-card/model/card/ai:go_default_library",
"//app/interface/main/app-feed/conf:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/metadata:go_default_library",
"//library/queue/databus:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,150 @@
package recommend
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strconv"
"time"
"go-common/app/interface/main/app-card/model"
"go-common/app/interface/main/app-card/model/card/ai"
"go-common/library/ecode"
"go-common/library/log"
"go-common/library/net/metadata"
"github.com/pkg/errors"
)
const (
_rcmd = "/pegasus/feed/%d"
_hot = "/data/rank/reco-tmzb.json"
_group = "/group_changes/pegasus.json"
_top = "/feed/tag/top"
_followModeList = "/data/rank/others/followmode_whitelist.json"
)
// Recommend is.
func (d *Dao) Recommend(c context.Context, plat int8, buvid string, mid int64, build, loginEvent, parentMode, recsysMode int, zoneID int64, group int, interest, network string, style int, column model.ColumnStatus, flush int, autoplay string, now time.Time) (rs []*ai.Item, userFeature json.RawMessage, respCode int, newUser bool, err error) {
if mid == 0 && buvid == "" {
return
}
ip := metadata.String(c, metadata.RemoteIP)
uri := fmt.Sprintf(d.rcmd, group)
params := url.Values{}
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("buvid", buvid)
params.Set("plat", strconv.Itoa(int(plat)))
params.Set("build", strconv.Itoa(build))
params.Set("login_event", strconv.Itoa(loginEvent))
params.Set("zone_id", strconv.FormatInt(zoneID, 10))
params.Set("interest", interest)
params.Set("network", network)
if column > -1 {
params.Set("column", strconv.Itoa(int(column)))
}
params.Set("style", strconv.Itoa(style))
params.Set("flush", strconv.Itoa(flush))
params.Set("autoplay_card", autoplay)
params.Set("parent_mode", strconv.Itoa(parentMode))
params.Set("recsys_mode", strconv.Itoa(recsysMode))
var res struct {
Code int `json:"code"`
NewUser bool `json:"new_user"`
UserFeature json.RawMessage `json:"user_feature"`
Data []*ai.Item `json:"data"`
}
if err = d.client.Get(c, uri, ip, params, &res); err != nil {
respCode = ecode.ServerErr.Code()
return
}
code := ecode.Int(res.Code)
if !code.Equal(ecode.OK) {
respCode = res.Code
err = errors.Wrap(code, uri+"?"+params.Encode())
return
}
rs = res.Data
userFeature = res.UserFeature
newUser = res.NewUser
return
}
// Hots is.
func (d *Dao) Hots(c context.Context) (aids []int64, err error) {
ip := metadata.String(c, metadata.RemoteIP)
var res struct {
Code int `json:"code"`
List []struct {
Aid int64 `json:"aid"`
} `json:"list"`
}
if err = d.clientAsyn.Get(c, d.hot, ip, nil, &res); err != nil {
return
}
code := ecode.Int(res.Code)
if !code.Equal(ecode.OK) {
err = errors.Wrap(code, d.hot)
return
}
for _, list := range res.List {
if list.Aid != 0 {
aids = append(aids, list.Aid)
}
}
return
}
// TagTop is.
func (d *Dao) TagTop(c context.Context, mid, tid int64, rn int) (aids []int64, err error) {
params := url.Values{}
params.Set("src", "2")
params.Set("pn", "1")
params.Set("mid", strconv.FormatInt(mid, 10))
params.Set("tag", strconv.FormatInt(tid, 10))
params.Set("rn", strconv.Itoa(rn))
var res struct {
Code int `json:"code"`
Data []int64 `json:"data"`
}
if err = d.client.Get(c, d.top, "", params, &res); err != nil {
return
}
code := ecode.Int(res.Code)
if !code.Equal(ecode.OK) {
err = errors.Wrap(code, d.top+"?"+params.Encode())
return
}
aids = res.Data
return
}
// Group is.
func (d *Dao) Group(c context.Context) (gm map[int64]int, err error) {
err = d.clientAsyn.Get(c, d.group, "", nil, &gm)
return
}
// FollowModeList is.
func (d *Dao) FollowModeList(c context.Context) (list map[int64]struct{}, err error) {
var res struct {
Code int `json:"code"`
Data []int64 `json:"data"`
}
if err = d.clientAsyn.Get(c, d.followModeList, "", nil, &res); err != nil {
return
}
code := ecode.Int(res.Code)
if !code.Equal(ecode.OK) {
err = errors.Wrap(code, d.followModeList)
return
}
b, _ := json.Marshal(&res)
log.Warn("FollowModeList param(%s) res(%s)", b, d.followModeList)
list = make(map[int64]struct{}, len(res.Data))
for _, mid := range res.Data {
list[mid] = struct{}{}
}
return
}

View File

@@ -0,0 +1,75 @@
package recommend
import (
"context"
"testing"
"time"
cdm "go-common/app/interface/main/app-card/model"
"github.com/smartystreets/goconvey/convey"
)
func TestRecommend(t *testing.T) {
var (
c = context.TODO()
plat = int8(1)
buvid = ""
mid = int64(1)
build = int(1111)
loginEvent = int(0)
parentMode = int(0)
recsysMode = int(1)
zoneID = int64(0)
group = int(0)
interest = ""
network = ""
style = int(0)
column cdm.ColumnStatus
flush = int(0)
autoPlay = ""
now time.Time
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, _, _, _, err := d.Recommend(c, plat, buvid, mid, build, loginEvent, parentMode, recsysMode, zoneID, group, interest, network, style, column, flush, autoPlay, now)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestHots(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, err := d.Hots(c)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestTagTop(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
tid = int64(1)
rn = int(0)
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, err := d.TagTop(c, mid, tid, rn)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestGroup(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("Ping", t, func(ctx convey.C) {
d.Group(c)
})
}

View File

@@ -0,0 +1,68 @@
package recommend
import (
"time"
"go-common/app/interface/main/app-feed/conf"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
httpx "go-common/library/net/http/blademaster"
"go-common/library/queue/databus"
)
// Dao is show dao.
type Dao struct {
// http client
client *httpx.Client
clientAsyn *httpx.Client
// hetongzi
hot string
// bigdata
rcmd string
group string
top string
followModeList string
// redis
redis *redis.Pool
expireRds int
// databus
databus *databus.Databus
// mc
mc *memcache.Pool
expireMc int32
}
// New new a show dao.
func New(c *conf.Config) (d *Dao) {
d = &Dao{
// http client
client: httpx.NewClient(c.HTTPData),
clientAsyn: httpx.NewClient(c.HTTPClientAsyn),
// hetongzi
hot: c.Host.Hetongzi + _hot,
// bigdata
rcmd: c.Host.Data + _rcmd,
group: c.Host.BigData + _group,
top: c.Host.Data + _top,
followModeList: c.Host.Data + _followModeList,
// redis
redis: redis.NewPool(c.Redis.Feed.Config),
expireRds: int(time.Duration(c.Redis.Feed.ExpireRecommend) / time.Second),
// databus
databus: databus.New(c.DislikeDatabus),
// mc
mc: memcache.NewPool(c.Memcache.Cache.Config),
expireMc: int32(time.Duration(c.Memcache.Cache.ExpireCache) / time.Second),
}
return
}
// Close close resource.
func (d *Dao) Close() {
if d.redis != nil {
d.redis.Close()
}
if d.mc != nil {
d.mc.Close()
}
}

View File

@@ -0,0 +1,34 @@
package recommend
import (
"flag"
"os"
"testing"
"go-common/app/interface/main/app-feed/conf"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.app-svr.app-feed")
flag.Set("conf_token", "OC30xxkAOyaH9fI6FRuXA0Ob5HL0f3kc")
flag.Set("tree_id", "2686")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
d = New(conf.Conf)
m.Run()
os.Exit(0)
}

View File

@@ -0,0 +1,46 @@
package recommend
import (
"context"
"strconv"
"time"
"go-common/library/log"
"github.com/pkg/errors"
)
// PubDislike is.
func (d *Dao) PubDislike(c context.Context, buvid, gt string, id, mid, reasonID, cmreasonID, feedbackID, upperID, rid, tagID int64, adcb string, now time.Time) (err error) {
return d.pub(c, buvid, gt, id, mid, reasonID, cmreasonID, feedbackID, upperID, rid, tagID, adcb, 1, now)
}
// PubDislikeCancel is.
func (d *Dao) PubDislikeCancel(c context.Context, buvid, gt string, id, mid, reasonID, cmreasonID, feedbackID, upperID, rid, tagID int64, adcb string, now time.Time) (err error) {
return d.pub(c, buvid, gt, id, mid, reasonID, cmreasonID, feedbackID, upperID, rid, tagID, adcb, 2, now)
}
func (d *Dao) pub(c context.Context, buvid, gt string, id, mid, reasonID, cmreasonID, feedbackID, upperID, rid, tagID int64, adcb string, state int8, now time.Time) (err error) {
key := strconv.FormatInt(mid, 10)
msg := struct {
Buvid string `json:"buvid"`
Goto string `json:"goto"`
ID int64 `json:"id"`
Mid int64 `json:"mid"`
ReasonID int64 `json:"reason_id"`
CMReasonID int64 `json:"cm_reason_id"`
FeedbackID int64 `json:"feedback_id"`
UpperID int64 `json:"upper_id"`
Rid int64 `json:"rid"`
TagID int64 `json:"tag_id"`
ADCB string `json:"ad_cb"`
State int8 `json:"state"`
Time int64 `json:"time"`
}{Buvid: buvid, Goto: gt, ID: id, Mid: mid, ReasonID: reasonID, CMReasonID: cmreasonID, FeedbackID: feedbackID, UpperID: upperID, Rid: rid, TagID: tagID, ADCB: adcb, State: state, Time: now.Unix()}
if err = d.databus.Send(c, key, msg); err != nil {
err = errors.Wrapf(err, "%s %v", key, msg)
return
}
log.Info("d.dataBus.Pub(%s,%v)", key, msg)
return
}

View File

@@ -0,0 +1,51 @@
package recommend
import (
"context"
"testing"
"time"
"github.com/smartystreets/goconvey/convey"
)
func TestPubDislike(t *testing.T) {
var (
c = context.TODO()
buvid string
gt string
id int64
mid int64
reasonID int64
cmReasonID int64
feedbackID int64
upperID int64
rid int64
tagID int64
adcb string
now time.Time
)
convey.Convey("Ping", t, func(ctx convey.C) {
d.PubDislike(c, buvid, gt, id, mid, reasonID, cmReasonID, feedbackID, upperID, rid, tagID, adcb, now)
})
}
func TestPubDislikeCancel(t *testing.T) {
var (
c = context.TODO()
buvid = ""
gt = ""
id = int64(1)
mid = int64(1)
reasonID = int64(1)
cmReasonID = int64(1)
feedbackID = int64(1)
upperID = int64(1)
rid = int64(1)
tagID = int64(1)
adcb = ""
now time.Time
)
convey.Convey("Ping", t, func(ctx convey.C) {
d.PubDislikeCancel(c, buvid, gt, id, mid, reasonID, cmReasonID, feedbackID, upperID, rid, tagID, adcb, now)
})
}

View File

@@ -0,0 +1,124 @@
package recommend
import (
"context"
"go-common/app/interface/main/app-card/model/card/ai"
"go-common/library/cache/memcache"
"github.com/pkg/errors"
)
const (
_prefixRcmdAids = "rc"
_prefixRcmd = "rc2"
_prefixFollowModeList = "fml"
)
func keyRcmdAids() string {
return _prefixRcmdAids
}
func keyRcmd() string {
return _prefixRcmd
}
func keyFollowModeList() string {
return _prefixFollowModeList
}
// AddRcmdCache add ai into cahce.
func (d *Dao) AddRcmdAidsCache(c context.Context, aids []int64) (err error) {
conn := d.mc.Get(c)
key := keyRcmdAids()
item := &memcache.Item{Key: key, Object: aids, Flags: memcache.FlagJSON, Expiration: d.expireMc}
if err = conn.Set(item); err != nil {
err = errors.Wrapf(err, "%v", aids)
}
conn.Close()
return
}
// RcmdCache get ai cache data from cache
func (d *Dao) RcmdAidsCache(c context.Context) (aids []int64, err error) {
var r *memcache.Item
conn := d.mc.Get(c)
key := keyRcmdAids()
defer conn.Close()
if r, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
err = errors.Wrap(err, key)
return
}
if err = conn.Scan(r, &aids); err != nil {
err = errors.Wrapf(err, "%s", r.Value)
}
return
}
// AddRcmdCache add ai into cahce.
func (d *Dao) AddRcmdCache(c context.Context, is []*ai.Item) (err error) {
conn := d.mc.Get(c)
key := keyRcmd()
item := &memcache.Item{Key: key, Object: is, Flags: memcache.FlagJSON, Expiration: d.expireMc}
if err = conn.Set(item); err != nil {
err = errors.Wrap(err, key)
}
conn.Close()
return
}
// RcmdCache get ai cache data from cache
func (d *Dao) RcmdCache(c context.Context) (is []*ai.Item, err error) {
var r *memcache.Item
conn := d.mc.Get(c)
key := keyRcmd()
defer conn.Close()
if r, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
err = errors.Wrap(err, key)
return
}
if err = conn.Scan(r, &is); err != nil {
err = errors.Wrapf(err, "%s", r.Value)
}
return
}
// AddFollowModeListCache is.
func (d *Dao) AddFollowModeListCache(c context.Context, list map[int64]struct{}) (err error) {
conn := d.mc.Get(c)
key := keyFollowModeList()
item := &memcache.Item{Key: key, Object: list, Flags: memcache.FlagJSON, Expiration: d.expireMc}
if err = conn.Set(item); err != nil {
err = errors.Wrap(err, key)
}
conn.Close()
return
}
// FollowModeListCache is.
func (d *Dao) FollowModeListCache(c context.Context) (list map[int64]struct{}, err error) {
var r *memcache.Item
conn := d.mc.Get(c)
key := keyFollowModeList()
defer conn.Close()
if r, err = conn.Get(key); err != nil {
if err == memcache.ErrNotFound {
err = nil
return
}
err = errors.Wrap(err, key)
return
}
if err = conn.Scan(r, &list); err != nil {
err = errors.Wrapf(err, "%s", r.Value)
}
return
}

View File

@@ -0,0 +1,60 @@
package recommend
import (
"context"
"testing"
"go-common/app/interface/main/app-card/model/card/ai"
"github.com/smartystreets/goconvey/convey"
)
func TestAddRcmdAidsCache(t *testing.T) {
var (
c = context.TODO()
aids = []int64{1}
)
convey.Convey("Ping", t, func(ctx convey.C) {
err := d.AddRcmdAidsCache(c, aids)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestRcmdAidsCache(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, err := d.RcmdAidsCache(c)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestAddRcmdCache(t *testing.T) {
var (
c = context.TODO()
is = []*ai.Item{}
)
convey.Convey("Ping", t, func(ctx convey.C) {
err := d.AddRcmdCache(c, is)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestRcmdCache(t *testing.T) {
var (
c = context.TODO()
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, err := d.RcmdCache(c)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}

View File

@@ -0,0 +1,46 @@
package recommend
import (
"context"
"strconv"
"go-common/library/cache/redis"
"github.com/pkg/errors"
)
const (
_prefixPos = "p_"
)
func keyPos(mid int64) string {
return _prefixPos + strconv.FormatInt(mid%100000, 10)
}
func (d *Dao) PositionCache(c context.Context, mid int64) (pos int, err error) {
var (
key = keyPos(mid)
conn = d.redis.Get(c)
)
defer conn.Close()
if pos, err = redis.Int(conn.Do("HGET", key, mid)); err != nil {
if err == redis.ErrNil {
err = nil
return
}
err = errors.Wrapf(err, "conn.Do(HGET,%s,%d)", key, mid)
}
return
}
func (d *Dao) AddPositionCache(c context.Context, mid int64, pos int) (err error) {
var (
key = keyPos(mid)
conn = d.redis.Get(c)
)
defer conn.Close()
if _, err = conn.Do("HSET", key, mid, pos); err != nil {
err = errors.Wrapf(err, "conn.Do(HSET,%s,%d,%d)", key, mid, pos)
}
return
}

View File

@@ -0,0 +1,35 @@
package recommend
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestPositionCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
)
convey.Convey("Ping", t, func(ctx convey.C) {
_, err := d.PositionCache(c, mid)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}
func TestAddPositionCache(t *testing.T) {
var (
c = context.TODO()
mid = int64(1)
pos = int(1)
)
convey.Convey("Ping", t, func(ctx convey.C) {
err := d.AddPositionCache(c, mid, pos)
ctx.Convey("Then err should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
}