Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

37
app/job/live/BUILD Normal file
View File

@@ -0,0 +1,37 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/dao-anchor-job/api/v1:all-srcs",
"//app/job/live/dao-anchor-job/cmd:all-srcs",
"//app/job/live/dao-anchor-job/internal/conf:all-srcs",
"//app/job/live/dao-anchor-job/internal/dao:all-srcs",
"//app/job/live/dao-anchor-job/internal/model:all-srcs",
"//app/job/live/dao-anchor-job/internal/server/http:all-srcs",
"//app/job/live/dao-anchor-job/internal/service:all-srcs",
"//app/job/live/gift:all-srcs",
"//app/job/live/push-search/cmd:all-srcs",
"//app/job/live/push-search/conf:all-srcs",
"//app/job/live/push-search/dao:all-srcs",
"//app/job/live/push-search/http:all-srcs",
"//app/job/live/push-search/model:all-srcs",
"//app/job/live/push-search/service:all-srcs",
"//app/job/live/recommend-job:all-srcs",
"//app/job/live/wallet:all-srcs",
"//app/job/live/xlottery:all-srcs",
"//app/job/live/xroom-feed/cmd:all-srcs",
"//app/job/live/xroom-feed/internal/dao:all-srcs",
"//app/job/live/xroom-feed/internal/model:all-srcs",
"//app/job/live/xroom-feed/internal/server/http:all-srcs",
"//app/job/live/xroom-feed/internal/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,6 @@
# Owner
liuzhen
lidongyang
xiehaishen
zhaohailin
yangbaibing

12
app/job/live/OWNERS Normal file
View File

@@ -0,0 +1,12 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- lidongyang
- liuzhen
- xiehaishen
- yangbaibing
- zhaohailin
labels:
- job
- live
- new-project

View File

@@ -0,0 +1,49 @@
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
proto_library(
name = "v1_proto",
srcs = ["api.proto"],
tags = ["automanaged"],
)
go_proto_library(
name = "v1_go_proto",
compilers = ["@io_bazel_rules_go//proto:go_proto"],
importpath = "go-common/app/job/live/dao-anchor-job/api/v1",
proto = ":v1_proto",
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["generate.go"],
embed = [":v1_go_proto"],
importpath = "go-common/app/job/live/dao-anchor-job/api/v1",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@com_github_gogo_protobuf//proto:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,33 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: app/job/live/dao-anchor-job/api/v1/api.proto
package v1
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
func init() {
proto.RegisterFile("app/job/live/dao-anchor-job/api/v1/api.proto", fileDescriptor_api_246501f09888d3ec)
}
var fileDescriptor_api_246501f09888d3ec = []byte{
// 89 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x49, 0x2c, 0x28, 0xd0,
0xcf, 0xca, 0x4f, 0xd2, 0xcf, 0xc9, 0x2c, 0x4b, 0xd5, 0x4f, 0x49, 0xcc, 0xd7, 0x4d, 0xcc, 0x4b,
0xce, 0xc8, 0x2f, 0xd2, 0x05, 0x89, 0x25, 0x16, 0x64, 0xea, 0x97, 0x19, 0x82, 0x28, 0xbd, 0x82,
0xa2, 0xfc, 0x92, 0x7c, 0x27, 0x81, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0,
0x48, 0x8e, 0x31, 0x8a, 0xa9, 0xcc, 0x30, 0x89, 0x0d, 0x2c, 0x61, 0x0c, 0x08, 0x00, 0x00, 0xff,
0xff, 0x75, 0x85, 0x9c, 0x41, 0x48, 0x00, 0x00, 0x00,
}

View File

@@ -0,0 +1,16 @@
// 定义项目 API 的 proto 文件 可以同时描述 gRPC 和 HTTP API
// protobuf 文件参考:
// - https://developers.google.com/protocol-buffers/
// - http://info.bilibili.co/display/documentation/gRPC+Proto
// protobuf 生成 HTTP 工具:
// - http://git.bilibili.co/platform/go-common/tree/master/app/tool/protoc-gen-bm
syntax = "proto3";
// package 命名使用 {discovery_id}.{version} 的方式, version 形如 v1, v2, v1beta ..
// NOTE: 不知道的 discovery_id 请询问大佬, 新项目找大佬申请 discovery_id先到先得抢注
// e.g. account.service.v1
// package {discovery_id}.{version}
// NOTE: 最后请删除这些无用的注释 (゜-゜)つロ
option go_package = "v1";

View File

@@ -0,0 +1,4 @@
package v1
// 生成 gRPC 代码
//go:generate $GOPATH/src/go-common/app/tool/warden/protoc.sh

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["test.toml"],
importpath = "go-common/app/job/live/dao-anchor-job/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/dao-anchor-job/internal/conf:go_default_library",
"//app/job/live/dao-anchor-job/internal/server/http:go_default_library",
"//app/job/live/dao-anchor-job/internal/service:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,47 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/live/dao-anchor-job/internal/conf"
"go-common/app/job/live/dao-anchor-job/internal/server/http"
"go-common/app/job/live/dao-anchor-job/internal/service"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("dao-anchor-job start")
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
svc := service.New(conf.Conf)
http.Init(conf.Conf, svc)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
svc.Close()
log.Info("dao-anchor-job exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,22 @@
[httpCli]
key = "fb06a25c6338edbc"
secret = "fd10bd177559780c2e4a44f1fa47fa83"
dial = "100ms"
timeout = "5s"
keepAlive = "60s"
[httpClient.breaker]
window = "3s"
sleep = "100ms"
bucket = 10
ratio = 0.5
request = 100
[coverControl]
CoverCron = "0 */5 * * * *"
pieceSize = 50
[MinuteControl]
MinuteCron = "0 */1 * * * *"
pieceSize = 50
[Minute3Control]
Minute3Cron = "0 */3 * * * *"
pieceSize = 50

View File

@@ -0,0 +1,40 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/dao-anchor-job/internal/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/conf:go_default_library",
"//library/database/bfs:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/rpc/liverpc:go_default_library",
"//library/net/rpc/warden:go_default_library",
"//library/net/trace:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,105 @@
package conf
import (
"errors"
"flag"
"go-common/library/database/bfs"
"go-common/library/net/rpc/liverpc"
"go-common/library/net/rpc/warden"
"go-common/library/conf"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Ecode *ecode.Config
LiveRpc map[string]*liverpc.ClientConfig
GrpcCli *warden.ClientConfig
BfsCli *bfs.Config
HttpCli *bm.ClientConfig
CoverControl *CoverControl
Minute3Control *Minute3Control
MinuteControl *MinuteControl
}
type CoverControl struct {
CoverCron string
PieceSize int
}
type Minute3Control struct {
Minute3Cron string
PieceSize int
}
type MinuteControl struct {
MinuteCron string
PieceSize int
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,47 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"av.go",
"bfs.go",
"dao.go",
"daoAnchor.go",
"video.go",
],
importpath = "go-common/app/job/live/dao-anchor-job/internal/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/dao-anchor-job/internal/conf:go_default_library",
"//app/service/live/av/api/liverpc:go_default_library",
"//app/service/live/av/api/liverpc/v1:go_default_library",
"//app/service/live/dao-anchor/api/grpc/v0:go_default_library",
"//app/service/live/dao-anchor/api/grpc/v1:go_default_library",
"//app/service/video/stream-mng/api/v1:go_default_library",
"//library/database/bfs:go_default_library",
"//library/ecode:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/rpc/liverpc:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,24 @@
package dao
import (
"context"
"go-common/app/service/live/av/api/liverpc/v1"
"go-common/library/log"
)
// GetFansMedalInfo 获取粉丝勋章信息
func (d *Dao) GetPkStatus(c context.Context, roomId int64) (resp *v1.PkGetPkStatusResp_Data, err error) {
reply, err := d.AvApi.V1Pk.GetPkStatus(c, &v1.PkGetPkStatusReq{RoomId: roomId})
if err != nil {
log.Error("av_GetPkStatus_error:%v", err)
return
}
if reply.Code != 0 {
log.Error("av_GetPkStatus_error:%d,%s,%v", reply.Code, reply.Msg, reply.Data)
return
}
log.Info("av_GetPkStatus:%d,%s,$v", reply.Code, reply.Msg, reply.Data)
resp = reply.Data
return
}

View File

@@ -0,0 +1,56 @@
package dao
import (
"context"
"errors"
"io/ioutil"
"net/http"
"strconv"
"go-common/library/database/bfs"
"go-common/library/log"
)
const BUCKET = "live"
const FILE_PATH = "/data/www/cover/"
//上传至bfs相关接口
//ImgUpload 上传图片至bfs
func (d *Dao) ImgUpload(ctx context.Context, roomId int64, pic string, file []byte) (resp string, err error) {
log.Info("ImgUpload_start")
fileName := strconv.Itoa(int(roomId)) + ".jpg"
reply, err := d.BfsClient.Upload(ctx, &bfs.Request{
Bucket: BUCKET,
Dir: "",
ContentType: "",
Filename: fileName,
File: []byte(file),
})
if err != nil {
log.Error("ImgUpload_bfs_Upload_failed,err:%v", err)
return
}
resp = reply
return
}
func (d *Dao) ImgDownload(ctx context.Context, pic string) (resp []byte, err error) {
reply, err := http.Get(pic)
if err != nil {
log.Warn("ImgDownload_failed_err:%v", err)
return
}
defer reply.Body.Close()
if reply.StatusCode != 200 {
err = errors.New("curl error http code not equal to 200")
log.Warn("ImgDownload_failed_httpCode:%d", reply.StatusCode)
return
}
resp, err = ioutil.ReadAll(reply.Body)
if err != nil {
log.Warn("ImgDownload_read_err:%v", err)
return
}
return
}

View File

@@ -0,0 +1,70 @@
package dao
import (
"context"
"go-common/app/job/live/dao-anchor-job/internal/conf"
av_api "go-common/app/service/live/av/api/liverpc"
daoAnchor_api_v0 "go-common/app/service/live/dao-anchor/api/grpc/v0"
daoAnchor_api "go-common/app/service/live/dao-anchor/api/grpc/v1"
video_api "go-common/app/service/video/stream-mng/api/v1"
"go-common/library/database/bfs"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/rpc/liverpc"
)
// Dao dao
type Dao struct {
c *conf.Config
AvApi *av_api.Client
daoAnchorApi *daoAnchor_api.Client
VideoApi video_api.StreamClient
BfsClient *bfs.BFS
HttpClient *bm.Client
daoAnchorApiV0 *daoAnchor_api_v0.Client
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
AvApi: av_api.New(getConf("av")),
BfsClient: bfs.New(c.BfsCli),
HttpClient: bm.NewClient(c.HttpCli),
}
daoAnchorApi, err := daoAnchor_api.NewClient(c.GrpcCli)
if err != nil {
panic(err)
}
dao.daoAnchorApi = daoAnchorApi
videoApi, err := video_api.NewClient(c.GrpcCli)
if err != nil {
panic(err)
}
dao.VideoApi = videoApi
daoAnchorApiV0, err := daoAnchor_api_v0.NewClient(c.GrpcCli)
if err != nil {
panic(err)
}
dao.daoAnchorApiV0 = daoAnchorApiV0
return
}
// Close close the resource.
func (d *Dao) Close() {
}
// Ping dao ping
func (d *Dao) Ping(ctx context.Context) error {
// TODO: add mc,redis... if you use
return nil
}
func getConf(appName string) *liverpc.ClientConfig {
c := conf.Conf.LiveRpc
if c != nil {
return c[appName]
}
return nil
}

View File

@@ -0,0 +1,146 @@
package dao
import (
"context"
"time"
"go-common/library/ecode"
"fmt"
daoAnchorV0 "go-common/app/service/live/dao-anchor/api/grpc/v0"
"go-common/app/service/live/dao-anchor/api/grpc/v1"
"go-common/library/log"
)
//LIVE_OPEN 开播
const LIVE_OPEN = 1
//LIVE_CLOSE 关播
const LIVE_CLOSE = 0
//LIVE_ROUND 轮播
const LIVE_ROUND = 2
//PAGE_SIZE 分页数据量
const PAGE_SIZE = 100
//RETRY_TIME 接口充实次数
const RETRY_TIME = 3
//GetAllLiveRoom 获取在播列表
func (d *Dao) GetAllLiveRoom(c context.Context, fields []string) (resp map[int64]*v1.RoomData, err error) {
page := 0
resp = map[int64]*v1.RoomData{}
retry := 1
for true {
reply, err := d.daoAnchorApi.RoomOnlineList(c, &v1.RoomOnlineListReq{Fields: fields, Page: int64(page), PageSize: PAGE_SIZE})
if err != nil {
if retry >= RETRY_TIME {
break
}
retry++
time.Sleep(time.Second * 3)
log.Errorw(c, "log", fmt.Sprintf("getAllLiveRoom_RoomOnlineList_error:page=%d;err=%v", page, err))
continue
}
if len(reply.RoomDataList) <= 0 {
break
}
page = page + 1
roomDataList := reply.RoomDataList
for roomId, info := range roomDataList {
v := info
resp[roomId] = v
}
time.Sleep(time.Millisecond)
}
return
}
//GetAllLiveRoomIds 获取在播列表
func (d *Dao) GetAllLiveRoomIds(c context.Context) (resp []int64, err error) {
page := 0
reply, err := d.daoAnchorApi.RoomOnlineListByArea(c, &v1.RoomOnlineListByAreaReq{})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("GetAllLiveRoomIds_RoomOnlineList_error:page=%d;err=%v", page, err))
return
}
if len(reply.RoomIds) <= 0 {
return
}
resp = reply.RoomIds
return
}
//GetInfosByRoomIds 获取主播房间信息
func (d *Dao) GetInfosByRoomIds(c context.Context, roomIds []int64, fields []string) (resp map[int64]*v1.RoomData, err error) {
if roomIds == nil {
err = ecode.InvalidParam
log.Errorw(c, "log", fmt.Sprintf("getInfosByRoomIds_params_error:%v", roomIds))
return
}
reply, err := d.daoAnchorApi.FetchRoomByIDs(c, &v1.RoomByIDsReq{RoomIds: roomIds, Fields: fields})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("getInfosByRoomIds_FetchRoomByIDs_error:reply=%v;err=%v", reply, err))
return
}
if reply == nil {
err = ecode.CallDaoAnchorError
log.Errorw(c, "log", "getInfosByRoomIds_FetchRoomByIDs_error")
return
}
resp = reply.RoomDataSet
return
}
//UpdateRoomEx ...
func (d *Dao) UpdateRoomEx(c context.Context, roomId int64, fields []string, keyFrame string) (resp int64, err error) {
if roomId < 0 {
err = ecode.InvalidParam
log.Errorw(c, "log", fmt.Sprintf("updateRoom_params_error:%v", roomId))
return
}
reply, err := d.daoAnchorApi.RoomExtendUpdate(c, &v1.RoomExtendUpdateReq{Fields: fields, RoomId: roomId, Keyframe: keyFrame})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("updateRoom_RoomUpdate_error:reply=%v;err=%v", reply, err))
return
}
resp = reply.AffectedRows
return
}
func (d *Dao) CreateCacheList(c context.Context, roomIds []int64, content string) (err error) {
if len(roomIds) <= 0 || content == "" {
log.Errorw(c, "log", fmt.Sprintf("CreateCacheList_params_error:room_id=%v;content=%s", roomIds, content))
return
}
reply, err := d.daoAnchorApiV0.CreateLiveCacheList(c, &daoAnchorV0.CreateLiveCacheListReq{RoomIds: roomIds, Content: content})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("CreateCacheList_error:reply=%v;err=%v", reply, err))
return
}
log.Info("CreateCacheList_info:roomIds=%v;content=%s;reply=%v", roomIds, content, reply)
return
}
func (d *Dao) GetContentMap(c context.Context) (resp map[string]int64, err error) {
reply, err := d.daoAnchorApiV0.GetContentMap(c, &daoAnchorV0.GetContentMapReq{})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("GetContentMap_error:reply=%v;err=%v", reply, err))
return
}
resp = reply.List
return
}
func (d *Dao) CreateDBData(c context.Context, roomIds []int64, content string) (err error) {
if len(roomIds) <= 0 || content == "" {
log.Errorw(c, "log", fmt.Sprintf("CreateCacheList_params_error:room_id=%v;content=%s", roomIds, content))
return
}
reply, err := d.daoAnchorApiV0.CreateDBData(c, &daoAnchorV0.CreateDBDataReq{RoomIds: roomIds, Content: content})
if err != nil {
log.Errorw(c, "log", fmt.Sprintf("CreateCacheList_error:reply=%v;err=%v", reply, err))
return
}
return
}

View File

@@ -0,0 +1,26 @@
package dao
import (
"context"
"time"
"go-common/app/service/video/stream-mng/api/v1"
"go-common/library/log"
)
//视频云接口调用
//GetPicsByRoomId 根据房间id获取当前关键帧
func (d *Dao) GetPicsByRoomId(c context.Context, roomId int64, startTime time.Time, endTime time.Time) (resp []string, err error) {
reply, err := d.VideoApi.GetSingleScreeShot(c, &v1.GetSingleScreeShotReq{RoomId: roomId, StartTime: startTime.Format("2006-01-02 15:04:05"), EndTime: endTime.Format("2006-01-02 15:04:05")})
if err != nil {
log.Error("getPicsByRoomId_GetSingleScreeShot_error:reply:%v;err=%v", reply, err)
return
}
if reply == nil {
log.Error("getPicsByRoomId_GetSingleScreeShot_error")
return
}
resp = reply.List
return
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/job/live/dao-anchor-job/internal/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1 @@
package model

View File

@@ -0,0 +1,35 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/live/dao-anchor-job/internal/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/dao-anchor-job/internal/conf:go_default_library",
"//app/job/live/dao-anchor-job/internal/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,53 @@
package http
import (
"net/http"
"go-common/app/job/live/dao-anchor-job/internal/conf"
"go-common/app/job/live/dao-anchor-job/internal/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
vfy *verify.Verify
svc *service.Service
)
// Init init
func Init(c *conf.Config, s *service.Service) {
svc = s
vfy = verify.New(c.Verify)
engine := bm.DefaultServer(c.BM)
route(engine)
if err := engine.Start(); err != nil {
log.Error("bm Start error(%v)", err)
panic(err)
}
}
func route(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/x/dao-anchor-job")
{
g.GET("/start", vfy.Verify, howToStart)
}
}
func ping(ctx *bm.Context) {
if err := svc.Ping(ctx); err != nil {
log.Error("ping error(%v)", err)
ctx.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}
// example for http request handler
func howToStart(c *bm.Context) {
c.String(0, "Golang 大法好 !!!")
}

View File

@@ -0,0 +1,55 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"cover_test.go",
"data_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/live/dao-anchor-job/internal/conf:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"cover.go",
"data.go",
"service.go",
],
importpath = "go-common/app/job/live/dao-anchor-job/internal/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/dao-anchor-job/internal/conf:go_default_library",
"//app/job/live/dao-anchor-job/internal/dao:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
"//library/sync/errgroup:go_default_library",
"//vendor/github.com/robfig/cron:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,112 @@
package service
import (
"context"
"strings"
"time"
"go-common/app/job/live/dao-anchor-job/internal/dao"
"go-common/library/sync/errgroup"
"go-common/library/log"
)
//封面图/关键帧相关脚本
const ROOM_LEN_KEY_FRAME = 500
//updateKeyFrame 更新关键帧
func (s *Service) updateKeyFrame() {
ctx := context.TODO()
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
defer cancel()
log.Info("updateKeyFrame_start")
//获取全量开播房间
allLiveingRoom, err := s.dao.GetAllLiveRoomIds(ctx)
if allLiveingRoom == nil || err != nil {
log.Error("updateKeyFrame_allLiveingRoom_error:reply=%v;err=%v", allLiveingRoom, err)
return
}
slice := make([]int64, 0)
eg := errgroup.Group{}
for i := 0; i < len(allLiveingRoom); {
end := ROOM_LEN_KEY_FRAME + i
if (ROOM_LEN_KEY_FRAME + i) >= len(allLiveingRoom) {
end = len(allLiveingRoom)
}
slice = allLiveingRoom[i:end]
if len(slice) <= 0 {
break
} else {
eg.Go(func(sliceParam []int64) func() error {
return func() (err error) {
for _, roomId := range sliceParam {
coverUrl, err := s.dealKeyFrame(ctx, roomId)
if err != nil {
time.Sleep(time.Second)
log.Error("updateKeyFrame_deal_error:roomId=%d;ketFrame=%s", roomId, coverUrl)
continue
}
if coverUrl == "" {
continue
}
//更新关键帧
coverUrlArr := strings.Split(coverUrl, "?")
coverUrl = coverUrlArr[0] + "?" + time.Now().Format("01021504")
s.dao.UpdateRoomEx(ctx, roomId, []string{"keyframe"}, coverUrl)
time.Sleep(time.Millisecond * 10)
}
return
}
}(slice))
}
i = end
}
eg.Wait()
log.Info("updateKeyFrame_end")
return
}
func (s *Service) dealKeyFrame(ctx context.Context, roomId int64) (coverUrl string, err error) {
//二次确认是否关播,关播不再做
roomInfos, err := s.dao.GetInfosByRoomIds(ctx, []int64{roomId}, []string{"live_status"})
if err != nil {
log.Error("updateKeyFrame_GetInfosByRoomIds_error:room_id=%d;err=%v", roomId, err)
return
}
roomInfo := roomInfos[roomId]
//未开播,不更新关键帧
if roomInfo == nil || roomInfo.LiveStatus != dao.LIVE_OPEN {
return
}
//判断是否为pk房间,pk房间不更新关键帧
pkReply, err := s.dao.GetPkStatus(ctx, roomId)
if err != nil {
log.Error("updateKeyFrame_GetPkStatus_error:room_id=%d", roomId)
return
}
if pkReply.PkStatus > 0 {
return
}
//获取关键帧
startTime := time.Now().Add(-time.Minute)
endTime := time.Now()
pics, err := s.dao.GetPicsByRoomId(ctx, roomId, startTime, endTime)
if err != nil || pics == nil || pics[0] == "" {
log.Warn("updateKeyFrame_GetPicsByRoomId_error:room_id=%d;pics=%v;err=%v", roomId, pics, err)
return
}
//上传至bfs
reply, err := s.dao.ImgDownload(ctx, pics[0])
if err != nil || reply == nil {
log.Warn("updateKeyFrame_ImgDownload_error:room_id=%d;pic=%s;err=%v;reply=%v", roomId, pics[0], err, reply)
return
}
coverUrl, err = s.dao.ImgUpload(ctx, roomId, pics[0], reply)
if err != nil || coverUrl == "" {
log.Error("updateKeyFrame_ImgUploadBfs_error:room_id=%d;pic=%s", roomId, pics[0])
return
}
return
}

View File

@@ -0,0 +1,26 @@
package service
import (
"flag"
"testing"
. "github.com/smartystreets/goconvey/convey"
"go-common/app/job/live/dao-anchor-job/internal/conf"
)
var s *Service
func init() {
flag.Set("conf", "../../cmd/test.toml")
if err := conf.Init(); err != nil {
panic(err)
}
s = New(conf.Conf)
}
func TestCover(t *testing.T) {
Convey("testCover", t, func() {
s.updateKeyFrame()
})
}

View File

@@ -0,0 +1,95 @@
package service
import (
"context"
"fmt"
"time"
"go-common/library/log"
"go-common/library/sync/errgroup"
)
const ROOM_LEN = 300
//实时数据处理逻辑,生成list
func (s *Service) minuteDataToCacheList() {
ctx := context.TODO()
ctx, cancel := context.WithTimeout(ctx, time.Minute*5)
defer cancel()
ctx = GetTraceLogCtx(ctx, "minuteDataToCacheList")
log.Infow(ctx, "log", "minuteDataToCacheList_start")
//获取需要生成的数据的content列表
contentMap, err := s.dao.GetContentMap(ctx)
if err != nil {
log.Errorw(ctx, "log", fmt.Sprintf("minuteDataToCacheList_GetContentMap_error:reply=%v;err=%v", contentMap, err))
return
}
//获取全量开播房间
allLiveingRoomIds, err := s.dao.GetAllLiveRoomIds(ctx)
if allLiveingRoomIds == nil || err != nil {
log.Errorw(ctx, "log", fmt.Sprintf("minuteDataToCacheList_allLiveingRoomIds_error:reply=%v;err=%v", allLiveingRoomIds, err))
return
}
eg := errgroup.Group{}
for content := range contentMap {
log.Infow(ctx, fmt.Sprintf("minuteDataToCacheList_start:%s", content))
eg.Go(func(contentParam string) func() error {
slice := make([]int64, 0)
for i := 0; i < len(allLiveingRoomIds); {
end := ROOM_LEN + i
if ROOM_LEN+i >= len(allLiveingRoomIds) {
end = len(allLiveingRoomIds)
}
slice = allLiveingRoomIds[i:end]
if len(slice) <= 0 {
break
} else {
s.dao.CreateCacheList(ctx, slice, contentParam)
}
i = end
}
log.Infow(ctx, "log", fmt.Sprintf("minuteDataToCacheList_end_content=%s;err=%v", contentParam, err))
return nil
}(content))
}
eg.Wait()
log.Infow(ctx, "log", "minuteDataToCacheList_end")
return
}
func (s *Service) minuteDataToDB() {
ctx := context.TODO()
ctx, cancel := context.WithTimeout(ctx, time.Minute*3)
defer cancel()
ctx = GetTraceLogCtx(ctx, "minuteDataToDB")
log.Infow(ctx, "log", "minuteDataToDB_start")
//获取需要生成的数据的content列表
contentMap, err := s.dao.GetContentMap(ctx)
if err != nil {
log.Errorw(ctx, "log", fmt.Sprintf("data_allLiveingRoomIds_error:reply=%v;err=%v", contentMap, err))
return
}
//获取全量开播房间
allLiveingRoomIds, err := s.dao.GetAllLiveRoomIds(ctx)
if allLiveingRoomIds == nil || err != nil {
log.Errorw(ctx, "log", fmt.Sprintf("data_allLiveingRoomIds_error:reply=%v;err=%v", allLiveingRoomIds, err))
return
}
eg := errgroup.Group{}
for content := range contentMap {
log.Info("minuteDataToCacheList_start:" + content)
eg.Go(func(contentParam string) func() error {
return func() (err error) {
for _, roomId := range allLiveingRoomIds {
s.dao.CreateDBData(ctx, []int64{int64(roomId)}, contentParam)
}
log.Infow(ctx, "log", fmt.Sprintf("minuteDataToCacheList_end_content=%s;err=%v", contentParam, err))
return
}
}(content))
}
eg.Wait()
log.Infow(ctx, "log", fmt.Sprintf("minuteDataToDB_end"))
return
}

View File

@@ -0,0 +1,30 @@
package service
import (
"flag"
"testing"
. "github.com/smartystreets/goconvey/convey"
"go-common/app/job/live/dao-anchor-job/internal/conf"
)
func init() {
flag.Set("conf", "../../cmd/test.toml")
if err := conf.Init(); err != nil {
panic(err)
}
s = New(conf.Conf)
}
func TestMinuteDataToCacheList(t *testing.T) {
Convey("testMinuteDataToCacheList", t, func() {
s.minuteDataToCacheList()
})
}
func TestMinuteDataToDB(t *testing.T) {
Convey("TestMinuteDataToDB", t, func() {
s.minuteDataToDB()
})
}

View File

@@ -0,0 +1,61 @@
package service
import (
"context"
"go-common/library/net/trace"
"go-common/library/log"
"github.com/robfig/cron"
"go-common/app/job/live/dao-anchor-job/internal/conf"
"go-common/app/job/live/dao-anchor-job/internal/dao"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
cron *cron.Cron
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
cron: cron.New(),
}
if err := s.cron.AddFunc(s.c.CoverControl.CoverCron, s.updateKeyFrame); err != nil {
panic(err)
}
if err := s.cron.AddFunc(s.c.Minute3Control.Minute3Cron, s.minuteDataToDB); err != nil {
panic(err)
}
if err := s.cron.AddFunc(s.c.MinuteControl.MinuteCron, s.minuteDataToCacheList); err != nil {
panic(err)
}
s.cron.Start()
return s
}
// Ping Service
func (s *Service) Ping(ctx context.Context) (err error) {
return s.dao.Ping(ctx)
}
// Close Service
func (s *Service) Close() {
log.Info("Crontab Closed!")
s.cron.Stop()
log.Info("Physical Dao Closed!")
s.dao.Close()
log.Info("tv-job has been closed.")
}
func GetTraceLogCtx(ctx context.Context, title string) (ctxNew context.Context) {
t := trace.New(title)
ctxNew = trace.NewContext(ctx, t)
return
}

22
app/job/live/gift/BUILD Normal file
View File

@@ -0,0 +1,22 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/gift/api:all-srcs",
"//app/job/live/gift/cmd:all-srcs",
"//app/job/live/gift/internal/conf:all-srcs",
"//app/job/live/gift/internal/dao:all-srcs",
"//app/job/live/gift/internal/model:all-srcs",
"//app/job/live/gift/internal/server/http:all-srcs",
"//app/job/live/gift/internal/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,2 @@
### v1.0.0
1. 上线功能xxx

View File

@@ -0,0 +1,6 @@
# Owner
shenli01
# Author
# Reviewer

10
app/job/live/gift/OWNERS Normal file
View File

@@ -0,0 +1,10 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- shenli01
labels:
- job
- job/live/gift
- live
options:
no_parent_owners: true

View File

@@ -0,0 +1,12 @@
# gift-job
## 项目简介
1.
## 编译环境
## 依赖包
## 编译执行

View File

@@ -0,0 +1,63 @@
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
proto_library(
name = "api_proto",
srcs = ["api.proto"],
tags = ["automanaged"],
deps = [
"@com_google_protobuf//:empty_proto",
"@gogo_special_proto//github.com/gogo/protobuf/gogoproto",
],
)
go_proto_library(
name = "api_go_proto",
compilers = ["@io_bazel_rules_go//proto:gogofast_grpc"],
importpath = "go-common/app/job/live/gift/api",
proto = ":api_proto",
tags = ["automanaged"],
deps = [
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@io_bazel_rules_go//proto/wkt:empty_go_proto",
],
)
go_library(
name = "go_default_library",
srcs = ["generate.go"],
embed = [":api_go_proto"],
importpath = "go-common/app/job/live/gift/api",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"@com_github_gogo_protobuf//gogoproto:go_default_library",
"@com_github_gogo_protobuf//proto:go_default_library",
"@io_bazel_rules_go//proto/wkt:empty_go_proto",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_x_net//context:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,424 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: app/job/live/gift/api/api.proto
package api
/*
package 命名使用 {discovery_id}.{version} 的方式, version 形如 v1, v2, v1beta ..
NOTE: 不知道的 discovery_id 请询问大佬, 新项目找大佬申请 discovery_id先到先得抢注
e.g. account.service.v1
*/
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import empty "github.com/golang/protobuf/ptypes/empty"
import (
context "golang.org/x/net/context"
grpc "google.golang.org/grpc"
)
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type HelloReq struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty" form:"name" validate:"required"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *HelloReq) Reset() { *m = HelloReq{} }
func (m *HelloReq) String() string { return proto.CompactTextString(m) }
func (*HelloReq) ProtoMessage() {}
func (*HelloReq) Descriptor() ([]byte, []int) {
return fileDescriptor_api_d309cedea9eacff7, []int{0}
}
func (m *HelloReq) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *HelloReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_HelloReq.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *HelloReq) XXX_Merge(src proto.Message) {
xxx_messageInfo_HelloReq.Merge(dst, src)
}
func (m *HelloReq) XXX_Size() int {
return m.Size()
}
func (m *HelloReq) XXX_DiscardUnknown() {
xxx_messageInfo_HelloReq.DiscardUnknown(m)
}
var xxx_messageInfo_HelloReq proto.InternalMessageInfo
func init() {
proto.RegisterType((*HelloReq)(nil), "demo.service.v1.HelloReq")
}
// Reference imports to suppress errors if they are not otherwise used.
var _ context.Context
var _ grpc.ClientConn
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion4
// DemoClient is the client API for Demo service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type DemoClient interface {
SayHello(ctx context.Context, in *HelloReq, opts ...grpc.CallOption) (*empty.Empty, error)
}
type demoClient struct {
cc *grpc.ClientConn
}
func NewDemoClient(cc *grpc.ClientConn) DemoClient {
return &demoClient{cc}
}
func (c *demoClient) SayHello(ctx context.Context, in *HelloReq, opts ...grpc.CallOption) (*empty.Empty, error) {
out := new(empty.Empty)
err := c.cc.Invoke(ctx, "/demo.service.v1.Demo/SayHello", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// DemoServer is the server API for Demo service.
type DemoServer interface {
SayHello(context.Context, *HelloReq) (*empty.Empty, error)
}
func RegisterDemoServer(s *grpc.Server, srv DemoServer) {
s.RegisterService(&_Demo_serviceDesc, srv)
}
func _Demo_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(HelloReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DemoServer).SayHello(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/demo.service.v1.Demo/SayHello",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DemoServer).SayHello(ctx, req.(*HelloReq))
}
return interceptor(ctx, in, info, handler)
}
var _Demo_serviceDesc = grpc.ServiceDesc{
ServiceName: "demo.service.v1.Demo",
HandlerType: (*DemoServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "SayHello",
Handler: _Demo_SayHello_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "app/job/live/gift/api/api.proto",
}
func (m *HelloReq) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *HelloReq) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Name) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintApi(dAtA, i, uint64(len(m.Name)))
i += copy(dAtA[i:], m.Name)
}
if m.XXX_unrecognized != nil {
i += copy(dAtA[i:], m.XXX_unrecognized)
}
return i, nil
}
func encodeVarintApi(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *HelloReq) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Name)
if l > 0 {
n += 1 + l + sovApi(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovApi(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozApi(x uint64) (n int) {
return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *HelloReq) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: HelloReq: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: HelloReq: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowApi
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthApi
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Name = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipApi(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthApi
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipApi(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowApi
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowApi
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowApi
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthApi
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowApi
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipApi(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowApi = fmt.Errorf("proto: integer overflow")
)
func init() {
proto.RegisterFile("app/job/live/gift/api/api.proto", fileDescriptor_api_d309cedea9eacff7)
}
var fileDescriptor_api_d309cedea9eacff7 = []byte{
// 250 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4f, 0x2c, 0x28, 0xd0,
0xcf, 0xca, 0x4f, 0xd2, 0xcf, 0xc9, 0x2c, 0x4b, 0xd5, 0x4f, 0xcf, 0x4c, 0x2b, 0xd1, 0x4f, 0x2c,
0xc8, 0x04, 0x61, 0xbd, 0x82, 0xa2, 0xfc, 0x92, 0x7c, 0x21, 0xfe, 0x94, 0xd4, 0xdc, 0x7c, 0xbd,
0xe2, 0xd4, 0xa2, 0xb2, 0xcc, 0xe4, 0x54, 0xbd, 0x32, 0x43, 0x29, 0xdd, 0xf4, 0xcc, 0x92, 0x8c,
0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, 0xf4, 0xfc, 0xf4, 0x7c, 0x7d, 0xb0, 0xba, 0xa4, 0xd2,
0x34, 0x30, 0x0f, 0xcc, 0x01, 0xb3, 0x20, 0xfa, 0xa5, 0xa4, 0xd3, 0xf3, 0xf3, 0xd3, 0x73, 0x52,
0x11, 0xaa, 0x52, 0x73, 0x0b, 0x4a, 0x2a, 0x21, 0x92, 0x4a, 0xce, 0x5c, 0x1c, 0x1e, 0xa9, 0x39,
0x39, 0xf9, 0x41, 0xa9, 0x85, 0x42, 0xe6, 0x5c, 0x2c, 0x79, 0x89, 0xb9, 0xa9, 0x12, 0x8c, 0x0a,
0x8c, 0x1a, 0x9c, 0x4e, 0xca, 0x9f, 0xee, 0xc9, 0xcb, 0xa7, 0xe5, 0x17, 0xe5, 0x5a, 0x29, 0x81,
0x44, 0x95, 0x14, 0xca, 0x12, 0x73, 0x32, 0x53, 0x12, 0x4b, 0x52, 0xad, 0x94, 0x8a, 0x52, 0x0b,
0x4b, 0x33, 0x8b, 0x52, 0x53, 0x94, 0x82, 0xc0, 0x1a, 0x8c, 0x5c, 0xb9, 0x58, 0x5c, 0x52, 0x73,
0xf3, 0x85, 0x6c, 0xb9, 0x38, 0x82, 0x13, 0x2b, 0xc1, 0xe6, 0x09, 0x49, 0xea, 0xa1, 0x39, 0x5b,
0x0f, 0x66, 0x8f, 0x94, 0x98, 0x1e, 0xc4, 0x45, 0x7a, 0x30, 0x17, 0xe9, 0xb9, 0x82, 0x5c, 0xe4,
0x24, 0x7a, 0xe2, 0xa1, 0x1c, 0xc3, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78,
0x24, 0xc7, 0x18, 0xc5, 0x9c, 0x58, 0x90, 0x99, 0xc4, 0x06, 0x56, 0x66, 0x0c, 0x08, 0x00, 0x00,
0xff, 0xff, 0x7d, 0xee, 0x27, 0xd0, 0x29, 0x01, 0x00, 0x00,
}

View File

@@ -0,0 +1,29 @@
// 定义项目 API 的 proto 文件 可以同时描述 gRPC 和 HTTP API
// protobuf 文件参考:
// - https://developers.google.com/protocol-buffers/
// - http://info.bilibili.co/display/documentation/gRPC+Proto
// protobuf 生成 HTTP 工具:
// - http://git.bilibili.co/platform/go-common/tree/master/app/tool/protoc-gen-bm
syntax = "proto3";
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "google/protobuf/empty.proto";
// package 命名使用 {discovery_id}.{version} 的方式, version 形如 v1, v2, v1beta ..
// NOTE: 不知道的 discovery_id 请询问大佬, 新项目找大佬申请 discovery_id先到先得抢注
// e.g. account.service.v1
package demo.service.v1;
// NOTE: 最后请删除这些无用的注释 (゜-゜)つロ
option go_package = "api";
// do not generate getXXX() method
option (gogoproto.goproto_getters_all) = false;
service Demo {
rpc SayHello (HelloReq) returns (.google.protobuf.Empty);
}
message HelloReq {
string name = 1 [(gogoproto.moretags)='form:"name" validate:"required"'];
}

View File

@@ -0,0 +1,4 @@
package api
// 生成 gRPC 代码
//go:generate $GOPATH/src/go-common/app/tool/warden/protoc.sh

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["test.toml"],
importpath = "go-common/app/job/live/gift/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/gift/internal/conf:go_default_library",
"//app/job/live/gift/internal/server/http:go_default_library",
"//app/job/live/gift/internal/service:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,47 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/live/gift/internal/conf"
"go-common/app/job/live/gift/internal/server/http"
"go-common/app/job/live/gift/internal/service"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("gift-job start")
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
svc := service.New(conf.Conf)
http.Init(conf.Conf, svc)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
svc.Close()
log.Info("gift-job exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,70 @@
[mysql]
addr = "172.22.34.101:3312"
#dsn = "livetestuat:livetestuat20180711@tcp(172.22.34.101:3312)/live-ugift?timeout=200ms&readTimeout=200ms&writeTimeout=200ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
dsn = "live:oWni@ElNs0P0C(dphdj*F1y4@tcp(172.16.38.117:3312)/live-ugift?timeout=400ms&readTimeout=400ms&writeTimeout=400ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
active = 20
idle = 10
idleTimeout ="4h"
queryTimeout = "800ms"
execTimeout = "800ms"
tranTimeout = "800ms"
[redis]
name = "gift-job"
proto = "tcp"
addr = "127.0.0.1:6379"
idle = 10
active = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1m"
[memcache]
name = "gift-job"
proto = "tcp"
addr = ""
active = 50
idle = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"
[databus]
[databus.AddGift]
key = "ec4c0820d525d67b"
secret = "e20f8f664bf10722efeb6aac0cc16011"
group = "LiveAddGift-LiveLive-S"
topic = "LiveAddGift-T"
action = "sub"
name = "gift-job/addFreeGift"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 10
active = 10
dialTimeout = "10s"
readTimeout = "33s"
writeTimeout = "10s"
idleTimeout = "10s"
[consumer]
[consumer.AddGift]
num = 100
[infoc]
[infoc.bagLog]
taskID = "000736"
proto = "tcp"
addr = "172.18.33.124:15140"
chanSize = 10240
[infoc.giftAction]
taskID = "000579"
proto = "tcp"
addr = "172.18.33.124:15140"
chanSize = 10240
[log]
stdout = true

View File

@@ -0,0 +1,42 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/gift/internal/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/log/infoc:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,107 @@
package conf
import (
"errors"
"flag"
"go-common/library/log/infoc"
"go-common/library/queue/databus"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/conf"
"go-common/library/database/sql"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Redis *redis.Config
Memcache *memcache.Config
MySQL *sql.Config
Ecode *ecode.Config
Databus *Databus
Infoc map[string]*infoc.Config
Consumer *consumer
}
type consumer struct {
AddGift *consumeConfig
}
type consumeConfig struct {
Num int
}
//Databus Databus
type Databus struct {
AddGift *databus.Config
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,41 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"mysql.go",
"redis.go",
],
importpath = "go-common/app/job/live/gift/internal/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/gift/internal/conf:go_default_library",
"//app/job/live/gift/internal/model:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/database/sql:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/satori/go.uuid:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,42 @@
package dao
import (
"context"
"go-common/app/job/live/gift/internal/conf"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
xsql "go-common/library/database/sql"
)
// Dao dao
type Dao struct {
c *conf.Config
mc *memcache.Pool
redis *redis.Pool
db *xsql.DB
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
mc: memcache.NewPool(c.Memcache),
redis: redis.NewPool(c.Redis),
db: xsql.NewMySQL(c.MySQL),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.mc.Close()
d.redis.Close()
d.db.Close()
}
// Ping dao ping
func (d *Dao) Ping(ctx context.Context) error {
// TODO: add mc,redis... if you use
return d.db.Ping(ctx)
}

View File

@@ -0,0 +1,79 @@
package dao
import (
"context"
"crypto/md5"
"database/sql"
"encoding/hex"
"fmt"
"go-common/app/job/live/gift/internal/model"
"go-common/library/log"
"strconv"
)
var (
_getBag = "SELECT id,gift_num FROM user_gift_%s WHERE uid = ? AND gift_id = ? AND expireat = ? LIMIT 1"
_getBagByID = "SELECT id,gift_num FROM user_gift_%s WHERE id = ?"
_updateBagNum = "UPDATE user_gift_%s SET gift_num = gift_num + ? WHERE id = ?"
_insertBag = "INSERT INTO user_gift_%s (uid,gift_id,gift_num,expireat) VALUES (?,?,?,?)"
)
// GetBag GetBag
func (d *Dao) GetBag(ctx context.Context, uid, giftID, expireAt int64) (res *model.BagInfo, err error) {
log.Info("GetBag,uid:%d,giftID:%d,expireAt:%d", uid, giftID, expireAt)
row := d.db.QueryRow(ctx, fmt.Sprintf(_getBag, getPostFix(uid)), uid, giftID, expireAt)
res = &model.BagInfo{}
if err = row.Scan(&res.ID, &res.GiftNum); err != nil {
if err == sql.ErrNoRows {
err = nil
return
}
log.Error("GetBag row.Scan error(%v)", err)
}
return
}
// UpdateBagNum UpdateBagNum
func (d *Dao) UpdateBagNum(ctx context.Context, uid, id, num int64) (affected int64, err error) {
log.Info("UpdateBagNum,uid:%d,id:%d,num:%d", uid, id, num)
res, err := d.db.Exec(ctx, fmt.Sprintf(_updateBagNum, getPostFix(uid)), num, id)
if err != nil {
log.Error("UpdateBagNum error(%v)", err)
return
}
return res.RowsAffected()
}
// AddBag AddBag
func (d *Dao) AddBag(ctx context.Context, uid, giftID, giftNum, expireAt int64) (affected int64, err error) {
log.Info("AddBag,uid:%d,giftID:%d,giftNum:%d,expireAt:%d", uid, giftID, giftNum, expireAt)
res, err := d.db.Exec(ctx, fmt.Sprintf(_insertBag, getPostFix(uid)), uid, giftID, giftNum, expireAt)
if err != nil {
log.Error("AddBag error(%v)", err)
return
}
return res.LastInsertId()
}
// GetBagByID GetBagByID
func (d *Dao) GetBagByID(ctx context.Context, uid, id int64) (res *model.BagInfo, err error) {
log.Info("GetBagByID,uid:%d,id:%d", uid, id)
row := d.db.QueryRow(ctx, fmt.Sprintf(_getBagByID, getPostFix(uid)), id)
res = &model.BagInfo{}
if err = row.Scan(&res.ID, &res.GiftNum); err != nil {
if err == sql.ErrNoRows {
err = nil
return
}
log.Error("GetBagByID row.Scan error(%v)", err)
}
return
}
func getPostFix(uid int64) string {
uidStr := strconv.Itoa(int(uid))
h := md5.New()
h.Write([]byte(uidStr))
md5Str := hex.EncodeToString(h.Sum(nil))
return md5Str[0:1]
}

View File

@@ -0,0 +1,110 @@
package dao
import (
"context"
"fmt"
"go-common/library/cache/redis"
"go-common/library/log"
"time"
"github.com/satori/go.uuid"
)
func bagIDCache(uid, giftID, expireAt int64) string {
return fmt.Sprintf("bag_id:%d:%d:%d", uid, giftID, expireAt)
}
// SetBagIDCache SetBagIDCache
func (d *Dao) SetBagIDCache(ctx context.Context, uid, giftID, expireAt, bagID, expire int64) (err error) {
key := bagIDCache(uid, giftID, expireAt)
conn := d.redis.Get(ctx)
defer conn.Close()
_, err = conn.Do("SETEX", key, expire, bagID)
if err != nil {
log.Error("conn.Do(SETEX, %s) error(%v)", key, err)
}
return
}
func bagListKey(uid int64) string {
return fmt.Sprintf("bag_list:%d", uid)
}
// ClearBagListCache ClearBagListCache
func (d *Dao) ClearBagListCache(ctx context.Context, uid int64) (err error) {
key := bagListKey(uid)
conn := d.redis.Get(ctx)
defer conn.Close()
_, err = conn.Do("DEL", key)
if err != nil {
log.Error("conn.Do(DEL, %s) error(%v)", key, err)
}
return
}
// GetBagIDCache GetBagIDCache
func (d *Dao) GetBagIDCache(ctx context.Context, uid, giftID, expireAt int64) (bagID int64, err error) {
key := bagIDCache(uid, giftID, expireAt)
conn := d.redis.Get(ctx)
defer conn.Close()
bagID, err = redis.Int64(conn.Do("GET", key))
if err != nil {
if err == redis.ErrNil {
err = nil
} else {
log.Error("conn.Do(GET, %s) error(%v)", key, err)
}
return
}
return
}
func bagNumKey(uid, giftID, expireAt int64) string {
return fmt.Sprintf("bag_num:%d:%d:%d", uid, giftID, expireAt)
}
// SetBagNumCache SetBagNumCache
func (d *Dao) SetBagNumCache(ctx context.Context, uid, giftID, expireAt, giftNum, expire int64) (err error) {
key := bagNumKey(uid, giftID, expireAt)
conn := d.redis.Get(ctx)
defer conn.Close()
_, err = conn.Do("SETEX", key, expire, giftNum)
if err != nil {
log.Error("conn.Do(SETEX, %s) error(%v)", key, err)
}
return
}
//Lock Lock
func (d *Dao) Lock(ctx context.Context, key string, ttl int, retry int, retryDelay int) (gotLock bool, lockValue string, err error) {
if retry <= 0 {
retry = 1
}
lockValue = uuid.NewV4().String()
retryTimes := 0
conn := d.redis.Get(ctx)
defer conn.Close()
realKey := lockKey(key)
for ; retryTimes < retry; retryTimes++ {
var res interface{}
res, err = conn.Do("SET", realKey, lockValue, "PX", ttl, "NX")
if err != nil {
log.Error("redis_lock failed:%s:%v", realKey, err)
break
}
if res != nil {
gotLock = true
break
}
time.Sleep(time.Duration(retryDelay) * time.Millisecond)
}
return
}
func lockKey(key string) string {
return fmt.Sprintf("gift_job_lock:%s", key)
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/job/live/gift/internal/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,17 @@
package model
//AddFreeGift AddFreeGift
type AddFreeGift struct {
UID int64 `json:"uid"`
GiftID int64 `json:"gift_id"`
GiftNum int64 `json:"gift_num"`
ExpireAt int64 `json:"expire_at"`
Source string `json:"source"`
MsgID string `json:"msg_id"`
}
// BagInfo BagInfo
type BagInfo struct {
ID int64 `json:"id"`
GiftNum int64 `json:"gift_num"`
}

View File

@@ -0,0 +1,35 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/live/gift/internal/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/gift/internal/conf:go_default_library",
"//app/job/live/gift/internal/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,53 @@
package http
import (
"net/http"
"go-common/app/job/live/gift/internal/conf"
"go-common/app/job/live/gift/internal/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
vfy *verify.Verify
svc *service.Service
)
// Init init
func Init(c *conf.Config, s *service.Service) {
svc = s
vfy = verify.New(c.Verify)
engine := bm.DefaultServer(c.BM)
route(engine)
if err := engine.Start(); err != nil {
log.Error("bm Start error(%v)", err)
panic(err)
}
}
func route(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/x/gift")
{
g.GET("/start", vfy.Verify, howToStart)
}
}
func ping(ctx *bm.Context) {
if err := svc.Ping(ctx); err != nil {
log.Error("ping error(%v)", err)
ctx.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}
// example for http request handler
func howToStart(c *bm.Context) {
c.String(0, "Golang 大法好 !!!")
}

View File

@@ -0,0 +1,60 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"infoc.go",
"package.go",
"service.go",
],
importpath = "go-common/app/job/live/gift/internal/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/gift/api:go_default_library",
"//app/job/live/gift/internal/conf:go_default_library",
"//app/job/live/gift/internal/dao:go_default_library",
"//app/job/live/gift/internal/model:go_default_library",
"//library/log:go_default_library",
"//library/log/infoc:go_default_library",
"//library/queue/databus:go_default_library",
"//library/sync/errgroup:go_default_library",
"@io_bazel_rules_go//proto/wkt:empty_go_proto",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"infoc_test.go",
"package_test.go",
"service_test.go",
],
embed = [":go_default_library"],
tags = ["automanaged"],
deps = [
"//app/job/live/gift/internal/conf:go_default_library",
"//app/job/live/gift/internal/model:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@@ -0,0 +1,121 @@
package service
import (
"go-common/library/log"
"go-common/library/log/infoc"
"strconv"
"time"
)
var inCh = make(chan interface{}, 10240)
const maxInt = int(^uint(0) >> 1)
type bagLogInfoc struct {
id string
uid string
bagID string
giftID string
num string
afterNum string
source string
infoType string
ctime string
}
type giftActionInfoc struct {
uid int64
roomid int64
item int64
value int64
change int64
describe string
extra string
ts int64
platform string
clientver string
buvid string
ua string
referer string
}
// bagLogInfoc 包裹日志打点
func (s *Service) bagLogInfoc(uid, bagID, giftID, num, afterNum int64, source string) {
s.infoc(bagLogInfoc{
id: MakeID(uid),
uid: strconv.FormatInt(uid, 10),
bagID: strconv.FormatInt(bagID, 10),
giftID: strconv.FormatInt(giftID, 10),
num: strconv.FormatInt(num, 10),
afterNum: strconv.FormatInt(afterNum, 10),
source: source,
infoType: "1",
ctime: time.Now().Format("2006-01-02 15:04:05"),
})
}
//giftActionInfoc 道具打点
func (s *Service) giftActionInfoc(uid, roomid, item, value, change int64, describe, platform string) {
s.infoc(giftActionInfoc{
uid: uid,
roomid: roomid,
item: item,
value: value,
change: change,
describe: describe,
extra: "",
ts: time.Now().Unix(),
platform: platform,
clientver: "",
buvid: "",
ua: "",
referer: "",
})
}
// MakeID MakeID
func MakeID(uid int64) string {
prefix := strconv.FormatInt(uid%10, 10)
postfix := strconv.Itoa(maxInt - int(time.Now().Unix()*10000))
uidStr := strconv.FormatInt(uid, 10)
l := len(uidStr)
var middle string
if l >= 10 {
middle = uidStr
} else {
var s string
for i := 0; i < (10 - l); i++ {
s += "0"
}
middle = s + uidStr
}
return prefix + middle + postfix
}
//infoc
func (s *Service) infoc(i interface{}) {
select {
case inCh <- i:
default:
log.Warn("infocproc chan full")
}
}
// infocproc
func (s *Service) infocproc() {
var bl = infoc.New(s.c.Infoc["bagLog"])
var ga = infoc.New(s.c.Infoc["giftAction"])
for {
i := <-inCh
switch v := i.(type) {
case bagLogInfoc:
err := bl.Info(v.id, v.uid, v.bagID, v.giftID, v.num, v.afterNum, v.source, v.infoType, v.ctime)
log.Info("bagLogInfoc info %v,ret:%v", v, err)
case giftActionInfoc:
err := ga.Info(v.uid, v.roomid, v.item, v.value, v.change, v.describe, v.extra, v.ts, v.platform, v.clientver, v.buvid, v.ua, v.referer)
log.Info("giftActionInfoc info %v,ret:%v", v, err)
default:
log.Warn("infocproc can't process the type")
}
}
}

View File

@@ -0,0 +1,58 @@
package service
import (
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServicebagLogInfoc(t *testing.T) {
convey.Convey("bagLogInfoc", t, func(ctx convey.C) {
var (
uid = int64(0)
bagID = int64(0)
giftID = int64(0)
num = int64(0)
afterNum = int64(0)
source = ""
)
ctx.Convey("When everything gose positive", func(ctx convey.C) {
s.bagLogInfoc(uid, bagID, giftID, num, afterNum, source)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}
func TestServicegiftActionInfoc(t *testing.T) {
convey.Convey("giftActionInfoc", t, func(ctx convey.C) {
var (
uid = int64(0)
roomid = int64(0)
item = int64(0)
value = int64(0)
change = int64(0)
describe = ""
platform = ""
)
ctx.Convey("When everything gose positive", func(ctx convey.C) {
s.giftActionInfoc(uid, roomid, item, value, change, describe, platform)
ctx.Convey("No return values", func(ctx convey.C) {
})
})
})
}
func TestServiceMakeID(t *testing.T) {
convey.Convey("MakeID", t, func(ctx convey.C) {
var (
uid = int64(0)
)
ctx.Convey("When everything gose positive", func(ctx convey.C) {
p1 := MakeID(uid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
})
}

View File

@@ -0,0 +1,99 @@
package service
import (
"context"
"errors"
"go-common/app/job/live/gift/internal/model"
"go-common/library/log"
"go-common/library/sync/errgroup"
)
// AddGift AddGift
func (s *Service) AddGift(ctx context.Context, m *model.AddFreeGift) (bagId int64, err error) {
uid := m.UID
giftID := m.GiftID
giftNum := m.GiftNum
expireAt := m.ExpireAt
source := m.Source
if uid == 0 || giftID == 0 || giftNum == 0 {
log.Error("add gift params error,uid:%d,giftID:%d,giftNum:%d", uid, giftID, giftNum)
err = errors.New("params error")
return
}
bagID, err := s.GetBagID(ctx, uid, giftID, expireAt)
if err != nil {
return
}
var (
affectNum int64
isUpdate = false
eg, _ = errgroup.WithContext(ctx)
)
if bagID != 0 {
isUpdate = true
affectNum, _ = s.dao.UpdateBagNum(ctx, uid, bagID, giftNum)
} else {
affectNum, _ = s.dao.AddBag(ctx, uid, giftID, giftNum, expireAt)
bagID = affectNum
eg.Go(
func() error {
s.dao.SetBagIDCache(ctx, uid, giftID, expireAt, bagID, 14400)
return nil
})
}
newNum := giftNum
if affectNum > 0 {
eg.Go(
func() error {
s.dao.ClearBagListCache(ctx, uid)
return nil
})
if isUpdate {
res, _ := s.dao.GetBagByID(ctx, uid, bagID)
newNum = res.GiftNum
//上报lancer TODO
s.bagLogInfoc(uid, bagID, giftID, giftNum, newNum, source)
}
}
// 更新免费礼物数量缓存
eg.Go(
func() error {
s.UpdateFreeGiftCache(ctx, uid, giftID, expireAt, newNum)
return nil
})
eg.Wait()
return
}
// GetBagID GetBagID
func (s *Service) GetBagID(ctx context.Context, uid, giftID, expireAt int64) (id int64, err error) {
id, err = s.dao.GetBagIDCache(ctx, uid, giftID, expireAt)
if err != nil {
return
}
if id == 0 {
//queryDB
var r *model.BagInfo
r, err = s.dao.GetBag(ctx, uid, giftID, expireAt)
if err != nil {
return
}
id = r.ID
}
// 缓存或数据库本身有,再更新缓存
if id != 0 {
s.dao.SetBagIDCache(ctx, uid, giftID, expireAt, id, 14400)
}
return
}
// UpdateFreeGiftCache UpdateFreeGiftCache
func (s *Service) UpdateFreeGiftCache(ctx context.Context, uid, giftID, expireAt, num int64) {
//giftInfo := s.GetGiftInfoByID(ctx, giftID)
//if giftInfo.Id == 0 || giftInfo.Type != 3 {
// return
//}
//s.dao.SetBagNumCache(ctx, uid, giftID, expireAt, num, 14400)
}

View File

@@ -0,0 +1,66 @@
package service
import (
"context"
"go-common/app/job/live/gift/internal/model"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestServiceAddGift(t *testing.T) {
convey.Convey("AddGift", t, func(c convey.C) {
var (
ctx = context.Background()
m = &model.AddFreeGift{
UID: 1,
GiftID: 1,
GiftNum: 1,
ExpireAt: 0,
Source: "test",
}
)
c.Convey("When everything gose positive", func(c convey.C) {
bagId, err := s.AddGift(ctx, m)
c.Convey("Then err should be nil.bagId should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(bagId, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceGetBagID(t *testing.T) {
convey.Convey("GetBagID", t, func(c convey.C) {
var (
ctx = context.Background()
uid = int64(1)
giftID = int64(1)
expireAt = int64(0)
)
c.Convey("When everything gose positive", func(c convey.C) {
id, err := s.GetBagID(ctx, uid, giftID, expireAt)
c.Convey("Then err should be nil.id should not be nil.", func(c convey.C) {
c.So(err, convey.ShouldBeNil)
c.So(id, convey.ShouldNotBeNil)
})
})
})
}
func TestServiceUpdateFreeGiftCache(t *testing.T) {
convey.Convey("UpdateFreeGiftCache", t, func(c convey.C) {
var (
ctx = context.Background()
uid = int64(0)
giftID = int64(0)
expireAt = int64(0)
num = int64(0)
)
c.Convey("When everything gose positive", func(c convey.C) {
s.UpdateFreeGiftCache(ctx, uid, giftID, expireAt, num)
c.Convey("No return values", func(ctx convey.C) {
})
})
})
}

View File

@@ -0,0 +1,95 @@
package service
import (
"context"
"encoding/json"
"fmt"
"sync"
"go-common/app/job/live/gift/internal/model"
"go-common/library/log"
"go-common/library/queue/databus"
pb "go-common/app/job/live/gift/api"
"go-common/app/job/live/gift/internal/conf"
"go-common/app/job/live/gift/internal/dao"
"github.com/golang/protobuf/ptypes/empty"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
addFreeGift *databus.Databus
waiter sync.WaitGroup
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
addFreeGift: databus.New(c.Databus.AddGift),
}
go s.infocproc()
for i := 0; i < c.Consumer.AddGift.Num; i++ {
s.waiter.Add(1)
go s.addGiftConsumeProc()
}
return s
}
func (s *Service) addGiftConsumeProc() {
defer s.waiter.Done()
var err error
for {
msg, ok := <-s.addFreeGift.Messages()
if !ok {
log.Error("s.addFreeGift.Messages channel closed")
return
}
m := &model.AddFreeGift{}
if err = json.Unmarshal(msg.Value, &m); err != nil {
log.Error("json.Unmarshal(%v) error(%v)", msg, err)
continue
}
ctx := context.Background()
// 消息幂等
if m.MsgID != "" {
key := m.MsgID + m.Source
gotLock, _, errLock := s.dao.Lock(ctx, key, 3600000, 0, 0)
if errLock != nil {
continue
}
if !gotLock {
log.Error("msg has been processed,%v", m)
continue
}
}
s.AddGift(ctx, m)
// 打点上报
s.giftActionInfoc(m.UID, 0, m.GiftID, 0, m.GiftNum, m.Source, "")
log.Info("consume addFreeGift topic:%s, Key:%s, Value:%s ", msg.Topic, msg.Key, msg.Value)
if err = msg.Commit(); err != nil {
log.Error("commit msg(%v) error(%v)", msg, err)
}
}
}
// SayHello grpc demo func
func (s *Service) SayHello(ctx context.Context, req *pb.HelloReq) (reply *empty.Empty, err error) {
reply = new(empty.Empty)
fmt.Printf("hello %s", req.Name)
return
}
// Ping Service
func (s *Service) Ping(ctx context.Context) (err error) {
return s.dao.Ping(ctx)
}
// Close Service
func (s *Service) Close() {
s.dao.Close()
}

View File

@@ -0,0 +1,34 @@
package service
import (
"flag"
"go-common/app/job/live/gift/internal/conf"
"os"
"testing"
)
var (
s *Service
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "")
flag.Set("conf_token", "")
flag.Set("tree_id", "")
flag.Set("conf_version", "docker-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../../cmd/test.toml")
}
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
s = New(conf.Conf)
os.Exit(m.Run())
}

View File

@@ -0,0 +1,9 @@
# v1.0.2
1. 用户信息 multiV3 切换为 account
# v1.0.1
1. waitgroup
# v1.0.0
1. 上线功能xxx

View File

@@ -0,0 +1,6 @@
# Owner
fuyu01
# Author
# Reviewer

View File

@@ -0,0 +1,14 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- fuyu01
labels:
- job
- job/live/push-search
- live
options:
no_parent_owners: true
reviewers:
- liuzhen
- xiehaishen
- yangbaibing

View File

@@ -0,0 +1,12 @@
# push-search-service
# 项目简介
1.
# 编译环境
# 依赖包
# 编译执行

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["push-search-test.toml"],
importpath = "go-common/app/job/live/push-search/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//app/job/live/push-search/http:go_default_library",
"//app/job/live/push-search/service/migrate:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,67 @@
package main
import (
"flag"
"go-common/library/net/trace"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/live/push-search/conf"
"go-common/app/job/live/push-search/http"
"go-common/app/job/live/push-search/service/migrate"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("push-search-service start")
if len(os.Args[1:]) > 0 && os.Args[1:][0] == "migrate" {
roomId := os.Args[1:][1]
isTest := os.Args[1:][2]
ms := migrate.NewMigrateS(conf.Conf)
go ms.Migrate(roomId, isTest)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
m := <-c
switch m {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
ms.Close()
log.Info("push-search-service-migrate exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
http.Init(conf.Conf)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
http.Srv.Close()
log.Info("push-search-service exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,97 @@
[log]
stdout=true
[group]
[group.RoomInfo]
num = 32
chan = 1024
[group.Attention]
num = 32
chan = 1024
[group.UserInfo]
num = 32
chan = 1024
[databus]
[databus.PushSearch]
key = "ec4c0820d525d67b"
secret= "e20f8f664bf10722efeb6aac0cc16011"
group= "RoomUpdateToSearch-LiveLive-P"
topic= "RoomUpdateToSearch-T"
action="pub"
name = "live-search"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "10s"
readTimeout = "40s"
writeTimeout = "10s"
idleTimeout = "60s"
[databus.RoomInfo]
key = "ec4c0820d525d67b"
secret= "e20f8f664bf10722efeb6aac0cc16011"
group= "ApRoomBinlog-LiveLive-S"
topic= "ApRoomBinlog-T"
action="sub"
name = "live-roomInfo-change"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "10s"
readTimeout = "40s"
writeTimeout = "10s"
idleTimeout = "60s"
[databus.Attention]
key = "ec4c0820d525d67b"
secret= "e20f8f664bf10722efeb6aac0cc16011"
group= "LiveRelationChanged-LiveLive-S"
topic= "LiveRelationChanged-T"
action="sub"
name = "live-attention-change"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "10s"
readTimeout = "40s"
writeTimeout = "10s"
idleTimeout = "60s"
[databus.UserName]
key = "ec4c0820d525d67b"
secret= "e20f8f664bf10722efeb6aac0cc16011"
group= "BannedUserSyn-LiveLive-Search-S"
topic= "BannedUserSyn-T"
action="sub"
name = "live-userName-change"
proto = "tcp"
addr = "172.18.33.50:6205"
idle = 100
active = 100
dialTimeout = "10s"
readTimeout = "40s"
writeTimeout = "10s"
idleTimeout = "60s"
[SearchHBase]
master = ""
meta = ""
dialTimeout = "1s"
readTimeout = "2s"
readsTimeout = "5s"
writeTimeout = "2s"
writesTimeout = "5s"
[SearchHBase.zookeeper]
root = ""
addrs = ["172.18.33.131:2181","172.18.33.168:2181","172.18.33.169:2181"]
timeout = "30s"
[liverpc]
[liverpc.room]
AppID = "live.room"
ConnTimeout = "50ms"
[liverpc.relation]
AppID = "live.relation"
ConnTimeout = "50ms"
[liverpc.user]
AppID = "live.user"
ConnTimeout = "50ms"

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/push-search/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/database/sql:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/rpc/liverpc:go_default_library",
"//library/net/trace:go_default_library",
"//library/queue/databus:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,126 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/cache/redis"
"go-common/library/conf"
"go-common/library/database/sql"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
"go-common/library/queue/databus"
"go-common/library/database/hbase.v2"
"go-common/library/net/rpc/liverpc"
xtime "go-common/library/time"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Redis *redis.Config
Memcache *memcache.Config
MySQL *sql.Config
Ecode *ecode.Config
LiveRpc map[string]*liverpc.ClientConfig
//DataBus
DataBus *DataBus
Group *Group
SearchHBase *hbaseConf
MigrateNum int
}
type DataBus struct {
RoomInfo *databus.Config
Attention *databus.Config
UserName *databus.Config
PushSearch *databus.Config
}
// Group group.
type Group struct {
RoomInfo *GroupConf
Attention *GroupConf
UserInfo *GroupConf
}
// GroupConf group conf.
type GroupConf struct {
Num int
Chan int
}
type hbaseConf struct {
hbase.Config
ReadTimeout xtime.Duration
ReadsTimeout xtime.Duration
WriteTimeout xtime.Duration
WritesTimeout xtime.Duration
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,45 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"api.go",
"dao.go",
"pub.go",
],
importpath = "go-common/app/job/live/push-search/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//app/service/live/relation/api/liverpc:go_default_library",
"//app/service/live/room/api/liverpc:go_default_library",
"//app/service/live/user/api/liverpc:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/log:go_default_library",
"//library/net/rpc/liverpc:go_default_library",
"//library/queue/databus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/push-search/dao/migrate:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,28 @@
package dao
import (
"go-common/app/job/live/push-search/conf"
userApi "go-common/app/service/live/user/api/liverpc"
relationApi "go-common/app/service/live/relation/api/liverpc"
roomApi "go-common/app/service/live/room/api/liverpc"
"go-common/library/net/rpc/liverpc"
)
var UserApi *userApi.Client
var RelationApi *relationApi.Client
var RoomApi *roomApi.Client
// InitAPI init all service APIs
func InitAPI() {
UserApi = userApi.New(getConf("user"))
RelationApi = relationApi.New(getConf("relation"))
RoomApi = roomApi.New(getConf("room"))
}
func getConf(appName string) *liverpc.ClientConfig {
c := conf.Conf.LiveRpc
if c != nil {
return c[appName]
}
return nil
}

View File

@@ -0,0 +1,46 @@
package dao
import (
"context"
"go-common/app/job/live/push-search/conf"
"go-common/library/queue/databus"
"go-common/library/database/hbase.v2"
)
// Dao dao
type Dao struct {
c *conf.Config
RoomInfoDataBus *databus.Databus
AttentionDataBus *databus.Databus
UserNameDataBus *databus.Databus
PushSearchDataBus *databus.Databus
SearchHBase *hbase.Client
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
RoomInfoDataBus: databus.New(c.DataBus.RoomInfo),
AttentionDataBus: databus.New(c.DataBus.Attention),
UserNameDataBus: databus.New(c.DataBus.UserName),
PushSearchDataBus: databus.New(c.DataBus.PushSearch),
SearchHBase: hbase.NewClient(&c.SearchHBase.Config),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.RoomInfoDataBus.Close()
d.AttentionDataBus.Close()
d.UserNameDataBus.Close()
return
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) error {
// TODO: if you need use mc,redis, please add
return nil
}

View File

@@ -0,0 +1,33 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["dao.go"],
importpath = "go-common/app/job/live/push-search/dao/migrate",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//library/database/hbase.v2:go_default_library",
"//library/database/sql:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,37 @@
package migrate
import (
"context"
"go-common/app/job/live/push-search/conf"
"go-common/library/database/hbase.v2"
"go-common/library/database/sql"
)
// Dao dao
type Dao struct {
c *conf.Config
SearchHBase *hbase.Client
RoomDb *sql.DB
}
// New init mysql db
func NewMigrate(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
SearchHBase: hbase.NewClient(&c.SearchHBase.Config),
RoomDb: sql.NewMySQL(c.MySQL),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.RoomDb.Close()
return
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) error {
// TODO: if you need use mc,redis, please add
return nil
}

View File

@@ -0,0 +1,16 @@
package dao
import (
"context"
"go-common/library/log"
"strconv"
)
func (d *Dao) Pub(c context.Context, uid int64, msg interface{}) error {
key := strconv.FormatInt(uid, 10)
err := d.PushSearchDataBus.Send(c, key, msg)
if err != nil {
log.Error("pub wallet change failed uid:%d, msg:%+v", uid, msg)
}
return err
}

View File

@@ -0,0 +1,35 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["http.go"],
importpath = "go-common/app/job/live/push-search/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//app/job/live/push-search/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,53 @@
package http
import (
"net/http"
"go-common/app/job/live/push-search/conf"
"go-common/app/job/live/push-search/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
Srv *service.Service
vfy *verify.Verify
)
// Init init
func Init(c *conf.Config) {
Srv = service.New(c)
vfy = verify.New(c.Verify)
engine := bm.DefaultServer(c.BM)
route(engine)
if err := engine.Start(); err != nil {
log.Error("bm Start error(%v)", err)
panic(err)
}
}
func route(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/x/push-search")
{
g.GET("/start", vfy.Verify, howToStart)
}
}
func ping(c *bm.Context) {
if err := Srv.Ping(c); err != nil {
log.Error("ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}
// example for http request handler
func howToStart(c *bm.Context) {
c.String(0, "Golang 大法好 !!!")
}

View File

@@ -0,0 +1,31 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"model.go",
"room_info.go",
],
importpath = "go-common/app/job/live/push-search/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1 @@
package model

View File

@@ -0,0 +1,75 @@
package model
import (
"encoding/json"
)
// NotifyInfo notify info.
type ApRoomNotifyInfo struct {
Action string `json:"action"`
Table string `json:"table"`
New json.RawMessage `json:"new"`
Old json.RawMessage `json:"old"`
}
type LiveDatabusAttention struct {
Topic string `json:"topic"`
MsgId string `json:"msg_id"`
MsgContent *AttentionNotifyInfo `json:"msg_content"`
}
// NotifyInfo notify info.
type AttentionNotifyInfo struct {
Uid int64 `json:"uid"`
UpUid int64 `json:"up_uid"`
ExtInfo *ExInfo `json:"ext_info"`
}
type ExInfo struct {
UpUidFans int `json:"up_uid_fans"`
}
type LiveDatabus struct {
Topic string `json:"topic"`
MsgId string `json:"msg_id"`
MsgContent string `json:"msg_content"`
}
type UnameNotifyInfo struct{
Uid int64 `json:"uid"`
Uname string `json:"uname"`
Identification int `json:"identification"`
}
type TableField struct {
RoomId int `json:"roomid"`
ShortId int `json:"short_id"`
Uid int64 `json:"uid"`
UName string `json:"uname"`
Area int `json:"area"`
Title string `json:"title"`
Tag string `json:"tags"`
MTime string `json:"mtime"`
CTime string `json:"ctime"`
TryTime string `json:"try_time"`
Cover string `json:"cover"`
UserCover string `json:"user_cover"`
LockStatus string `json:"lock_status"`
HiddenStatus string `json:"hidden_status"`
Attentions int `json:"attentions"`
Online int `json:"online"`
LiveTime string `json:"live_time"`
AreaV2Id int `json:"area_v2_id"`
AreaV2Name string `json:"area_v2_name"`
AreaV2ParentId int `json:"area_v2_parent_id"`
Virtual int `json:"virtual"`
RoundStatus int `json:"round_status"`
OnFlag int `json:"on_flag"`
}
type DataMap struct {
Action string
Table string
New *TableField
Old *TableField
}

View File

@@ -0,0 +1,51 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"attention_notify.go",
"common.go",
"roominfo_notify.go",
"service.go",
"uname_notify.go",
],
importpath = "go-common/app/job/live/push-search/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//app/job/live/push-search/dao:go_default_library",
"//app/job/live/push-search/model:go_default_library",
"//app/service/live/relation/api/liverpc/v1:go_default_library",
"//app/service/live/room/api/liverpc/v1:go_default_library",
"//app/service/live/room/api/liverpc/v2:go_default_library",
"//app/service/live/user/api/liverpc/v3:go_default_library",
"//app/service/main/account/api:go_default_library",
"//library/log:go_default_library",
"//library/net/rpc/liverpc/context:go_default_library",
"//library/queue/databus:go_default_library",
"//library/sync/errgroup:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/push-search/service/migrate:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,154 @@
package service
import (
"context"
"encoding/json"
"go-common/app/job/live/push-search/model"
"go-common/library/log"
"go-common/library/sync/errgroup"
"strconv"
)
const (
_retry = 3
)
func (s *Service) attentionNotifyConsumeProc() {
defer s.waiter.Done()
for {
msg, ok := <-s.dao.AttentionDataBus.Messages()
if !ok {
log.Error("attentionNotifyConsumeProc closed")
if err := s.dao.AttentionDataBus.Close(); err != nil {
log.Error("s.dao.AttentionDataBus.Close() error(%v)", err)
}
return
}
m := &message{data: msg}
p := new(model.LiveDatabusAttention)
if err := json.Unmarshal(msg.Value, p); err != nil {
msg.Commit()
log.Error("[AttentionDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
continue
}
if p.MsgContent == nil {
log.Error("[AttentionDataBus]attentionNotifyConsumeProc msg object msgContent is nil, msg:%+v", string(msg.Value))
return
}
m.object = p
s.attentionMergeChan[p.MsgContent.UpUid%int64(s.c.Group.Attention.Num)] <- m
}
}
func (s *Service) attentionNotifyHandleProc(c chan *message) {
defer s.waiterChan.Done()
for {
msgData, ok := <-c
if !ok {
log.Error("[AttentionDataBus]attentionNotifyHandleProc closed")
return
}
//先提交防止阻塞,关闭时等待任务执行完
msgData.data.Commit()
p, assertOk := msgData.object.(*model.LiveDatabusAttention)
if !assertOk {
log.Error("[AttentionDataBus]attentionNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
return
}
uid := p.MsgContent.UpUid
uName := ""
newMap := &model.TableField{}
wg := errgroup.Group{}
wg.Go(func() (err error) {
userInfo, err := s.getMultiUserInfo(uid)
if err == nil && userInfo != nil && userInfo.Uname != "" {
uName = userInfo.Uname
}
return
})
wg.Go(func() (err error) {
roomInfo, err := s.getBaseRoomInfo(uid)
if err == nil && roomInfo != nil {
newMap.RoomId = int(roomInfo.Roomid)
newMap.ShortId = int(roomInfo.ShortId)
newMap.Uid = roomInfo.Uid
newMap.UName = roomInfo.Uname
newMap.Area = int(roomInfo.Area)
newMap.Title = roomInfo.Title
newMap.Tag = roomInfo.Tags
newMap.TryTime = roomInfo.TryTime
newMap.Cover = roomInfo.Cover
newMap.UserCover = roomInfo.UserCover
newMap.LockStatus = roomInfo.LockStatus
newMap.HiddenStatus = roomInfo.HiddenStatus
newMap.Attentions = int(roomInfo.Attentions)
newMap.Online = int(roomInfo.Online)
newMap.LiveTime = roomInfo.LiveTime
newMap.AreaV2Id = int(roomInfo.AreaV2Id)
newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
newMap.Virtual = int(roomInfo.Virtual)
newMap.AreaV2Name = roomInfo.AreaV2Name
newMap.CTime = roomInfo.Ctime
newMap.MTime = roomInfo.Mtime
newMap.RoundStatus = int(roomInfo.RoundStatus)
newMap.OnFlag = int(roomInfo.OnFlag)
}
return
})
err := wg.Wait()
if err == nil && newMap.RoomId != 0 {
ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
if uName != "" {
ret["new"].(map[string]interface{})["uname"] = uName
retByte["uname"] = []byte(uName)
}
if p.MsgContent.ExtInfo != nil {
ret["new"].(map[string]interface{})["attentions"] = p.MsgContent.ExtInfo.UpUidFans
ret["new"].(map[string]interface{})["attention"] = p.MsgContent.ExtInfo.UpUidFans
retByte["attentions"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
retByte["attention"] = []byte(strconv.Itoa(p.MsgContent.ExtInfo.UpUidFans))
}
//构造假old
ret["old"].(map[string]interface{})["attention"] = 0
ret["old"].(map[string]interface{})["attentions"] = 0
wg := errgroup.Group{}
wg.Go(func() (err error) {
for i := 0; i < _retry; i++ {
hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
err = hbaseErr
if hbaseErr != nil {
continue
}
break
}
if err != nil {
log.Error("[AttentionDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
}
return
})
wg.Go(func() (err error) {
err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
if err != nil {
log.Error("[AttentionDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
}
return
})
wg.Wait()
log.Info("[AttentionDataBus]success to handle, error(%v), msg:(%v)", err, ret)
continue
}
log.Error("[AttentionDataBus]fail to getData, error(%v),msg:(%v)", err, p)
}
}

View File

@@ -0,0 +1,344 @@
package service
import (
"context"
"errors"
"fmt"
"go-common/app/job/live/push-search/dao"
"go-common/app/job/live/push-search/model"
relationV1 "go-common/app/service/live/relation/api/liverpc/v1"
roomV1 "go-common/app/service/live/room/api/liverpc/v1"
roomV2 "go-common/app/service/live/room/api/liverpc/v2"
userV3 "go-common/app/service/live/user/api/liverpc/v3"
accountApi "go-common/app/service/main/account/api"
"go-common/library/log"
rpccontext "go-common/library/net/rpc/liverpc/context"
"strconv"
"time"
)
var (
hbaseTable = "live:PushSearch"
hbaseFamily = "search"
fields = []string{
"roomid",
"short_id",
"uid",
"uname",
"area",
"title",
"tags",
"try_time",
"cover",
"user_cover",
"lock_status",
"hidden_status",
"attentions",
"online",
"live_time",
"area_v2_id",
"area_v2_parent_id",
"virtual",
"round_status",
"on_flag",
"area_v2_name",
"ctime",
"mtime",
}
)
func (s *Service) getBaseRoomInfo(uid int64) (roomInfo *roomV2.RoomGetByIdsResp_RoomInfo, err error) {
roomIdResp, err := dao.RoomApi.V2Room.RoomIdByUid(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomRoomIdByUidReq{
Uid: uid,
})
if err != nil {
log.Error("[getBaseRoomInfo]RoomIdByUid rpc error, error:%+v", err)
return
}
if roomIdResp.Code != 0 {
log.Error("[getBaseRoomInfo]RoomIdByUid return error, code:%d, msg:%s", roomIdResp.Code, roomIdResp.Msg)
err = errors.New("getRoomId return error")
return
}
if roomIdResp.Data == nil {
log.Error("[getBaseRoomInfo]GetMultiple empty data")
err = errors.New("getRoomId empty error")
return
}
if roomIdResp.Data.RoomId == 0 {
log.Error("[getBaseRoomInfo]GetMultiple empty data")
err = errors.New("roomId not found error")
return
}
roomInfoResp := &roomV2.RoomGetByIdsResp{}
roomInfoResp, err = dao.RoomApi.V2Room.GetByIds(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV2.RoomGetByIdsReq{
Ids: []int64{roomIdResp.Data.RoomId},
From: "push-search",
Fields: fields,
})
if err != nil {
log.Error("[getBaseRoomInfo]GetByIds rpc error, error:%+v", err)
return
}
if roomInfoResp.Code != 0 {
log.Error("[getBaseRoomInfo]GetByIds return error, code:%d, msg:%s", roomInfoResp.Code, roomInfoResp.Msg)
err = errors.New("GetByIds return error")
return
}
if roomInfoResp.Data == nil {
log.Error("[getBaseRoomInfo]GetByIds empty data")
err = errors.New("GetByIds empty error")
return
}
info, ok := roomInfoResp.Data[roomIdResp.Data.RoomId]
if !ok {
log.Error("[getBaseRoomInfo]GetByIds not found")
err = errors.New("roomId not found error")
return
}
roomInfo = info
return
}
func (s *Service) getMultiUserInfo(uid int64) (userInfo *userV3.UserGetMultipleResp_Info, err error) {
userInfo = &userV3.UserGetMultipleResp_Info{}
pr, err := s.AccountClient.Profile3(context.TODO(), &accountApi.MidReq{Mid: uid})
if err != nil {
log.Error("[getMultiUserInfo]Profile3 rpc error, error:%+v", err)
return
}
if pr == nil {
log.Error("[getMultiUserInfo]Profile3 empty data")
err = errors.New("user empty error")
return
}
userInfo.Uid = uid
userInfo.Uname = pr.GetProfile().GetName()
userInfo.Face = pr.GetProfile().GetFace()
if userInfo.Uname != "" {
return
}
err = errors.New("user not found")
log.Error("[getMultiUserInfo]GetMultiple no user, data:%+v", userInfo)
return
}
func (s *Service) getFc(uid int64) (fc int, err error) {
fcResp := &relationV1.FeedGetUserFcResp{}
fcResp, err = dao.RelationApi.V1Feed.GetUserFc(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &relationV1.FeedGetUserFcReq{
Follow: uid,
})
if err != nil {
log.Error("[getFc]GetFc rpc error, error:%+v", err)
return
}
if fcResp.Code != 0 {
log.Error("[getFc]GetFc return error, code:%d, msg:%s", fcResp.Code, fcResp.Msg)
err = errors.New("fc return error")
return
}
if fcResp.Data == nil {
log.Error("[getFc]GetFc empty data")
err = errors.New("fc empty error")
return
}
fc = int(fcResp.Data.Fc)
return
}
func (s *Service) saveHBase(c context.Context, key string, columnInfo map[string][]byte) (err error) {
var ctx, cancel = context.WithTimeout(c, time.Duration(s.c.SearchHBase.WriteTimeout)*time.Millisecond)
defer cancel()
values := map[string]map[string][]byte{hbaseFamily: columnInfo}
if _, err = s.dao.SearchHBase.PutStr(ctx, hbaseTable, key, values); err != nil {
log.Error("SearchHBase.PutStr error(%v), table(%s), values(%+v)", err, hbaseTable, values)
}
return
}
func (s *Service) getLockStatus(lockStatus string) int {
status := 0
if lockStatus != "0000-00-00 00:00:00" {
status = 1
}
return status
}
func (s *Service) getHiddenStatus(HiddenStatus string) int {
status := 0
if HiddenStatus != "0000-00-00 00:00:00" {
status = 1
}
return status
}
//hbase key roomID md5
func (s *Service) rowKey(roomId int) string {
key := fmt.Sprintf("%d_%d", roomId%10, roomId)
return key
}
func (s *Service) generateSearchInfo(action string, table string, new *model.TableField, old *model.TableField) (ret map[string]interface{}, retByte map[string][]byte) {
ret = make(map[string]interface{})
ret["action"] = action
ret["table"] = table
//搜索字段转换
newMap := make(map[string]interface{})
newMap["id"] = new.RoomId
newMap["short_id"] = new.ShortId
newMap["uid"] = new.Uid
newMap["uname"] = new.UName
newMap["category"] = new.Area
newMap["title"] = new.Title
newMap["tag"] = new.Tag
newMap["try_time"] = new.TryTime
newMap["cover"] = new.Cover
newMap["user_cover"] = new.UserCover
newMap["lock_status"] = s.getLockStatus(new.LockStatus)
newMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
newMap["attentions"] = new.Attentions
newMap["attention"] = new.Attentions
newMap["online"] = new.Online
newMap["live_time"] = new.LiveTime
newMap["area_v2_id"] = new.AreaV2Id
newMap["ord"] = new.AreaV2ParentId
newMap["arcrank"] = new.Virtual
newMap["lastupdate"] = s.getLastUpdate(new)
newMap["is_live"] = s.getLiveStatus(new)
newMap["s_category"] = new.AreaV2Name
ret["new"] = newMap
oldMap := make(map[string]interface{})
if old != nil {
oldMap["id"] = old.RoomId
oldMap["short_id"] = old.ShortId
oldMap["uid"] = old.Uid
oldMap["uname"] = old.UName
oldMap["category"] = old.Area
oldMap["title"] = old.Title
oldMap["tag"] = old.Tag
oldMap["try_time"] = old.TryTime
oldMap["cover"] = old.Cover
oldMap["user_cover"] = old.UserCover
oldMap["lock_status"] = s.getLockStatus(old.LockStatus)
oldMap["hidden_status"] = s.getHiddenStatus(old.HiddenStatus)
oldMap["attentions"] = old.Attentions
oldMap["attention"] = old.Attentions
oldMap["online"] = old.Online
oldMap["live_time"] = old.LiveTime
oldMap["area_v2_id"] = old.AreaV2Id
oldMap["area_v2_name"] = old.AreaV2Name
oldMap["ord"] = old.AreaV2ParentId
oldMap["arcrank"] = old.Virtual
oldMap["lastupdate"] = s.getLastUpdate(old)
oldMap["is_live"] = s.getLiveStatus(old)
}
if action != "insert" && old == nil {
oldMap["id"] = new.RoomId
oldMap["short_id"] = new.ShortId
oldMap["uid"] = new.Uid
oldMap["uname"] = new.UName
oldMap["category"] = new.Area
oldMap["title"] = new.Title
oldMap["tag"] = new.Tag
oldMap["try_time"] = new.TryTime
oldMap["cover"] = new.Cover
oldMap["user_cover"] = new.UserCover
oldMap["lock_status"] = s.getLockStatus(new.LockStatus)
oldMap["hidden_status"] = s.getHiddenStatus(new.HiddenStatus)
oldMap["attentions"] = new.Attentions
oldMap["attention"] = new.Attentions
oldMap["online"] = new.Online
oldMap["live_time"] = new.LiveTime
oldMap["area_v2_id"] = new.AreaV2Id
oldMap["area_v2_name"] = new.AreaV2Name
oldMap["ord"] = new.AreaV2ParentId
oldMap["arcrank"] = new.Virtual
oldMap["lastupdate"] = s.getLastUpdate(new)
oldMap["is_live"] = s.getLiveStatus(new)
}
ret["old"] = oldMap
newByteMap := make(map[string][]byte)
newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
newByteMap["uname"] = []byte(new.UName)
newByteMap["category"] = []byte(strconv.Itoa(new.Area))
newByteMap["title"] = []byte(new.Title)
newByteMap["tag"] = []byte(new.Tag)
newByteMap["try_time"] = []byte(new.TryTime)
newByteMap["cover"] = []byte(new.Cover)
newByteMap["user_cover"] = []byte(new.UserCover)
newByteMap["lock_status"] = []byte(strconv.Itoa(s.getLockStatus(new.LockStatus)))
newByteMap["hidden_status"] = []byte(strconv.Itoa(s.getHiddenStatus(new.HiddenStatus)))
newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
newByteMap["online"] = []byte(strconv.Itoa(new.Online))
newByteMap["live_time"] = []byte(new.LiveTime)
newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
newByteMap["lastupdate"] = []byte(s.getLastUpdate(new))
newByteMap["is_live"] = []byte(strconv.Itoa(s.getLiveStatus(new)))
newByteMap["s_category"] = []byte(new.AreaV2Name)
return ret, newByteMap
}
//获取直播状态
func (s *Service) getLiveStatus(roomInfo *model.TableField) int {
if roomInfo.LiveTime != "0000-00-00 00:00:00" {
return 1
}
if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1 {
return 2
}
return 0
}
//获取房间最后更新时间
func (s *Service) getLastUpdate(roomInfo *model.TableField) string {
if roomInfo.MTime != "0000-00-00 00:00:00" {
return roomInfo.MTime
}
return roomInfo.CTime
}
func (s *Service) getAreaV2Detail(areaV2Id int) (areaInfo *roomV1.AreaGetDetailResp_AreaInfo, err error) {
areaResp, err := dao.RoomApi.V1Area.GetDetail(rpccontext.WithTimeout(context.TODO(), 50*time.Millisecond), &roomV1.AreaGetDetailReq{
Id: int64(areaV2Id),
})
if err != nil {
log.Error("[getAreaV2Detail]GetMultiple rpc error, error:%+v", err)
return
}
if areaResp.Code != 0 {
log.Error("[getAreaV2Detail]GetMultiple return error, code:%d, msg:%s", areaResp.Code, areaResp.Msg)
err = errors.New("user return error")
return
}
if areaResp.Data == nil {
log.Error("[getAreaV2Detail]GetMultiple empty data")
err = errors.New("area detail empty error")
return
}
return areaResp.Data, err
}

View File

@@ -0,0 +1,34 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["service.go"],
importpath = "go-common/app/job/live/push-search/service/migrate",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/push-search/conf:go_default_library",
"//app/job/live/push-search/dao/migrate:go_default_library",
"//app/job/live/push-search/model:go_default_library",
"//library/database/sql:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,243 @@
package migrate
import (
"context"
defaultsql "database/sql"
"fmt"
"go-common/app/job/live/push-search/conf"
"go-common/app/job/live/push-search/dao/migrate"
"go-common/app/job/live/push-search/model"
"go-common/library/database/sql"
"strconv"
"sync"
"time"
)
const _sql = "select roomid, short_id,uid,uname,area,title,tags, mtime,a.ctime,try_time,user_cover,a.lock_status,hidden_status,attentions,live_time,area_v2_id,area_v2_parent_id,b.name as area_v2_name, virtual,round_status,on_flag,online, cover from ap_room a left join ap_room_area_v2 b on a.area_v2_id=b.id where roomid > %d order by roomid asc limit 100 "
const hbaseTable = "live:PushSearch"
const hbaseFamily = "search"
type message struct {
rowKey string
values map[string]map[string][]byte
}
type MService struct {
c *conf.Config
dao *migrate.Dao
hChan []chan *message
waiterChan *sync.WaitGroup
mainWaiter *sync.WaitGroup
}
func NewMigrateS(c *conf.Config) (s *MService) {
s = &MService{
c: c,
dao: migrate.NewMigrate(c),
hChan: make([]chan *message, c.MigrateNum),
waiterChan: new(sync.WaitGroup),
mainWaiter: new(sync.WaitGroup),
}
//ap room 表 binlog qps 高, hash roomId 并行
for i := 0; i < c.MigrateNum; i++ {
ch := make(chan *message, 1024)
s.hChan[i] = ch
go s.handle(ch)
}
return s
}
func (ms *MService) Migrate (roomid string, isTest string) {
id, err := strconv.Atoi(roomid)
if err != nil {
fmt.Println("roomid error")
}
ms.mainWaiter.Add(1)
defer ms.mainWaiter.Done()
var rows *sql.Rows
online := &defaultsql.NullInt64{}
cover := &defaultsql.NullString{}
areaV2Name := &defaultsql.NullString{}
for {
rows, err := ms.dao.RoomDb.Query(context.TODO(), fmt.Sprintf(_sql, id))
if err != nil {
fmt.Println("query error:%+v", err)
return
}
for rows.Next() {
r := new(model.TableField)
if err = rows.Scan(&r.RoomId, &r.ShortId, &r.Uid, &r.UName, &r.Area, &r.Title, &r.Tag, &r.MTime, &r.CTime, &r.TryTime, &r.UserCover, &r.LockStatus, &r.HiddenStatus, &r.Attentions, &r.LiveTime, &r.AreaV2Id, &r.AreaV2ParentId, areaV2Name, &r.Virtual, &r.RoundStatus, &r.OnFlag, online, cover); err != nil {
if !online.Valid {
r.Online = 0
}
if !cover.Valid {
r.Cover = ""
}
}
r.AreaV2Name = areaV2Name.String
r.Online = int(online.Int64)
r.Cover = cover.String
zijie := ms.generateSearchInfo(r)
if r.LiveTime != "0000-00-00 00:00:00" {
fmt.Println(r.RoomId, "jump live room")
continue
}
values := map[string]map[string][]byte{hbaseFamily: zijie}
rowKey := ms.rowKey(r.RoomId)
m := &message{
rowKey: rowKey,
values: values,
}
ms.hChan[r.RoomId % ms.c.MigrateNum] <- m
fmt.Println(r.RoomId)
if isTest == "1" {
return
}
id = r.RoomId
}
}
rows.Close()
}
func (ms *MService) handle(c chan *message) {
ms.waiterChan.Add(1)
defer ms.waiterChan.Done()
for {
msgData, ok := <-c
if !ok {
fmt.Println("close chan")
return
}
ms.dao.SearchHBase.PutStr(context.TODO(), hbaseTable, msgData.rowKey, msgData.values)
}
}
func (ms *MService) Close() {
ms.dao.Close()
ms.mainWaiter.Wait()
for _, ch := range ms.hChan {
close(ch)
}
ms.waiterChan.Wait()
ms.dao.SearchHBase.Close()
}
func (ms *MService) rowKey(roomId int) string{
key := fmt.Sprintf("%d_%d", roomId % 10, roomId)
return key
}
func (ms *MService) generateSearchInfo(new *model.TableField) (retByte map[string][]byte){
newByteMap := make(map[string][]byte)
newByteMap["id"] = []byte(strconv.Itoa(new.RoomId))
newByteMap["short_id"] = []byte(strconv.Itoa(new.ShortId))
newByteMap["uid"] = []byte(strconv.FormatInt(new.Uid, 10))
newByteMap["uname"] = []byte(new.UName)
newByteMap["category"] = []byte(strconv.Itoa(new.Area))
newByteMap["title"] = []byte(new.Title)
newByteMap["tag"] = []byte(new.Tag)
tryTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.TryTime, time.Local)
tryTimeStr := tryTime.Format("2006-01-02 15:04:05")
if tryTimeStr == "0001-01-01 00:00:00" {
tryTimeStr = "0000-00-00 00:00:00"
new.TryTime = tryTimeStr
}
newByteMap["try_time"] = []byte(tryTimeStr)
newByteMap["cover"] = []byte(new.Cover)
newByteMap["user_cover"] = []byte(new.UserCover)
lockStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LockStatus, time.Local)
lockStatusStr := lockStatus.Format("2006-01-02 15:04:05")
if lockStatusStr == "0001-01-01 00:00:00" {
lockStatusStr = "0000-00-00 00:00:00"
new.LockStatus = lockStatusStr
}
newByteMap["lock_status"] = []byte(strconv.Itoa(ms.getLockStatus(lockStatusStr)))
hiddenStatus, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.HiddenStatus, time.Local)
hiddenStatusStr := hiddenStatus.Format("2006-01-02 15:04:05")
if hiddenStatusStr == "0001-01-01 00:00:00" {
hiddenStatusStr = "0000-00-00 00:00:00"
new.HiddenStatus = hiddenStatusStr
}
newByteMap["hidden_status"] = []byte(strconv.Itoa(ms.getHiddenStatus(hiddenStatusStr)))
newByteMap["attentions"] = []byte(strconv.Itoa(new.Attentions))
newByteMap["attention"] = []byte(strconv.Itoa(new.Attentions))
newByteMap["online"] = []byte(strconv.Itoa(new.Online))
liveTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.LiveTime, time.Local)
liveTimeStr := liveTime.Format("2006-01-02 15:04:05")
if liveTimeStr == "0001-01-01 00:00:00" {
liveTimeStr = "0000-00-00 00:00:00"
new.LiveTime = liveTimeStr
}
newByteMap["live_time"] = []byte(liveTimeStr)
newByteMap["area_v2_id"] = []byte(strconv.Itoa(new.AreaV2Id))
newByteMap["ord"] = []byte(strconv.Itoa(new.AreaV2ParentId))
newByteMap["arcrank"] = []byte(strconv.Itoa(new.Virtual))
cTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.CTime, time.Local)
cTimeStr := cTime.Format("2006-01-02 15:04:05")
if cTimeStr == "0001-01-01 00:00:00" {
new.CTime = "0000-00-00 00:00:00"
}else{
new.CTime = cTimeStr
}
mTime, _ := time.ParseInLocation("2006-01-02T15:04:05+08:00", new.MTime, time.Local)
mTimeStr := mTime.Format("2006-01-02 15:04:05")
if mTimeStr == "0001-01-01 00:00:00" {
new.MTime = "0000-00-00 00:00:00"
}else{
new.MTime = mTimeStr
}
newByteMap["lastupdate"] = []byte(ms.getLastUpdate(new))
newByteMap["is_live"] = []byte(strconv.Itoa(ms.getLiveStatus(new)))
newByteMap["s_category"] = []byte(new.AreaV2Name)
return newByteMap
}
//获取直播状态
func (ms *MService) getLiveStatus(roomInfo *model.TableField) int{
if roomInfo.LiveTime != "0000-00-00 00:00:00" {
return 1
}
if roomInfo.RoundStatus == 1 && roomInfo.OnFlag == 1{
return 2
}
return 0
}
//获取房间最后更新时间
func (ms *MService) getLastUpdate(roomInfo *model.TableField) string{
if roomInfo.MTime != "0000-00-00 00:00:00" {
return roomInfo.MTime
}
return roomInfo.CTime
}
func (ms *MService) getLockStatus(lockStatus string) int{
status := 0
if lockStatus != "0000-00-00 00:00:00" {
status = 1
}
return status
}
func (ms *MService) getHiddenStatus(HiddenStatus string) int{
status := 0
if HiddenStatus != "0000-00-00 00:00:00" {
status = 1
}
return status
}

View File

@@ -0,0 +1,194 @@
package service
import (
"encoding/json"
"go-common/app/job/live/push-search/model"
"go-common/library/log"
"context"
roomV1 "go-common/app/service/live/room/api/liverpc/v1"
"go-common/library/sync/errgroup"
"strconv"
)
const (
_updateAct = "update"
_insertAct = "insert"
)
func (s *Service) roomInfoNotifyConsumeProc() {
defer s.waiter.Done()
for {
msg, ok := <-s.dao.RoomInfoDataBus.Messages()
// databus关闭chan导致,服务自杀或异常退出
if !ok {
log.Error("roomInfoNotifyConsumeProc closed")
if err := s.dao.RoomInfoDataBus.Close(); err != nil {
log.Error("s.dao.RoomInfoDataBus.Close() error(%v)", err)
}
return
}
m := &message{data: msg}
p := new(model.ApRoomNotifyInfo)
if err := json.Unmarshal(msg.Value, p); err != nil {
msg.Commit()
log.Error("[RoomInfoDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
continue
}
if p.Action != _insertAct && p.Action != _updateAct {
msg.Commit()
log.Error("[RoomInfoDataBus]Action Invalid error(%v)", p.Action)
continue
}
//判断是否是关注or昵称变更,如果是则跳过,顺便解出新旧map
isAttentionUpdate, oldMap, newMap, err := isAttentionChange(p.Action, p.Old, p.New)
if err != nil {
msg.Commit()
log.Error("[RoomInfoDataBus]isAttentionChange,json.Unmarshal(old:%s, new:%s) error(%v)", string(p.Old), string(p.New), err)
continue
}
if isAttentionUpdate {
msg.Commit()
log.Error("[RoomInfoDataBus]attention change pass")
continue
}
//hash chan
if newMap == nil || newMap.RoomId <= 0 {
msg.Commit()
log.Error("[RoomInfoDataBus]roomId type conversion error, roomId:%+v", newMap)
continue
}
dataMap := new(model.DataMap)
dataMap.Action = p.Action
dataMap.Table = p.Table
dataMap.New = newMap
dataMap.Old = oldMap
m.object = dataMap
log.Info("[RoomInfoDataBus]roomInfoNotifyConsumeProc key:%s partition:%d offset:%d", msg.Key, msg.Partition, msg.Offset)
s.binLogMergeChan[newMap.RoomId%s.c.Group.RoomInfo.Num] <- m
}
}
func isAttentionChange(action string, old []byte, new []byte) (bool, *model.TableField, *model.TableField, error) {
newMap := &model.TableField{}
oldMap := &model.TableField{}
err := json.Unmarshal(new, newMap)
if err != nil {
return false, oldMap, newMap, err
}
if action == _updateAct {
err := json.Unmarshal(old, oldMap)
if err != nil {
return false, oldMap, newMap, err
}
if oldMap != nil && oldMap.Attentions != newMap.Attentions {
return true, oldMap, newMap, err
}
}
if action == _insertAct {
oldMap = nil
}
return false, oldMap, newMap, err
}
func (s *Service) roomInfoNotifyHandleProc(c chan *message) {
defer s.waiterChan.Done()
for {
msgData, ok := <-c
if !ok {
log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc closed")
return
}
msgData.data.Commit()
p, assertOk := msgData.object.(*model.DataMap)
if !assertOk {
log.Error("[RoomInfoDataBus]roomInfoNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
return
}
uid := p.New.Uid
wg := errgroup.Group{}
uName := ""
fc := 0
areaInfo := &roomV1.AreaGetDetailResp_AreaInfo{}
wg.Go(func() (err error) {
userInfo, err := s.getMultiUserInfo(uid)
if err == nil && userInfo != nil && userInfo.Uname != "" {
uName = userInfo.Uname
}
return
})
//fc任何错误都要返回,不然fc为0无法判断是接口返回0还是初始化的0!!!!
wg.Go(func() (err error) {
fc, err = s.getFc(uid)
return
})
wg.Go(func() (err error) {
areaInfo, err = s.getAreaV2Detail(p.New.AreaV2Id)
return
})
err := wg.Wait()
//成功返回则替换,否则输出原数据
ret, retByte := s.generateSearchInfo(p.Action, p.Table, p.New, p.Old)
if err == nil {
if uName != "" {
ret["new"].(map[string]interface{})["uname"] = uName
retByte["uname"] = []byte(uName)
}
if areaInfo != nil && areaInfo.Name != "" {
ret["new"].(map[string]interface{})["s_category"] = areaInfo.Name
retByte["s_category"] = []byte(areaInfo.Name)
}
ret["new"].(map[string]interface{})["attentions"] = fc
ret["new"].(map[string]interface{})["attention"] = fc
retByte["attentions"] = []byte(strconv.Itoa(fc))
retByte["attention"] = []byte(strconv.Itoa(fc))
}
writeWg := errgroup.Group{}
writeWg.Go(func() (err error) {
for i := 0; i < _retry; i++ {
hbaseErr := s.saveHBase(context.TODO(), s.rowKey(p.New.RoomId), retByte)
err = hbaseErr
if hbaseErr != nil {
continue
}
break
}
if err != nil {
log.Error("[RoomInfoDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
}
return
})
writeWg.Go(func() (err error) {
err = s.dao.Pub(context.TODO(), int64(p.New.RoomId), ret)
if err != nil {
log.Error("[RoomInfoDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
}
return
})
wg.Wait()
log.Info("[RoomInfoDataBus]success handle, error(%v),msg:(%v)", err, ret)
}
}

View File

@@ -0,0 +1,109 @@
package service
import (
"context"
"go-common/app/job/live/push-search/conf"
"go-common/app/job/live/push-search/dao"
accountApi "go-common/app/service/main/account/api"
"go-common/library/queue/databus"
"sync"
)
const (
_tableArchive = "ap_room"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
binLogMergeChan []chan *message
attentionMergeChan []chan *message
unameMergeChan []chan *message
waiter *sync.WaitGroup
waiterChan *sync.WaitGroup
AccountClient accountApi.AccountClient
}
type message struct {
next *message
data *databus.Message
object interface{}
done bool
}
// New init
func New(c *conf.Config) (s *Service) {
dao.InitAPI()
s = &Service{
c: c,
dao: dao.New(c),
binLogMergeChan: make([]chan *message, c.Group.RoomInfo.Num),
attentionMergeChan: make([]chan *message, c.Group.Attention.Num),
unameMergeChan: make([]chan *message, c.Group.UserInfo.Num),
waiter: new(sync.WaitGroup),
waiterChan: new(sync.WaitGroup),
}
accountClient, err := accountApi.NewClient(nil)
if err != nil {
panic(err)
}
s.AccountClient = accountClient
//ap room 表 binlog qps 高, hash roomId 并行
for i := 0; i < c.Group.RoomInfo.Num; i++ {
ch := make(chan *message, c.Group.RoomInfo.Chan)
s.binLogMergeChan[i] = ch
s.waiterChan.Add(1)
go s.roomInfoNotifyHandleProc(ch)
}
for i := 0; i < c.Group.Attention.Num; i++ {
ch := make(chan *message, c.Group.Attention.Chan)
s.attentionMergeChan[i] = ch
s.waiterChan.Add(1)
go s.attentionNotifyHandleProc(ch)
}
for i := 0; i < c.Group.UserInfo.Num; i++ {
ch := make(chan *message, c.Group.UserInfo.Chan)
s.unameMergeChan[i] = ch
s.waiterChan.Add(1)
go s.unameNotifyHandleProc(ch)
}
s.waiter.Add(1)
go s.roomInfoNotifyConsumeProc()
s.waiter.Add(1)
go s.attentionNotifyConsumeProc()
s.waiter.Add(1)
go s.unameNotifyConsumeProc()
return s
}
// Ping Service
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close Service
func (s *Service) Close() {
//databus chan close
s.dao.Close()
s.waiter.Wait()
//task goroutine close
for _, ch := range s.binLogMergeChan {
close(ch)
}
for _, ch := range s.attentionMergeChan {
close(ch)
}
for _, ch := range s.unameMergeChan {
close(ch)
}
s.waiterChan.Wait()
s.dao.PushSearchDataBus.Close()
}

View File

@@ -0,0 +1,155 @@
package service
import (
"context"
"encoding/json"
"go-common/app/job/live/push-search/model"
"go-common/library/log"
"go-common/library/sync/errgroup"
"strconv"
)
func (s *Service) unameNotifyConsumeProc() {
defer s.waiter.Done()
for {
msg, ok := <-s.dao.UserNameDataBus.Messages()
if !ok {
log.Error("unameNotifyConsumeProc closed")
if err := s.dao.UserNameDataBus.Close(); err != nil {
log.Error("s.dao.UserNameDataBus.Close() error(%v)", err)
}
return
}
//先提交防止阻塞,关闭时等待任务执行完
m := &message{data: msg}
raw := new(model.LiveDatabus)
if err := json.Unmarshal(msg.Value, raw); err != nil {
msg.Commit()
log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", string(msg.Value), err)
continue
}
p := new(model.UnameNotifyInfo)
if err := json.Unmarshal([]byte(raw.MsgContent), p); err != nil {
msg.Commit()
log.Error("[UnameDataBus]json.Unmarshal(%s) error(%v)", raw.MsgContent, err)
continue
}
m.object = p
s.unameMergeChan[p.Uid%int64(s.c.Group.UserInfo.Num)] <- m
}
}
func (s *Service) unameNotifyHandleProc(c chan *message) {
defer s.waiterChan.Done()
for {
msgData, ok := <-c
if !ok {
log.Error("[UnameDataBus]unameNotifyHandleProc closed")
return
}
//先提交防止阻塞,关闭时等待任务执行完
msgData.data.Commit()
p, assertOk := msgData.object.(*model.UnameNotifyInfo)
if !assertOk {
log.Error("[UnameDataBus]unameNotifyHandleProc msg object type conversion error, msg:%+v", msgData)
return
}
uid := p.Uid
if uid == 0 {
log.Error("[UnameDataBus]empty uid, uid:%d", uid)
continue
}
fc := 0
newMap := &model.TableField{}
wg := errgroup.Group{}
wg.Go(func() (err error) {
fc, err = s.getFc(uid)
return
})
wg.Go(func() (err error) {
roomInfo, err := s.getBaseRoomInfo(uid)
if err == nil && roomInfo != nil {
newMap.RoomId = int(roomInfo.Roomid)
newMap.ShortId = int(roomInfo.ShortId)
newMap.Uid = roomInfo.Uid
newMap.UName = roomInfo.Uname
newMap.Area = int(roomInfo.Area)
newMap.Title = roomInfo.Title
newMap.Tag = roomInfo.Tags
newMap.TryTime = roomInfo.TryTime
newMap.Cover = roomInfo.Cover
newMap.UserCover = roomInfo.UserCover
newMap.LockStatus = roomInfo.LockStatus
newMap.HiddenStatus = roomInfo.HiddenStatus
newMap.Attentions = int(roomInfo.Attentions)
newMap.Online = int(roomInfo.Online)
newMap.LiveTime = roomInfo.LiveTime
newMap.AreaV2Id = int(roomInfo.AreaV2Id)
newMap.AreaV2ParentId = int(roomInfo.AreaV2ParentId)
newMap.Virtual = int(roomInfo.Virtual)
newMap.AreaV2Name = roomInfo.AreaV2Name
newMap.CTime = roomInfo.Ctime
newMap.MTime = roomInfo.Mtime
newMap.RoundStatus = int(roomInfo.RoundStatus)
newMap.OnFlag = int(roomInfo.OnFlag)
}
return
})
err := wg.Wait()
if err == nil && newMap.RoomId != 0 {
//非uname更新
if p.Uname == newMap.UName {
log.Info("[UnameDataBus]uname no change, msg:(%v)", p)
continue
}
ret, retByte := s.generateSearchInfo("update", _tableArchive, newMap, nil)
if p.Uname != "" {
ret["new"].(map[string]interface{})["uname"] = p.Uname
retByte["uname"] = []byte(p.Uname)
}
ret["new"].(map[string]interface{})["attentions"] = fc
ret["new"].(map[string]interface{})["attention"] = fc
retByte["attentions"] = []byte(strconv.Itoa(fc))
retByte["attention"] = []byte(strconv.Itoa(fc))
ret["old"].(map[string]interface{})["uname"] = ""
wg := errgroup.Group{}
wg.Go(func() (err error) {
for i := 0; i < _retry; i++ {
hbaseErr := s.saveHBase(context.TODO(), s.rowKey(newMap.RoomId), retByte)
err = hbaseErr
if hbaseErr != nil {
continue
}
break
}
if err != nil {
log.Error("[UnameDataBus]fail to write hbase, msg:(%v), err:(%v)", p, err)
}
return
})
wg.Go(func() (err error) {
err = s.dao.Pub(context.TODO(), int64(newMap.RoomId), ret)
if err != nil {
log.Error("[UnameDataBus]fail to pub, msg:(%v), err:(%v)", p, err)
}
return
})
wg.Wait()
log.Info("[UnameDataBus]success to handle, error(%v), msg:(%v)", err, ret)
continue
}
log.Error("[UnameDataBus]fail to getData, error(%v),msg:(%v)", err, p)
}
}

View File

@@ -0,0 +1,21 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/job/live/recommend-job/cmd:all-srcs",
"//app/job/live/recommend-job/internal/conf:all-srcs",
"//app/job/live/recommend-job/internal/dao:all-srcs",
"//app/job/live/recommend-job/internal/model:all-srcs",
"//app/job/live/recommend-job/internal/server/http:all-srcs",
"//app/job/live/recommend-job/internal/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,2 @@
### v1.0.0
1. 上线功能xxx

View File

@@ -0,0 +1,6 @@
# Owner
liugang
# Author
# Reviewer

View File

@@ -0,0 +1,10 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- liugang
labels:
- job
- job/live/recommend-job
- live
options:
no_parent_owners: true

View File

@@ -0,0 +1,22 @@
# live-web-interface
# 项目简介
推荐系统的离线脚本服务
# 编译环境
# 依赖包
# 编译执行
# 部署说明
必须从hadoop拉取数据文件所以需要hadoop客户端
hadoop客户端从 http://livedev.bilibili.co/download/hadoop-sz.tar.gz 拉取
文件服务部署在 shylf-live-app-27
对于uat环境
没有分布式hadoop部署一个http的服务提供单机hadoop的文件下载

View File

@@ -0,0 +1 @@
# HTTP API文档

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["test.toml"],
importpath = "go-common/app/job/live/recommend-job/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/recommend-job/internal/conf:go_default_library",
"//app/job/live/recommend-job/internal/server/http:go_default_library",
"//app/job/live/recommend-job/internal/service:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,90 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"time"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/app/job/live/recommend-job/internal/server/http"
"go-common/app/job/live/recommend-job/internal/service"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
var itemCFRun = false
var itemCFInputPath = ""
var itemCFWorkerNum = 1000
var userAreaRun = false
var uerAreaInputPath = ""
func main() {
flag.BoolVar(&userAreaRun, "userarea.run", false, "用户分区缓存:是否手动跑")
flag.StringVar(&uerAreaInputPath, "userarea.input", "", "用户分区缓存:输入文件地址")
flag.BoolVar(&itemCFRun, "itemcf.run", false, "协同过滤推荐缓存到redis是否手动跑")
flag.StringVar(&itemCFInputPath, "itemcf.input", "", "协同过滤结果文件地址")
flag.IntVar(&itemCFWorkerNum, "itemcf.workerNum", 1000, "worker数量")
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
log.Info("recommend-job start")
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
if itemCFRun {
c := conf.Conf.ItemCFJob
c.InputFile = itemCFInputPath
c.WorkerNum = itemCFWorkerNum
var job = service.ItemCFJob{
Conf: c,
RedisConf: conf.Conf.Redis,
HadoopConf: conf.Conf.Hadoop,
}
job.Run()
os.Exit(0)
}
if userAreaRun {
c := conf.Conf.UserAreaJob
c.InputFile = uerAreaInputPath
c.WorkerNum = 1000
var job = service.UserAreaJob{
JobConf: c,
RedisConf: conf.Conf.Redis,
HadoopConf: conf.Conf.Hadoop,
}
job.Run()
os.Exit(0)
}
svc := service.New(conf.Conf)
http.Init(conf.Conf, svc)
svc.RunCrontab()
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
svc.Close()
log.Info("recommend-job exit")
time.Sleep(time.Second)
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,31 @@
[redis]
name = "recommend-job"
proto = "tcp"
addr = "127.0.0.1:6379"
idle = 2000
active = 2000
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "1m"
[log]
stdout = true
[hadoop]
tarUrl = "http://livedev.bilibili.co/download/hadoop-2.8.4.tar.gz"
hadoopDir = "/tmp"
[itemcfJob]
schedule = "30 50 16 * * *" # 秒 分钟 小时 天 月 星期
workerNum = 1000
inputFile = "/department/live/recommend/item-cf-output"
hadoopFile = "/department/live/recommend/%s/item-cf-output"
localTmpFile = "/tmp/recommend-cf-output-%s"
[userAreaJob]
schedule = "20 37 14 * * *" # 秒 分钟 小时 天 月 星期
workerNum = 1000
inputFile = "/Users/liugang/Downloads/user-area.txt"
hadoopFile = "/department/live/recommend/%s/user-area"
localTmpFile = "/tmp/user-area-output-%s"

View File

@@ -0,0 +1,38 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/job/live/recommend-job/internal/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/redis:go_default_library",
"//library/conf:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/trace:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,108 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/redis"
"go-common/library/conf"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
)
var (
confPath string
client *conf.Client
// Conf config
Conf = &Config{}
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Redis *redis.Config
Ecode *ecode.Config
ItemCFJob *JobConfig
Hadoop *HadoopConfig
UserAreaJob *JobConfig
}
// HadoopConfig ...
type HadoopConfig struct {
HadoopDir string
TarUrl string
}
// JobConfig ...
type JobConfig struct {
Schedule string
// 多少个goroutine同时去写redis数据
WorkerNum int
// 如果指定了文件,使用这个文件
// 如果没有指定,自动去下载文件
// 可以是本地文件地址或者http文件地址
InputFile string
// 在hadoop里面文件的路径带日期
HadoopFile string
// Hadoop下载到本地的路径带日期
LocalTmpFile string
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init init conf
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}

View File

@@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["dao.go"],
importpath = "go-common/app/job/live/recommend-job/internal/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/job/live/recommend-job/internal/conf:go_default_library",
"//library/cache/redis:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,37 @@
package dao
import (
"context"
"go-common/app/job/live/recommend-job/internal/conf"
"go-common/library/cache/redis"
)
// Dao dao
type Dao struct {
c *conf.Config
redis *redis.Pool
}
// New init mysql db
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
redis: redis.NewPool(c.Redis),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.redis.Close()
}
// Ping dao ping
func (d *Dao) Ping(ctx context.Context) error {
// TODO: add mc,redis... if you use
c := d.redis.Get(ctx)
defer c.Close()
_, err := c.Do("ping")
return err
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/job/live/recommend-job/internal/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1 @@
package model

Some files were not shown because too many files have changed in this diff Show More