Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"databus.go",
"redis.go",
],
importpath = "go-common/app/service/main/broadcast/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/service/main/broadcast/model:go_default_library",
"//library/cache/redis:go_default_library",
"//library/conf/paladin:go_default_library",
"//library/log:go_default_library",
"//library/queue/databus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"dao_test.go",
"databus_test.go",
"redis_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/service/main/broadcast/model:go_default_library",
"//library/conf/paladin:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)

View File

@@ -0,0 +1,50 @@
package dao
import (
"context"
"go-common/library/cache/redis"
"go-common/library/conf/paladin"
"go-common/library/queue/databus"
)
// Dao dao.
type Dao struct {
redis *redis.Pool
pushBus *databus.Databus
}
func checkErr(err error) {
if err != nil {
panic(err)
}
}
// New new a dao and return.
func New() (dao *Dao) {
var (
rds struct {
Push *redis.Config
}
dbus struct {
Push *databus.Config
}
)
checkErr(paladin.Get("redis.toml").UnmarshalTOML(&rds))
checkErr(paladin.Get("databus.toml").UnmarshalTOML(&dbus))
dao = &Dao{
redis: redis.NewPool(rds.Push),
pushBus: databus.New(dbus.Push),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.redis.Close()
}
// Ping dao ping.
func (d *Dao) Ping(c context.Context) error {
return d.pingRedis(c)
}

View File

@@ -0,0 +1,46 @@
package dao
import (
"context"
"flag"
"os"
"testing"
"go-common/library/conf/paladin"
"github.com/smartystreets/goconvey/convey"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
if os.Getenv("DEPLOY_ENV") != "" {
flag.Set("app_id", "main.community.broadcast-service")
flag.Set("conf_token", "0cb70f9480b77c89626931c31ad2b65b")
flag.Set("tree_id", "39091")
flag.Set("conf_version", "server-1")
flag.Set("deploy_env", "uat")
flag.Set("conf_host", "config.bilibili.co")
flag.Set("conf_path", "/tmp")
flag.Set("region", "sh")
flag.Set("zone", "sh001")
} else {
flag.Set("conf", "../cmd/local")
}
flag.Parse()
if err := paladin.Init(); err != nil {
panic(err)
}
d = New()
m.Run()
d.Close()
os.Exit(0)
}
func TestClose(t *testing.T) {
convey.Convey("keyMidServer", t, func(ctx convey.C) {
ctx.So(d.Ping(context.TODO()), convey.ShouldBeNil)
})
}

View File

@@ -0,0 +1,73 @@
package dao
import (
"context"
"encoding/json"
"go-common/library/log"
)
const (
_pushMsg = "push"
_broadcastMsg = "broadcast"
_broadcastRoomMsg = "broadcast_room"
)
type pushMsg struct {
Type string `json:"type,omitempty"`
Operation int32 `json:"operation,omitempty"`
Server string `json:"server,omitempty"`
Keys []string `json:"keys,omitempty"`
Room string `json:"room,omitempty"`
Speed int32 `json:"speed,omitempty"`
Platform string `json:"platform,omitempty"`
ContentType int32 `json:"content_type,omitempty"`
Message json.RawMessage `json:"message,omitempty"`
}
// PushMsg push a message to databus.
func (d *Dao) PushMsg(c context.Context, op int32, server, msg string, keys []string, contentType int32) (err error) {
pushMsg := &pushMsg{
Type: _pushMsg,
Operation: op,
Server: server,
Keys: keys,
Message: []byte(msg),
ContentType: contentType,
}
if err = d.pushBus.Send(c, keys[0], pushMsg); err != nil {
log.Error("PushMsg.send(server:%v,pushMsg:%v).error(%v)", server, pushMsg, err)
}
return
}
// BroadcastRoomMsg push a message to databus.
func (d *Dao) BroadcastRoomMsg(c context.Context, op int32, room, msg string, contentType int32) (err error) {
pushMsg := &pushMsg{
Type: _broadcastRoomMsg,
Operation: op,
Room: room,
Message: []byte(msg),
ContentType: contentType,
}
if err = d.pushBus.Send(c, room, pushMsg); err != nil {
log.Error("BroadcastRoomMsg.send(room:%v,pushMsg:%v).error(%v)", room, pushMsg, err)
}
return
}
// BroadcastMsg push a message to databus.
func (d *Dao) BroadcastMsg(c context.Context, op, speed int32, msg, platform string, contentType int32) (err error) {
pushMsg := &pushMsg{
Operation: op,
Type: _broadcastMsg,
Speed: speed,
Message: []byte(msg),
Platform: platform,
ContentType: contentType,
}
if err = d.pushBus.Send(c, _broadcastMsg, pushMsg); err != nil {
log.Error("BroadcastMsg.send(_broadcastMsg:%v,pushMsg:%v).error(%v)", _broadcastMsg, pushMsg, err)
}
return
}

View File

@@ -0,0 +1,58 @@
package dao
import (
"context"
"testing"
"github.com/smartystreets/goconvey/convey"
)
func TestDaoPushMsg(t *testing.T) {
var (
c = context.Background()
op = int32(0)
server = ""
msg = ""
keys = []string{"key"}
contentType = int32(0)
)
convey.Convey("PushMsg", t, func(ctx convey.C) {
err := d.PushMsg(c, op, server, msg, keys, contentType)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoBroadcastRoomMsg(t *testing.T) {
var (
c = context.Background()
op = int32(0)
room = ""
msg = ""
contentType = int32(0)
)
convey.Convey("BroadcastRoomMsg", t, func(ctx convey.C) {
err := d.BroadcastRoomMsg(c, op, room, msg, contentType)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoBroadcastMsg(t *testing.T) {
var (
c = context.Background()
op = int32(0)
speed = int32(0)
msg = ""
platform = ""
contentType = int32(0)
)
convey.Convey("BroadcastMsg", t, func(ctx convey.C) {
err := d.BroadcastMsg(c, op, speed, msg, platform, contentType)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}

View File

@@ -0,0 +1,328 @@
package dao
import (
"context"
"encoding/json"
"fmt"
"strconv"
"time"
"go-common/app/service/main/broadcast/model"
"go-common/library/cache/redis"
"go-common/library/log"
)
const (
_prefixMidServer = "mid_%d" // mid -> key:server
_prefixKeyServer = "key_%s" // key -> server
_prefixServerOnline = "ol_%s" // server -> online
_keyServers = "servers"
_keyMigrateRooms = "migrate_rooms"
_keyMigrateServers = "migrate_servers"
)
var (
_redisExpire = int32(time.Minute * 30 / time.Second)
)
func keyMidServer(mid int64) string {
return fmt.Sprintf(_prefixMidServer, mid)
}
func keyKeyServer(key string) string {
return fmt.Sprintf(_prefixKeyServer, key)
}
func keyServerOnline(key string) string {
return fmt.Sprintf(_prefixServerOnline, key)
}
// pingRedis check redis connection.
func (d *Dao) pingRedis(c context.Context) (err error) {
conn := d.redis.Get(c)
_, err = conn.Do("SET", "PING", "PONG")
conn.Close()
return
}
// AddMapping add a mapping.
// Mapping:
// mid -> key_server
// key -> server
func (d *Dao) AddMapping(c context.Context, mid int64, key, server string) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
var n = 2
if mid > 0 {
if err = conn.Send("HSET", keyMidServer(mid), key, server); err != nil {
log.Error("conn.Send(HSET %d,%s,%s) error(%v)", mid, server, key, err)
return
}
if err = conn.Send("EXPIRE", keyMidServer(mid), _redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %d,%s,%s) error(%v)", mid, key, server, err)
return
}
n += 2
}
if err = conn.Send("SET", keyKeyServer(key), server); err != nil {
log.Error("conn.Send(HSET %d,%s,%s) error(%v)", mid, server, key, err)
return
}
if err = conn.Send("EXPIRE", keyKeyServer(key), _redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %d,%s,%s) error(%v)", mid, key, server, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < n; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// ExpireMapping expire a mapping.
func (d *Dao) ExpireMapping(c context.Context, mid int64, key string) (has bool, err error) {
conn := d.redis.Get(c)
defer conn.Close()
var n = 1
if mid > 0 {
if err = conn.Send("EXPIRE", keyMidServer(mid), _redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %d,%s) error(%v)", mid, key, err)
return
}
n++
}
if err = conn.Send("EXPIRE", keyKeyServer(key), _redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %d,%s) error(%v)", mid, key, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < n; i++ {
if has, err = redis.Bool(conn.Receive()); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// DelMapping del a mapping.
func (d *Dao) DelMapping(c context.Context, mid int64, key, server string) (has bool, err error) {
conn := d.redis.Get(c)
defer conn.Close()
n := 1
if mid > 0 {
if err = conn.Send("HDEL", keyMidServer(mid), key); err != nil {
log.Error("conn.Send(HDEL %d,%s,%s) error(%v)", mid, key, server, err)
return
}
n++
}
if err = conn.Send("DEL", keyKeyServer(key)); err != nil {
log.Error("conn.Send(HDEL %d,%s,%s) error(%v)", mid, key, server, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < n; i++ {
if has, err = redis.Bool(conn.Receive()); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// ServersByKeys get a server by key.
func (d *Dao) ServersByKeys(c context.Context, keys []string) (res []string, err error) {
conn := d.redis.Get(c)
defer conn.Close()
var args []interface{}
for _, key := range keys {
args = append(args, keyKeyServer(key))
}
if res, err = redis.Strings(conn.Do("MGET", args...)); err != nil {
log.Error("conn.Do(MGET %v) error(%v)", args, err)
}
return
}
// KeysByMids get a key server by mid.
func (d *Dao) KeysByMids(c context.Context, mids []int64) (ress map[string]string, olMids []int64, err error) {
conn := d.redis.Get(c)
defer conn.Close()
ress = make(map[string]string)
for _, mid := range mids {
if err = conn.Send("HGETALL", keyMidServer(mid)); err != nil {
log.Error("conn.Do(HGETALL %d) error(%v)", mid, err)
return
}
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for idx := 0; idx < len(mids); idx++ {
var (
res map[string]string
)
if res, err = redis.StringMap(conn.Receive()); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
if len(res) > 0 {
olMids = append(olMids, mids[idx])
}
for k, v := range res {
ress[k] = v
}
}
return
}
// AddServerOnline add server online.
func (d *Dao) AddServerOnline(c context.Context, server string, sharding int32, online *model.Online) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
b, _ := json.Marshal(online)
key := keyServerOnline(server)
if err = conn.Send("HSET", key, strconv.FormatInt(int64(sharding), 10), b); err != nil {
log.Error("conn.Send(SET %s,%d) error(%v)", key, sharding, err)
return
}
if err = conn.Send("EXPIRE", key, _redisExpire); err != nil {
log.Error("conn.Send(EXPIRE %s) error(%v)", key, err)
return
}
if err = conn.Flush(); err != nil {
log.Error("conn.Flush() error(%v)", err)
return
}
for i := 0; i < 2; i++ {
if _, err = conn.Receive(); err != nil {
log.Error("conn.Receive() error(%v)", err)
return
}
}
return
}
// ServerOnline get a server online.
func (d *Dao) ServerOnline(c context.Context, server string, shard int) (online *model.Online, err error) {
conn := d.redis.Get(c)
defer conn.Close()
key := keyServerOnline(server)
hashKey := fmt.Sprint(shard)
b, err := redis.Bytes(conn.Do("HGET", key, hashKey))
if err != nil {
if err != redis.ErrNil {
log.Error("conn.Do(HGET %s %s) error(%v)", key, hashKey, err)
} else {
err = nil
}
return
}
online = new(model.Online)
if err = json.Unmarshal(b, online); err != nil {
log.Error("serverOnline json.Unmarshal(%s) error(%v)", b, err)
}
return
}
// DelServerOnline del a server online.
func (d *Dao) DelServerOnline(c context.Context, server string, shard int) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
key := keyServerOnline(server)
hashKey := fmt.Sprint(shard)
if _, err = conn.Do("HDEL", key, hashKey); err != nil {
log.Error("conn.Do(DEL %s) error(%v)", key, err)
}
return
}
// SetServers set servers info.
func (d *Dao) SetServers(c context.Context, srvs []*model.ServerInfo) (err error) {
conn := d.redis.Get(c)
defer conn.Close()
b, _ := json.Marshal(srvs)
if _, err = conn.Do("SET", _keyServers, b); err != nil {
log.Error("conn.Do(SET %s,%s) error(%v)", _keyServers, b, err)
}
return
}
// Servers return servers.
func (d *Dao) Servers(c context.Context) (srvs []*model.ServerInfo, err error) {
conn := d.redis.Get(c)
defer conn.Close()
b, err := redis.Bytes(conn.Do("GET", _keyServers))
if err != nil {
if err != redis.ErrNil {
log.Error("conn.Do(GET %s) error(%v)", _keyServers, err)
} else {
err = nil
}
return
}
if err = json.Unmarshal(b, &srvs); err != nil {
log.Error("MigrateServers json.Unmarshal(%s) error(%v)", b, err)
}
return
}
// MigrateServers migrate servers.
func (d *Dao) MigrateServers(c context.Context) (conns, ips int64, err error) {
var servers struct {
Conns int64 `json:"conn_count"`
IPs int64 `json:"ip_count"`
}
conn := d.redis.Get(c)
defer conn.Close()
b, err := redis.Bytes(conn.Do("GET", _keyMigrateServers))
if err != nil {
if err != redis.ErrNil {
log.Error("conn.Do(GET %s) error(%v)", _keyMigrateServers, err)
} else {
err = nil
}
return
}
if err = json.Unmarshal(b, &servers); err != nil {
log.Error("MigrateServers json.Unmarshal(%s) error(%v)", b, err)
return
}
conns = servers.Conns
ips = servers.IPs
return
}
// MigrateRooms migrate rooms.
func (d *Dao) MigrateRooms(c context.Context, shard int) (rooms map[string]int32, err error) {
conn := d.redis.Get(c)
defer conn.Close()
b, err := redis.Bytes(conn.Do("HGET", _keyMigrateRooms, fmt.Sprint(shard)))
if err != nil {
if err != redis.ErrNil {
log.Error("conn.Do(HGET %s,%d) error(%v)", _keyMigrateRooms, shard, err)
} else {
err = nil
}
return
}
rooms = make(map[string]int32)
if err = json.Unmarshal(b, &rooms); err != nil {
log.Error("migrateRooms json.Unmarshal() error(%v)", err)
}
return
}

View File

@@ -0,0 +1,151 @@
package dao
import (
"context"
"testing"
"go-common/app/service/main/broadcast/model"
"github.com/smartystreets/goconvey/convey"
)
func TestDaokeyMidServer(t *testing.T) {
var (
mid = int64(1)
)
convey.Convey("keyMidServer", t, func(ctx convey.C) {
p1 := keyMidServer(mid)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaokeyKeyServer(t *testing.T) {
var (
key = "key"
)
convey.Convey("keyKeyServer", t, func(ctx convey.C) {
p1 := keyKeyServer(key)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaokeyServerOnline(t *testing.T) {
var (
key = "key"
)
convey.Convey("keyServerOnline", t, func(ctx convey.C) {
p1 := keyServerOnline(key)
ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
ctx.So(p1, convey.ShouldNotBeNil)
})
})
}
func TestDaopingRedis(t *testing.T) {
var (
c = context.Background()
)
convey.Convey("pingRedis", t, func(ctx convey.C) {
err := d.pingRedis(c)
ctx.Convey("Then err should be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
})
})
}
func TestDaoAddMapping(t *testing.T) {
var (
c = context.Background()
mid = int64(1)
key = "key"
server = "server"
)
convey.Convey("AddMapping", t, func(ctx convey.C) {
err := d.AddMapping(c, mid, key, server)
ctx.So(err, convey.ShouldBeNil)
has, err := d.ExpireMapping(c, mid, key)
ctx.So(err, convey.ShouldBeNil)
ctx.So(has, convey.ShouldBeTrue)
has, err = d.DelMapping(c, mid, key, server)
ctx.So(err, convey.ShouldBeNil)
ctx.So(has, convey.ShouldBeTrue)
// false
has, err = d.ExpireMapping(c, mid, key)
ctx.So(err, convey.ShouldBeNil)
ctx.So(has, convey.ShouldBeFalse)
has, err = d.DelMapping(c, mid, key, server)
ctx.So(err, convey.ShouldBeNil)
ctx.So(has, convey.ShouldBeFalse)
})
}
func TestDaoServersByKeys(t *testing.T) {
var (
c = context.Background()
keys = []string{"key"}
)
convey.Convey("ServersByKeys", t, func(ctx convey.C) {
res, err := d.ServersByKeys(c, keys)
ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldNotBeNil)
})
})
}
func TestDaoKeysByMids(t *testing.T) {
var (
c = context.Background()
mids = []int64{1, 2, 3}
)
convey.Convey("KeysByMids", t, func(ctx convey.C) {
ress, _, err := d.KeysByMids(c, mids)
ctx.Convey("Then err should be nil.ress should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(ress, convey.ShouldNotBeNil)
})
})
}
func TestDaoAddServerOnline(t *testing.T) {
var (
c = context.Background()
server = "key"
shard = 1
online = &model.Online{RoomCount: map[string]int32{
"test1": 100,
"test2": 200,
"test3": 300,
}, Updated: 1}
)
convey.Convey("AddServerOnline", t, func(ctx convey.C) {
err := d.AddServerOnline(c, server, int32(shard), online)
ctx.So(err, convey.ShouldBeNil)
res, err := d.ServerOnline(c, server, shard)
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldResemble, online)
err = d.DelServerOnline(c, server, shard)
ctx.So(err, convey.ShouldBeNil)
})
}
func TestDaoSetServers(t *testing.T) {
var (
c = context.Background()
servers = []*model.ServerInfo{}
)
convey.Convey("SetServers", t, func(ctx convey.C) {
ctx.So(d.SetServers(c, servers), convey.ShouldBeNil)
res, err := d.Servers(c)
ctx.So(err, convey.ShouldBeNil)
ctx.So(res, convey.ShouldResemble, servers)
_, _, err = d.MigrateServers(c)
ctx.So(err, convey.ShouldBeNil)
_, err = d.MigrateRooms(c, 0)
ctx.So(err, convey.ShouldBeNil)
})
}