Create & Init Project...

app/interface/main/broadcast/server/BUILD

@@ -0,0 +1,79 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucket.go",
+        "channel.go",
+        "errors.go",
+        "noauth.go",
+        "operation.go",
+        "report.go",
+        "ring.go",
+        "room.go",
+        "round.go",
+        "server.go",
+        "server_tcp.go",
+        "server_tcp_v1.go",
+        "server_websocket.go",
+        "server_websocket_v1.go",
+        "whitelist.go",
+    ],
+    importpath = "go-common/app/interface/main/broadcast/server",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/broadcast/conf:go_default_library",
+        "//app/interface/main/broadcast/model:go_default_library",
+        "//app/service/main/broadcast/api/grpc/v1:go_default_library",
+        "//app/service/main/broadcast/libs/bufio:go_default_library",
+        "//app/service/main/broadcast/libs/bytes:go_default_library",
+        "//app/service/main/broadcast/libs/time:go_default_library",
+        "//app/service/main/broadcast/libs/websocket:go_default_library",
+        "//app/service/main/broadcast/model:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/render:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//vendor/github.com/google/uuid:go_default_library",
+        "//vendor/github.com/zhenjl/cityhash:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_gogo_protobuf//types:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//encoding/gzip:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/interface/main/broadcast/server/grpc:all-srcs",
+        "//app/interface/main/broadcast/server/http:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["report_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)

app/interface/main/broadcast/server/bucket.go

@@ -0,0 +1,276 @@
+package server
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"go-common/app/interface/main/broadcast/conf"
+	"go-common/app/service/main/broadcast/model"
+)
+
+// ProtoRoom room proto.
+type ProtoRoom struct {
+	RoomID string
+	Proto  *model.Proto
+}
+
+// Bucket is a channel holder.
+type Bucket struct {
+	c     *conf.Bucket
+	cLock sync.RWMutex        // protect the channels for chs
+	chs   map[string]*Channel // map sub key to a channel
+	// room
+	rooms       map[string]*Room // bucket room channels
+	routines    []chan ProtoRoom
+	routinesNum uint64
+
+	ipCnts     map[string]int32
+	roomIPCnts map[string]int32
+}
+
+// NewBucket new a bucket struct. store the key with im channel.
+func NewBucket(c *conf.Bucket) (b *Bucket) {
+	b = new(Bucket)
+	b.chs = make(map[string]*Channel, c.Channel)
+	b.ipCnts = make(map[string]int32)
+	b.roomIPCnts = make(map[string]int32)
+	b.c = c
+	b.rooms = make(map[string]*Room, c.Room)
+	b.routines = make([]chan ProtoRoom, c.RoutineAmount)
+	for i := uint64(0); i < c.RoutineAmount; i++ {
+		c := make(chan ProtoRoom, c.RoutineSize)
+		b.routines[i] = c
+		go b.roomproc(c)
+	}
+	return
+}
+
+// ChannelCount channel count in the bucket
+func (b *Bucket) ChannelCount() int {
+	return len(b.chs)
+}
+
+// RoomCount room count in the bucket
+func (b *Bucket) RoomCount() int {
+	return len(b.rooms)
+}
+
+// RoomsCount get all room id where online number > 0.
+func (b *Bucket) RoomsCount() (res map[string]int32) {
+	var (
+		roomID string
+		room   *Room
+	)
+	b.cLock.RLock()
+	res = make(map[string]int32)
+	for roomID, room = range b.rooms {
+		if room.Online > 0 {
+			res[roomID] = room.Online
+		}
+	}
+	b.cLock.RUnlock()
+	return
+}
+
+// ChangeRoom change ro room
+func (b *Bucket) ChangeRoom(nrid string, ch *Channel) (err error) {
+	var (
+		nroom *Room
+		ok    bool
+		oroom = ch.Room
+	)
+	// change to no room
+	if nrid == model.NoRoom {
+		if oroom != nil && oroom.Del(ch) {
+			b.DelRoom(oroom)
+		}
+		ch.Room = nil
+		return
+	}
+	b.cLock.Lock()
+	if nroom, ok = b.rooms[nrid]; !ok {
+		nroom = NewRoom(nrid)
+		b.rooms[nrid] = nroom
+	}
+	b.cLock.Unlock()
+	if err = nroom.Put(ch); err != nil {
+		return
+	}
+	ch.Room = nroom
+	if oroom != nil && oroom.Del(ch) {
+		b.DelRoom(oroom)
+	}
+	return
+}
+
+// Put put a channel according with sub key.
+func (b *Bucket) Put(rid string, ch *Channel) (err error) {
+	var (
+		room *Room
+		ok   bool
+	)
+	b.cLock.Lock()
+	// close old channel
+	if dch := b.chs[ch.Key]; dch != nil {
+		dch.Close()
+	}
+	b.chs[ch.Key] = ch
+	if rid != model.NoRoom && rid != "" {
+		if room, ok = b.rooms[rid]; !ok {
+			room = NewRoom(rid)
+			b.rooms[rid] = room
+		}
+		ch.Room = room
+		b.roomIPCnts[ch.IP]++
+	}
+	b.ipCnts[ch.IP]++
+	b.cLock.Unlock()
+	if room != nil {
+		err = room.Put(ch)
+	}
+	return
+}
+
+// Del delete the channel by sub key.
+func (b *Bucket) Del(dch *Channel) {
+	var (
+		ok   bool
+		ch   *Channel
+		room *Room
+	)
+	b.cLock.Lock()
+	if ch, ok = b.chs[dch.Key]; ok {
+		if room = ch.Room; room != nil {
+			if b.roomIPCnts[ch.IP] > 1 {
+				b.roomIPCnts[ch.IP]--
+			} else {
+				delete(b.roomIPCnts, ch.IP)
+			}
+		}
+		if ch == dch {
+			delete(b.chs, ch.Key)
+		}
+		// ip counter
+		if b.ipCnts[ch.IP] > 1 {
+			b.ipCnts[ch.IP]--
+		} else {
+			delete(b.ipCnts, ch.IP)
+		}
+	}
+	b.cLock.Unlock()
+	if room != nil && room.Del(ch) {
+		// if empty room, must delete from bucket
+		b.DelRoom(room)
+	}
+}
+
+// Channel get a channel by sub key.
+func (b *Bucket) Channel(key string) (ch *Channel) {
+	b.cLock.RLock()
+	ch = b.chs[key]
+	b.cLock.RUnlock()
+	return
+}
+
+// Broadcast push msgs to all channels in the bucket.
+func (b *Bucket) Broadcast(p *model.Proto, op int32, platform string) {
+	var ch *Channel
+	b.cLock.RLock()
+	for _, ch = range b.chs {
+		if !ch.NeedPush(op, platform) {
+			continue
+		}
+		ch.Push(p)
+	}
+	b.cLock.RUnlock()
+}
+
+// Room get a room by roomid.
+func (b *Bucket) Room(rid string) (room *Room) {
+	b.cLock.RLock()
+	room = b.rooms[rid]
+	b.cLock.RUnlock()
+	return
+}
+
+// DelRoom delete a room by roomid.
+func (b *Bucket) DelRoom(room *Room) {
+	b.cLock.Lock()
+	delete(b.rooms, room.ID)
+	b.cLock.Unlock()
+	room.Close()
+}
+
+// BroadcastRoom broadcast a message to specified room
+func (b *Bucket) BroadcastRoom(arg ProtoRoom) {
+	num := atomic.AddUint64(&b.routinesNum, 1) % b.c.RoutineAmount
+	b.routines[num] <- arg
+}
+
+// Rooms get all room id where online number > 0.
+func (b *Bucket) Rooms() (res map[string]struct{}) {
+	var (
+		roomID string
+		room   *Room
+	)
+	res = make(map[string]struct{})
+	b.cLock.RLock()
+	for roomID, room = range b.rooms {
+		if room.Online > 0 {
+			res[roomID] = struct{}{}
+		}
+	}
+	b.cLock.RUnlock()
+	return
+}
+
+// IPCount get ip count.
+func (b *Bucket) IPCount() (res map[string]struct{}) {
+	var (
+		ip string
+	)
+	b.cLock.RLock()
+	res = make(map[string]struct{}, len(b.ipCnts))
+	for ip = range b.ipCnts {
+		res[ip] = struct{}{}
+	}
+	b.cLock.RUnlock()
+	return
+}
+
+// RoomIPCount get ip count.
+func (b *Bucket) RoomIPCount() (res map[string]struct{}) {
+	var (
+		ip string
+	)
+	b.cLock.RLock()
+	res = make(map[string]struct{}, len(b.roomIPCnts))
+	for ip = range b.roomIPCnts {
+		res[ip] = struct{}{}
+	}
+	b.cLock.RUnlock()
+	return
+}
+
+// UpRoomsCount update all room count
+func (b *Bucket) UpRoomsCount(roomCountMap map[string]int32) {
+	var (
+		roomID string
+		room   *Room
+	)
+	b.cLock.RLock()
+	for roomID, room = range b.rooms {
+		room.AllOnline = roomCountMap[roomID]
+	}
+	b.cLock.RUnlock()
+}
+
+// roomproc
+func (b *Bucket) roomproc(c chan ProtoRoom) {
+	for {
+		arg := <-c
+		if room := b.Room(arg.RoomID); room != nil {
+			room.Push(arg.Proto)
+		}
+	}
+}

app/interface/main/broadcast/server/channel.go

@@ -0,0 +1,101 @@
+package server
+
+import (
+	"sync"
+
+	"go-common/app/service/main/broadcast/libs/bufio"
+	"go-common/app/service/main/broadcast/model"
+)
+
+// Channel used by message pusher send msg to write goroutine.
+type Channel struct {
+	Room     *Room
+	CliProto Ring
+	signal   chan *model.Proto
+	Writer   bufio.Writer
+	Reader   bufio.Reader
+	Next     *Channel
+	Prev     *Channel
+
+	Mid      int64
+	Key      string
+	IP       string
+	Platform string
+	watchOps map[int32]struct{}
+	mutex    sync.RWMutex
+	V1       bool
+}
+
+// NewChannel new a channel.
+func NewChannel(cli, svr int) *Channel {
+	c := new(Channel)
+	c.CliProto.Init(cli)
+	c.signal = make(chan *model.Proto, svr)
+	c.watchOps = make(map[int32]struct{})
+	return c
+}
+
+// Watch watch a operation.
+func (c *Channel) Watch(accepts ...int32) {
+	c.mutex.Lock()
+	for _, op := range accepts {
+		if op >= model.MinBusinessOp && op <= model.MaxBusinessOp {
+			c.watchOps[op] = struct{}{}
+		}
+	}
+	c.mutex.Unlock()
+}
+
+// UnWatch unwatch an operation
+func (c *Channel) UnWatch(accepts ...int32) {
+	c.mutex.Lock()
+	for _, op := range accepts {
+		delete(c.watchOps, op)
+	}
+	c.mutex.Unlock()
+}
+
+// NeedPush verify if in watch.
+func (c *Channel) NeedPush(op int32, platform string) bool {
+	if c.Platform != platform && platform != "" {
+		return false
+	}
+	if op >= 0 && op < model.MinBusinessOp {
+		return true
+	}
+	c.mutex.RLock()
+	if _, ok := c.watchOps[op]; ok {
+		c.mutex.RUnlock()
+		return true
+	}
+	c.mutex.RUnlock()
+	return false
+}
+
+// Push server push message.
+func (c *Channel) Push(p *model.Proto) (err error) {
+	// NOTE: 兼容v1弹幕推送，等播放器接后可以去掉
+	if c.V1 && p.Operation != 5 {
+		return
+	}
+	select {
+	case c.signal <- p:
+	default:
+	}
+	return
+}
+
+// Ready check the channel ready or close?
+func (c *Channel) Ready() *model.Proto {
+	return <-c.signal
+}
+
+// Signal send signal to the channel, protocol ready.
+func (c *Channel) Signal() {
+	c.signal <- model.ProtoReady
+}
+
+// Close close the channel.
+func (c *Channel) Close() {
+	c.signal <- model.ProtoFinish
+}

app/interface/main/broadcast/server/errors.go

@@ -0,0 +1,32 @@
+package server
+
+import (
+	"errors"
+)
+
+// .
+var (
+	// server
+	ErrHandshake = errors.New("handshake failed")
+	ErrOperation = errors.New("request operation not valid")
+	// ring
+	ErrRingEmpty = errors.New("ring buffer empty")
+	ErrRingFull  = errors.New("ring buffer full")
+	// timer
+	ErrTimerFull   = errors.New("timer full")
+	ErrTimerEmpty  = errors.New("timer empty")
+	ErrTimerNoItem = errors.New("timer item not exist")
+	// channel
+	ErrPushMsgArg   = errors.New("rpc pushmsg arg error")
+	ErrPushMsgsArg  = errors.New("rpc pushmsgs arg error")
+	ErrMPushMsgArg  = errors.New("rpc mpushmsg arg error")
+	ErrMPushMsgsArg = errors.New("rpc mpushmsgs arg error")
+	// bucket
+	ErrBroadCastArg     = errors.New("rpc broadcast arg error")
+	ErrBroadCastRoomArg = errors.New("rpc broadcast  room arg error")
+
+	// room
+	ErrRoomDroped = errors.New("room droped")
+	// rpc
+	ErrLogic = errors.New("logic rpc is not available")
+)

app/interface/main/broadcast/server/grpc/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "go-common/app/interface/main/broadcast/server/grpc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/broadcast/api/grpc/v1:go_default_library",
+        "//app/interface/main/broadcast/conf:go_default_library",
+        "//app/interface/main/broadcast/server:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/broadcast/server/grpc/server.go

@@ -0,0 +1,105 @@
+// Package server generate by warden_gen
+package server
+
+import (
+	"context"
+	"time"
+
+	pb "go-common/app/interface/main/broadcast/api/grpc/v1"
+	"go-common/app/interface/main/broadcast/conf"
+	service "go-common/app/interface/main/broadcast/server"
+	"go-common/library/net/rpc/warden"
+)
+
+// New boradcast grpc server.
+func New(c *conf.Config, svr *service.Server) (ws *warden.Server) {
+	var (
+		err error
+	)
+	ws = warden.NewServer(c.WardenServer)
+	pb.RegisterZergServer(ws.Server(), &server{svr})
+	if ws, err = ws.Start(); err != nil {
+		panic(err)
+	}
+	return
+}
+
+type server struct {
+	srv *service.Server
+}
+
+var _ pb.ZergServer = &server{}
+
+// Ping Service
+func (s *server) Ping(ctx context.Context, req *pb.Empty) (*pb.Empty, error) {
+	return &pb.Empty{}, nil
+}
+
+// Close Service
+func (s *server) Close(ctx context.Context, req *pb.Empty) (*pb.Empty, error) {
+	// TODO: some graceful close
+	return &pb.Empty{}, nil
+}
+
+// PushMsg push a message to specified sub keys.
+func (s *server) PushMsg(ctx context.Context, req *pb.PushMsgReq) (reply *pb.PushMsgReply, err error) {
+	if len(req.Keys) == 0 || req.Proto == nil {
+		return nil, service.ErrPushMsgArg
+	}
+	for _, key := range req.Keys {
+		if channel := s.srv.Bucket(key).Channel(key); channel != nil {
+			if !channel.NeedPush(req.ProtoOp, "") {
+				continue
+			}
+			if err = channel.Push(req.Proto); err != nil {
+				return
+			}
+		}
+	}
+	return &pb.PushMsgReply{}, nil
+}
+
+// Broadcast broadcast msg to all user.
+func (s *server) Broadcast(ctx context.Context, req *pb.BroadcastReq) (*pb.BroadcastReply, error) {
+	if req.Proto == nil {
+		return nil, service.ErrBroadCastArg
+	}
+	go func() {
+		for _, bucket := range s.srv.Buckets() {
+			bucket.Broadcast(req.GetProto(), req.ProtoOp, req.Platform)
+			if req.Speed > 0 {
+				t := bucket.ChannelCount() / int(req.Speed)
+				time.Sleep(time.Duration(t) * time.Second)
+			}
+		}
+	}()
+
+	return &pb.BroadcastReply{}, nil
+}
+
+// BroadcastRoom broadcast msg to specified room.
+func (s *server) BroadcastRoom(ctx context.Context, req *pb.BroadcastRoomReq) (*pb.BroadcastRoomReply, error) {
+	if req.Proto == nil || req.RoomID == "" {
+		return nil, service.ErrBroadCastRoomArg
+	}
+	for _, bucket := range s.srv.Buckets() {
+		bucket.BroadcastRoom(service.ProtoRoom{
+			RoomID: req.RoomID,
+			Proto:  req.Proto,
+		})
+	}
+	return &pb.BroadcastRoomReply{}, nil
+}
+
+// Rooms gets all the room ids for the server.
+func (s *server) Rooms(ctx context.Context, req *pb.RoomsReq) (*pb.RoomsReply, error) {
+	var (
+		roomIds = make(map[string]bool)
+	)
+	for _, bucket := range s.srv.Buckets() {
+		for roomID := range bucket.Rooms() {
+			roomIds[roomID] = true
+		}
+	}
+	return &pb.RoomsReply{Rooms: roomIds}, nil
+}

app/interface/main/broadcast/server/http/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/interface/main/broadcast/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/broadcast/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/broadcast/server/http/http.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"go-common/app/interface/main/broadcast/conf"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// Init init http.
+func Init(c *conf.Config) {
+	engine := bm.DefaultServer(c.HTTP)
+	outerRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+func outerRouter(e *bm.Engine) {
+	e.Ping(ping)
+}
+
+func ping(c *bm.Context) {
+
+}

app/interface/main/broadcast/server/noauth.go

@@ -0,0 +1,46 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"go-common/library/log"
+)
+
+func encodeRoomID(aid, cid int64) string {
+	return fmt.Sprintf("video://%d/%d", aid, cid)
+}
+
+// NoAuthParam .
+type NoAuthParam struct {
+	Key    string `json:"key,omitempty"`
+	Aid    int64  `json:"aid,omitempty"`
+	RoomID int64  `json:"roomid,omitempty"`
+	UserID int64  `json:"uid,omitempty"`
+	From   int64  `json:"from,omitempty"`
+}
+
+// NoAuth .
+func (s *Server) NoAuth(ver int16, token []byte, ip string) (userID int64, roomID, key string, rpt *Report, err error) {
+	param := NoAuthParam{}
+	if err = json.Unmarshal(token, &param); err != nil {
+		log.Error("json.Unmarshal(%d, %s) error(%v)", ver, token, err)
+		return
+	}
+	if param.Key != "" {
+		key = param.Key
+	} else {
+		key = s.NextKey()
+	}
+	userID = param.UserID
+	roomID = encodeRoomID(param.Aid, param.RoomID)
+	rpt = &Report{
+		From: param.From,
+		Aid:  param.Aid,
+		Cid:  param.RoomID,
+		Mid:  param.UserID,
+		Key:  key,
+		IP:   ip,
+	}
+	return
+}

app/interface/main/broadcast/server/operation.go

@@ -0,0 +1,216 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	iModel "go-common/app/interface/main/broadcast/model"
+	pb "go-common/app/service/main/broadcast/api/grpc/v1"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/net/http/blademaster/render"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/gogo/protobuf/types"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/encoding/gzip"
+)
+
+const (
+	_apiConnect     = "/x/broadcast/conn/connect"
+	_apiDisconnect  = "/x/broadcast/conn/disconnect"
+	_apiHeartbeat   = "/x/broadcast/conn/heartbeat"
+	_apiRenewOnline = "/x/broadcast/online/renew"
+)
+
+func (s *Server) failedForword(ctx context.Context, url string, req, reply proto.Message) (err error) {
+	var (
+		b       []byte
+		httpReq *http.Request
+		res     = new(render.PB)
+		api     = fmt.Sprintf("%s%s?token=%s", s.c.Broadcast.APIHost, url, s.c.Broadcast.APIToken)
+	)
+	if b, err = proto.Marshal(req); err != nil {
+		return
+	}
+	if httpReq, err = http.NewRequest("POST", api, bytes.NewBuffer(b)); err != nil {
+		return
+	}
+	if err = s.httpCli.PB(ctx, httpReq, res); err != nil {
+		return
+	}
+	if int(res.Code) != ecode.OK.Code() {
+		err = ecode.Int(int(res.Code))
+		return
+	}
+	err = types.UnmarshalAny(res.Data, reply)
+	return
+}
+
+// Connect .
+func (s *Server) Connect(ctx context.Context, p *model.Proto, cookie string) (mid int64, key, rid, platform string, accepts []int32, err error) {
+	var (
+		req = &pb.ConnectReq{
+			Server:    s.serverID,
+			ServerKey: s.NextKey(),
+			Cookie:    cookie,
+			Token:     p.Body,
+		}
+		reply *pb.ConnectReply
+	)
+	if !s.c.Broadcast.Failover {
+		reply, err = s.rpcClient.Connect(ctx, req)
+	}
+	if s.c.Broadcast.Failover || err != nil {
+		reply = new(pb.ConnectReply)
+		if err = s.failedForword(ctx, _apiConnect, req, reply); err != nil {
+			return
+		}
+	}
+	return reply.Mid, reply.Key, reply.RoomID, reply.Platform, reply.Accepts, nil
+}
+
+// Disconnect .
+func (s *Server) Disconnect(ctx context.Context, mid int64, key string) (err error) {
+	var (
+		req = &pb.DisconnectReq{
+			Mid:    mid,
+			Server: s.serverID,
+			Key:    key,
+		}
+		reply *pb.DisconnectReply
+	)
+	if !s.c.Broadcast.Failover {
+		reply, err = s.rpcClient.Disconnect(ctx, req)
+	}
+	if s.c.Broadcast.Failover || err != nil {
+		reply = new(pb.DisconnectReply)
+		if err = s.failedForword(ctx, _apiDisconnect, req, reply); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// Heartbeat .
+func (s *Server) Heartbeat(ctx context.Context, mid int64, key string) (err error) {
+	var (
+		req = &pb.HeartbeatReq{
+			Mid:    mid,
+			Server: s.serverID,
+			Key:    key,
+		}
+		reply *pb.HeartbeatReply
+	)
+	if !s.c.Broadcast.Failover {
+		reply, err = s.rpcClient.Heartbeat(ctx, req)
+	}
+	if s.c.Broadcast.Failover || err != nil {
+		reply = new(pb.HeartbeatReply)
+		if err = s.failedForword(ctx, _apiHeartbeat, req, reply); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// RenewOnline .
+func (s *Server) RenewOnline(ctx context.Context, serverID string, shard int32, rommCount map[string]int32) (allRoom map[string]int32, err error) {
+	var (
+		req = &pb.OnlineReq{
+			Server:    s.serverID,
+			RoomCount: rommCount,
+			Sharding:  shard,
+		}
+		reply *pb.OnlineReply
+	)
+	if !s.c.Broadcast.Failover {
+		for r := 0; r < s.c.Broadcast.OnlineRetries; r++ {
+			if reply, err = s.rpcClient.RenewOnline(ctx, req, grpc.UseCompressor(gzip.Name)); err != nil {
+				time.Sleep(s.backoff.Backoff(r))
+				continue
+			}
+			break
+		}
+	}
+	if s.c.Broadcast.Failover || err != nil {
+		reply = new(pb.OnlineReply)
+		if err = s.failedForword(ctx, _apiRenewOnline, req, reply); err != nil {
+			return
+		}
+	}
+	return reply.RoomCount, nil
+}
+
+// Report .
+func (s *Server) Report(mid int64, proto *model.Proto) (rp *model.Proto, err error) {
+	var (
+		reply *pb.ReceiveReply
+	)
+	if reply, err = s.rpcClient.Receive(context.Background(), &pb.ReceiveReq{
+		Mid:   mid,
+		Proto: proto,
+	}); err != nil {
+		return
+	}
+	return reply.Proto, nil
+}
+
+// Operate .
+func (s *Server) Operate(p *model.Proto, ch *Channel, b *Bucket) error {
+	var err error
+	switch {
+	case p.Operation >= model.MinBusinessOp && p.Operation <= model.MaxBusinessOp:
+		_, err = s.Report(ch.Mid, p)
+		if err != nil {
+			log.Error("s.Reprot(%d,%v) error(%v)", ch.Mid, p, err)
+			return nil
+		}
+		p.Body = nil
+		// ignore down message
+	case p.Operation == model.OpChangeRoom:
+		p.Operation = model.OpChangeRoomReply
+		var req iModel.ChangeRoomReq
+		if err = json.Unmarshal(p.Body, &req); err == nil {
+			if err = b.ChangeRoom(req.RoomID, ch); err == nil {
+				p.Body = iModel.Message(map[string]interface{}{"room_id": string(p.Body)}, nil)
+			}
+		}
+	case p.Operation == model.OpRegister:
+		p.Operation = model.OpRegisterReply
+		var req iModel.RegisterOpReq
+		if err = json.Unmarshal(p.Body, &req); err == nil {
+			if len(req.Operations) > 0 {
+				ch.Watch(req.Operations...)
+				p.Body = iModel.Message(map[string]interface{}{"operations": req.Operations}, nil)
+			} else {
+				ch.Watch(req.Operation)
+				p.Body = iModel.Message(map[string]interface{}{"operation": req.Operation}, nil)
+			}
+		}
+	case p.Operation == model.OpUnregister:
+		p.Operation = model.OpUnregisterReply
+		var req iModel.UnregisterOpReq
+		if err = json.Unmarshal(p.Body, &req); err == nil {
+			if len(req.Operations) > 0 {
+				ch.UnWatch(req.Operations...)
+				p.Body = iModel.Message(map[string]interface{}{"operations": req.Operations}, nil)
+			} else {
+				ch.UnWatch(req.Operation)
+				p.Body = iModel.Message(map[string]interface{}{"operation": req.Operation}, nil)
+			}
+		}
+	default:
+		err = ErrOperation
+	}
+	if err != nil {
+		log.Error("Operate (%+v) failed!err:=%v", p, err)
+		p.Body = iModel.Message(nil, err)
+	}
+	return nil
+}

app/interface/main/broadcast/server/report.go

@@ -0,0 +1,96 @@
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	actionConnect    = 1
+	actionDisconnect = 2
+
+	_apiReport    = "http://dataflow.biliapi.com/log/system"
+	_reportFormat = "001659%d%d|%d|%d|%d|%d|%d|%s|%s|%d"
+)
+
+var (
+	httpCli     = &http.Client{Timeout: 1 * time.Second}
+	_reportSucc = []byte("succeed")
+	_reportOK   = []byte("OK")
+)
+
+// Report is report params.
+type Report struct {
+	From int64
+	Aid  int64
+	Cid  int64
+	Mid  int64
+	Key  string
+	IP   string
+}
+
+func reportCh(action int, ch *Channel) {
+	if ch.Room == nil {
+		return
+	}
+	u, err := url.Parse(ch.Room.ID)
+	if err != nil {
+		return
+	}
+	if u.Scheme != "video" {
+		return
+	}
+	paths := strings.Split(u.Path, "/")
+	if len(paths) < 2 {
+		return
+	}
+	r := &Report{Key: ch.Key, Mid: ch.Mid, IP: ch.IP}
+	r.Aid, _ = strconv.ParseInt(u.Host, 10, 64)
+	r.Cid, _ = strconv.ParseInt(paths[1], 10, 64)
+	switch ch.Platform {
+	case "ios":
+		r.From = 3
+	case "android":
+		r.From = 2
+	default:
+		r.From = 1
+	}
+	report(action, r, ch.Room.OnlineNum())
+}
+
+func report(action int, r *Report, online int32) {
+	timestamp := time.Now().UnixNano() / int64(time.Millisecond)
+	lines := fmt.Sprintf(_reportFormat, timestamp, timestamp, r.From, r.Aid, r.Cid, r.Mid, online, r.Key, r.IP, action)
+	req, err := http.NewRequest("POST", _apiReport, strings.NewReader(lines))
+	if err != nil {
+		return
+	}
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Error("report: httpCli.POST(%s) error(%d)", lines, resp.StatusCode)
+		return
+	}
+	if !bytes.Equal(b, _reportSucc) && !bytes.Equal(b, _reportOK) {
+		log.Error("report error(%s)", b)
+		return
+	}
+	if r.Mid == 19158909 {
+		log.Info("report: line(%s)", lines)
+	}
+}

app/interface/main/broadcast/server/report_test.go

@@ -0,0 +1,8 @@
+package server
+
+import "testing"
+
+func TestReportCh(t *testing.T) {
+	ch := &Channel{Key: "test_key", Mid: 19158909, IP: "127.0.0.1", Platform: "web", Room: &Room{ID: "video://111/222", AllOnline: 100}}
+	reportCh(1, ch)
+}

app/interface/main/broadcast/server/ring.go

@@ -0,0 +1,87 @@
+package server
+
+import (
+	"go-common/app/interface/main/broadcast/conf"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/log"
+)
+
+// Ring .
+type Ring struct {
+	// read
+	rp   uint64
+	num  uint64
+	mask uint64
+	// TODO split cacheline, many cpu cache line size is 64
+	// pad [40]byte
+	// write
+	wp   uint64
+	data []model.Proto
+}
+
+// NewRing .
+func NewRing(num int) *Ring {
+	r := new(Ring)
+	r.init(uint64(num))
+	return r
+}
+
+// Init .
+func (r *Ring) Init(num int) {
+	r.init(uint64(num))
+}
+
+func (r *Ring) init(num uint64) {
+	// 2^N
+	if num&(num-1) != 0 {
+		for num&(num-1) != 0 {
+			num &= (num - 1)
+		}
+		num = num << 1
+	}
+	r.data = make([]model.Proto, num)
+	r.num = num
+	r.mask = r.num - 1
+}
+
+// Get .
+func (r *Ring) Get() (proto *model.Proto, err error) {
+	if r.rp == r.wp {
+		return nil, ErrRingEmpty
+	}
+	proto = &r.data[r.rp&r.mask]
+	return
+}
+
+// GetAdv .
+func (r *Ring) GetAdv() {
+	r.rp++
+	if conf.Conf.Broadcast.Debug {
+		log.Info("ring rp: %d, idx: %d", r.rp, r.rp&r.mask)
+	}
+}
+
+// Set .
+func (r *Ring) Set() (proto *model.Proto, err error) {
+	if r.wp-r.rp >= r.num {
+		return nil, ErrRingFull
+	}
+	proto = &r.data[r.wp&r.mask]
+	return
+}
+
+// SetAdv .
+func (r *Ring) SetAdv() {
+	r.wp++
+	if conf.Conf.Broadcast.Debug {
+		log.Info("ring wp: %d, idx: %d", r.wp, r.wp&r.mask)
+	}
+}
+
+// Reset .
+func (r *Ring) Reset() {
+	r.rp = 0
+	r.wp = 0
+	// prevent pad compiler optimization
+	// r.pad = [40]byte{}
+}

app/interface/main/broadcast/server/room.go

@@ -0,0 +1,90 @@
+package server
+
+import (
+	"sync"
+
+	"go-common/app/service/main/broadcast/model"
+)
+
+// Room is a room.
+type Room struct {
+	ID        string
+	rLock     sync.RWMutex
+	next      *Channel
+	drop      bool
+	Online    int32 // dirty read is ok
+	AllOnline int32
+}
+
+// NewRoom new a room struct, store channel room info.
+func NewRoom(id string) (r *Room) {
+	r = new(Room)
+	r.ID = id
+	r.drop = false
+	r.next = nil
+	r.AllOnline = roomOnline(id)
+	return
+}
+
+// Put put channel into the room.
+func (r *Room) Put(ch *Channel) (err error) {
+	r.rLock.Lock()
+	if !r.drop {
+		if r.next != nil {
+			r.next.Prev = ch
+		}
+		ch.Next = r.next
+		ch.Prev = nil
+		r.next = ch // insert to header
+		r.Online++
+	} else {
+		err = ErrRoomDroped
+	}
+	r.rLock.Unlock()
+	return
+}
+
+// Del delete channel from the room.
+func (r *Room) Del(ch *Channel) bool {
+	r.rLock.Lock()
+	if ch.Next != nil {
+		// if not footer
+		ch.Next.Prev = ch.Prev
+	}
+	if ch.Prev != nil {
+		// if not header
+		ch.Prev.Next = ch.Next
+	} else {
+		r.next = ch.Next
+	}
+	r.Online--
+	r.drop = (r.Online == 0)
+	r.rLock.Unlock()
+	return r.drop
+}
+
+// Push push msg to the room, if chan full discard it.
+func (r *Room) Push(p *model.Proto) {
+	r.rLock.RLock()
+	for ch := r.next; ch != nil; ch = ch.Next {
+		ch.Push(p)
+	}
+	r.rLock.RUnlock()
+}
+
+// Close close the room.
+func (r *Room) Close() {
+	r.rLock.RLock()
+	for ch := r.next; ch != nil; ch = ch.Next {
+		ch.Close()
+	}
+	r.rLock.RUnlock()
+}
+
+// OnlineNum the room all online.
+func (r *Room) OnlineNum() int32 {
+	if r.AllOnline > 0 {
+		return r.AllOnline
+	}
+	return r.Online
+}

app/interface/main/broadcast/server/round.go

@@ -0,0 +1,75 @@
+package server
+
+import (
+	"go-common/app/interface/main/broadcast/conf"
+	"go-common/app/service/main/broadcast/libs/bytes"
+	"go-common/app/service/main/broadcast/libs/time"
+)
+
+// RoundOptions .
+type RoundOptions struct {
+	Timer        int
+	TimerSize    int
+	Reader       int
+	ReadBuf      int
+	ReadBufSize  int
+	Writer       int
+	WriteBuf     int
+	WriteBufSize int
+}
+
+// Round userd for connection round-robin get a reader/writer/timer for split big lock.
+type Round struct {
+	readers []bytes.Pool
+	writers []bytes.Pool
+	timers  []time.Timer
+	options RoundOptions
+}
+
+// NewRound new a round struct.
+func NewRound(c *conf.Config) (r *Round) {
+	var i int
+	r = new(Round)
+	options := RoundOptions{
+		Reader:       c.TCP.Reader,
+		ReadBuf:      c.TCP.ReadBuf,
+		ReadBufSize:  c.TCP.ReadBufSize,
+		Writer:       c.TCP.Writer,
+		WriteBuf:     c.TCP.WriteBuf,
+		WriteBufSize: c.TCP.WriteBufSize,
+		Timer:        c.Timer.Timer,
+		TimerSize:    c.Timer.TimerSize,
+	}
+	r.options = options
+	// reader
+	r.readers = make([]bytes.Pool, options.Reader)
+	for i = 0; i < options.Reader; i++ {
+		r.readers[i].Init(options.ReadBuf, options.ReadBufSize)
+	}
+	// writer
+	r.writers = make([]bytes.Pool, options.Writer)
+	for i = 0; i < options.Writer; i++ {
+		r.writers[i].Init(options.WriteBuf, options.WriteBufSize)
+	}
+	// timer
+	r.timers = make([]time.Timer, options.Timer)
+	for i = 0; i < options.Timer; i++ {
+		r.timers[i].Init(options.TimerSize)
+	}
+	return
+}
+
+// Timer get a timer.
+func (r *Round) Timer(rn int) *time.Timer {
+	return &(r.timers[rn%r.options.Timer])
+}
+
+// Reader get a reader memory buffer.
+func (r *Round) Reader(rn int) *bytes.Pool {
+	return &(r.readers[rn%r.options.Reader])
+}
+
+// Writer get a writer memory buffer pool.
+func (r *Round) Writer(rn int) *bytes.Pool {
+	return &(r.writers[rn%r.options.Writer])
+}

app/interface/main/broadcast/server/server.go

@@ -0,0 +1,159 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/interface/main/broadcast/conf"
+	pb "go-common/app/service/main/broadcast/api/grpc/v1"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/netutil"
+
+	"github.com/google/uuid"
+	"github.com/zhenjl/cityhash"
+)
+
+var (
+	_maxInt          = 1<<31 - 1
+	_roomOnlineValue atomic.Value
+)
+
+const (
+	_clientHeartbeat    = time.Second * 90
+	_minHeartbeatSecond = 600  // 10m
+	_maxHeartbeatSecond = 1200 // 20m
+	_roomsShard         = 32
+)
+
+// Server .
+type Server struct {
+	c         *conf.Config
+	round     *Round    // accept round store
+	buckets   []*Bucket // subkey bucket
+	bucketIdx uint32
+
+	serverID  string
+	rpcClient pb.ZergClient
+	httpCli   *bm.Client
+	backoff   *netutil.BackoffConfig
+}
+
+// NewServer returns a new Server.
+func NewServer(c *conf.Config) *Server {
+	var err error
+	s := new(Server)
+	s.c = c
+	s.serverID = env.Hostname
+	s.rpcClient, err = pb.NewClient(c.WardenClient)
+	if err != nil {
+		panic(err)
+	}
+	s.httpCli = bm.NewClient(c.HTTPClient)
+	s.round = NewRound(conf.Conf)
+	// init bucket
+	s.buckets = make([]*Bucket, c.Bucket.Size)
+	s.bucketIdx = uint32(c.Bucket.Size)
+	for i := 0; i < c.Bucket.Size; i++ {
+		s.buckets[i] = NewBucket(c.Bucket)
+	}
+	s.backoff = &netutil.BackoffConfig{
+		MaxDelay:  5 * time.Second,
+		BaseDelay: 1.0 * time.Second,
+		Factor:    1.6,
+		Jitter:    0.2,
+	}
+	s.loadOnline()
+	go s.onlineproc()
+	return s
+}
+
+// Buckets return all buckets.
+func (s *Server) Buckets() []*Bucket {
+	return s.buckets
+}
+
+// Bucket get the bucket by subkey.
+func (s *Server) Bucket(subKey string) *Bucket {
+	idx := cityhash.CityHash32([]byte(subKey), uint32(len(subKey))) % s.bucketIdx
+	if s.c.Broadcast.Debug {
+		log.Info("%s hit channel bucket index: %d use cityhash", subKey, idx)
+	}
+	return s.buckets[idx]
+}
+
+// NextKey generate a server key.
+func (s *Server) NextKey() string {
+	u, err := uuid.NewRandom()
+	if err == nil {
+		return u.String()
+	}
+	return fmt.Sprintf("%s-%d", s.serverID, time.Now().UnixNano())
+}
+
+// RandServerHearbeat rand server heartbeat.
+func (s *Server) RandServerHearbeat() time.Duration {
+	return time.Duration(_minHeartbeatSecond+rand.Intn(_maxHeartbeatSecond-_minHeartbeatSecond)) * time.Second
+}
+
+// Close close the server.
+func (s *Server) Close() (err error) {
+	return
+}
+
+func (s *Server) onlineproc() {
+	var retry int
+	for {
+		if err := s.loadOnline(); err != nil {
+			retry++
+			time.Sleep(s.backoff.Backoff(retry))
+			continue
+		}
+		retry = 0
+		time.Sleep(time.Duration(s.c.Broadcast.OnlineTick))
+	}
+}
+
+func (s *Server) loadOnline() (err error) {
+	roomCountShard := make(map[uint32]map[string]int32)
+	for _, bucket := range s.buckets {
+		for roomID, count := range bucket.RoomsCount() {
+			hash := cityhash.CityHash32([]byte(roomID), uint32(len(roomID))) % _roomsShard
+			roomCount, ok := roomCountShard[hash]
+			if !ok {
+				roomCount = make(map[string]int32)
+				roomCountShard[hash] = roomCount
+			}
+			roomCount[roomID] += count
+		}
+	}
+	allRoomsCount := make(map[string]int32)
+	for i := uint32(0); i < _roomsShard; i++ {
+		var mergedRoomsCount map[string]int32
+		mergedRoomsCount, err = s.RenewOnline(context.Background(), s.serverID, int32(i), roomCountShard[i])
+		if err != nil {
+			log.Error("s.RenewOnline(%s, %d, %d) error(%v)", s.serverID, i, len(roomCountShard[i]), err)
+			return
+		}
+		for roomID, count := range mergedRoomsCount {
+			allRoomsCount[roomID] = count
+		}
+	}
+	for _, bucket := range s.buckets {
+		bucket.UpRoomsCount(allRoomsCount)
+	}
+	_roomOnlineValue.Store(allRoomsCount)
+	return
+}
+
+func roomOnline(rid string) int32 {
+	online, ok := _roomOnlineValue.Load().(map[string]int32)
+	if !ok {
+		return 0
+	}
+	return online[rid]
+}

app/interface/main/broadcast/server/server_tcp.go

@@ -0,0 +1,355 @@
+package server
+
+import (
+	"context"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	iModel "go-common/app/interface/main/broadcast/model"
+	"go-common/app/service/main/broadcast/libs/bufio"
+	"go-common/app/service/main/broadcast/libs/bytes"
+	itime "go-common/app/service/main/broadcast/libs/time"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+// InitTCP listen all tcp.bind and start accept connections.
+func InitTCP(server *Server, addrs []string, accept int) (err error) {
+	var (
+		bind     string
+		listener *net.TCPListener
+		addr     *net.TCPAddr
+	)
+	for _, bind = range addrs {
+		if addr, err = net.ResolveTCPAddr("tcp", bind); err != nil {
+			log.Error("net.ResolveTCPAddr(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		if listener, err = net.ListenTCP("tcp", addr); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start tcp listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptTCP(server, listener)
+		}
+	}
+	return
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptTCP(server *Server, lis *net.TCPListener) {
+	var (
+		conn *net.TCPConn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.AcceptTCP(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			return
+		}
+		if err = conn.SetKeepAlive(server.c.TCP.Keepalive); err != nil {
+			log.Error("conn.SetKeepAlive() error(%v)", err)
+			return
+		}
+		if err = conn.SetReadBuffer(server.c.TCP.Rcvbuf); err != nil {
+			log.Error("conn.SetReadBuffer() error(%v)", err)
+			return
+		}
+		if err = conn.SetWriteBuffer(server.c.TCP.Sndbuf); err != nil {
+			log.Error("conn.SetWriteBuffer() error(%v)", err)
+			return
+		}
+		go serveTCP(server, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+func serveTCP(s *Server, conn *net.TCPConn, r int) {
+	var (
+		// timer
+		tr = s.round.Timer(r)
+		rp = s.round.Reader(r)
+		wp = s.round.Writer(r)
+		// ip addr
+		lAddr = conn.LocalAddr().String()
+		rAddr = conn.RemoteAddr().String()
+	)
+	if s.c.Broadcast.Debug {
+		log.Info("start tcp serve \"%s\" with \"%s\"", lAddr, rAddr)
+	}
+	s.ServeTCP(conn, rp, wp, tr)
+}
+
+// ServeTCP .
+func (s *Server) ServeTCP(conn *net.TCPConn, rp, wp *bytes.Pool, tr *itime.Timer) {
+	var (
+		err     error
+		rid     string
+		accepts []int32
+		white   bool
+		p       *model.Proto
+		b       *Bucket
+		trd     *itime.TimerData
+		lastHb  = time.Now()
+		rb      = rp.Get()
+		wb      = wp.Get()
+		ch      = NewChannel(s.c.ProtoSection.CliProto, s.c.ProtoSection.SvrProto)
+		rr      = &ch.Reader
+		wr      = &ch.Writer
+	)
+	ch.Reader.ResetBuffer(conn, rb.Bytes())
+	ch.Writer.ResetBuffer(conn, wb.Bytes())
+	// handshake
+	step := 0
+	trd = tr.Add(time.Duration(s.c.ProtoSection.HandshakeTimeout), func() {
+		conn.Close()
+		log.Error("key: %s remoteIP: %s step: %d tcp handshake timeout", ch.Key, conn.RemoteAddr().String(), step)
+	})
+	ch.IP, _, _ = net.SplitHostPort(conn.RemoteAddr().String())
+	// must not setadv, only used in auth
+	step = 1
+	md := metadata.MD{
+		metadata.RemoteIP: ch.IP,
+	}
+	ctx := metadata.NewContext(context.Background(), md)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	if p, err = ch.CliProto.Set(); err == nil {
+		if ch.Mid, ch.Key, rid, ch.Platform, accepts, err = s.authTCP(ctx, rr, wr, p); err == nil {
+			ch.Watch(accepts...)
+			b = s.Bucket(ch.Key)
+			err = b.Put(rid, ch)
+			if s.c.Broadcast.Debug {
+				log.Info("tcp connnected key:%s mid:%d proto:%+v", ch.Key, ch.Mid, p)
+			}
+		}
+	}
+	step = 2
+	if err != nil {
+		conn.Close()
+		rp.Put(rb)
+		wp.Put(wb)
+		tr.Del(trd)
+		log.Error("key: %s handshake failed error(%v)", ch.Key, err)
+		return
+	}
+	trd.Key = ch.Key
+	tr.Set(trd, _clientHeartbeat)
+	white = whitelist.Contains(ch.Mid)
+	if white {
+		whitelist.Printf("key: %s[%s] auth\n", ch.Key, rid)
+	}
+	step = 3
+	reportCh(actionConnect, ch)
+	// hanshake ok start dispatch goroutine
+	go s.dispatchTCP(conn, wr, wp, wb, ch)
+	serverHeartbeat := s.RandServerHearbeat()
+	for {
+		if p, err = ch.CliProto.Set(); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s start read proto\n", ch.Key)
+		}
+		if err = p.ReadTCP(rr); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s read proto:%v\n", ch.Key, p)
+		}
+		if p.Operation == model.OpHeartbeat {
+			tr.Set(trd, _clientHeartbeat)
+			p.Body = nil
+			p.Operation = model.OpHeartbeatReply
+			// last server heartbeat
+			if now := time.Now(); now.Sub(lastHb) > serverHeartbeat {
+				if err = s.Heartbeat(ctx, ch.Mid, ch.Key); err == nil {
+					lastHb = now
+				} else {
+					err = nil
+				}
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("tcp heartbeat receive key:%s, mid:%d", ch.Key, ch.Mid)
+			}
+			step++
+		} else {
+			if err = s.Operate(p, ch, b); err != nil {
+				break
+			}
+		}
+		if white {
+			whitelist.Printf("key: %s process proto:%v\n", ch.Key, p)
+		}
+		ch.CliProto.SetAdv()
+		ch.Signal()
+		if white {
+			whitelist.Printf("key: %s signal\n", ch.Key)
+		}
+	}
+	if white {
+		whitelist.Printf("key: %s server tcp error(%v)\n", ch.Key, err)
+	}
+	if err != nil && err != io.EOF && !strings.Contains(err.Error(), "closed") {
+		log.Error("key: %s server tcp failed error(%v)", ch.Key, err)
+	}
+	b.Del(ch)
+	tr.Del(trd)
+	rp.Put(rb)
+	conn.Close()
+	ch.Close()
+	if err = s.Disconnect(ctx, ch.Mid, ch.Key); err != nil {
+		log.Error("key: %s operator do disconnect error(%v)", ch.Key, err)
+	}
+	if white {
+		whitelist.Printf("key: %s disconnect error(%v)\n", ch.Key, err)
+	}
+	reportCh(actionDisconnect, ch)
+	if s.c.Broadcast.Debug {
+		log.Info("tcp disconnected key: %s mid:%d", ch.Key, ch.Mid)
+	}
+}
+
+// dispatch accepts connections on the listener and serves requests
+// for each incoming connection.  dispatch blocks; the caller typically
+// invokes it in a go statement.
+func (s *Server) dispatchTCP(conn *net.TCPConn, wr *bufio.Writer, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
+	var (
+		err    error
+		finish bool
+		online int32
+		white  = whitelist.Contains(ch.Mid)
+	)
+	if s.c.Broadcast.Debug {
+		log.Info("key: %s start dispatch tcp goroutine", ch.Key)
+	}
+	for {
+		if white {
+			whitelist.Printf("key: %s wait proto ready\n", ch.Key)
+		}
+		var p = ch.Ready()
+		if white {
+			whitelist.Printf("key: %s proto ready\n", ch.Key)
+		}
+		if s.c.Broadcast.Debug {
+			log.Info("key:%s dispatch msg:%v", ch.Key, *p)
+		}
+		switch p {
+		case model.ProtoFinish:
+			if white {
+				whitelist.Printf("key: %s receive proto finish\n", ch.Key)
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("key: %s wakeup exit dispatch goroutine", ch.Key)
+			}
+			finish = true
+			goto failed
+		case model.ProtoReady:
+			// fetch message from svrbox(client send)
+			for {
+				if p, err = ch.CliProto.Get(); err != nil {
+					err = nil // must be empty error
+					break
+				}
+				if white {
+					whitelist.Printf("key: %s start write client proto%v\n", ch.Key, p)
+				}
+				if p.Operation == model.OpHeartbeatReply {
+					if ch.Room != nil {
+						online = ch.Room.OnlineNum()
+						b := map[string]interface{}{"room": map[string]interface{}{"online": online, "room_id": ch.Room.ID}}
+						p.Body = iModel.Message(b, nil)
+					}
+					if err = p.WriteTCPHeart(wr); err != nil {
+						goto failed
+					}
+				} else {
+					if err = p.WriteTCP(wr); err != nil {
+						goto failed
+					}
+				}
+				if white {
+					whitelist.Printf("key: %s write client proto%v\n", ch.Key, p)
+				}
+				p.Body = nil // avoid memory leak
+				ch.CliProto.GetAdv()
+			}
+		default:
+			if white {
+				whitelist.Printf("key: %s start write server proto%v\n", ch.Key, p)
+			}
+			// server send
+			if err = p.WriteTCP(wr); err != nil {
+				goto failed
+			}
+			if white {
+				whitelist.Printf("key: %s write server proto%v\n", ch.Key, p)
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("tcp sent a message key:%s mid:%d proto:%+v", ch.Key, ch.Mid, p)
+			}
+		}
+		if white {
+			whitelist.Printf("key: %s start flush \n", ch.Key)
+		}
+		// only hungry flush response
+		if err = wr.Flush(); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s flush\n", ch.Key)
+		}
+	}
+failed:
+	if white {
+		whitelist.Printf("key: %s dispatch tcp error(%v)\n", ch.Key, err)
+	}
+	if err != nil {
+		log.Error("key: %s dispatch tcp error(%v)", ch.Key, err)
+	}
+	conn.Close()
+	wp.Put(wb)
+	// must ensure all channel message discard, for reader won't blocking Signal
+	for !finish {
+		finish = (ch.Ready() == model.ProtoFinish)
+	}
+	if s.c.Broadcast.Debug {
+		log.Info("key: %s dispatch goroutine exit", ch.Key)
+	}
+}
+
+// auth for goim handshake with client, use rsa & aes.
+func (s *Server) authTCP(ctx context.Context, rr *bufio.Reader, wr *bufio.Writer, p *model.Proto) (mid int64, key string, rid string, platform string, accepts []int32, err error) {
+	if err = p.ReadTCP(rr); err != nil {
+		log.Error("authTCP.ReadTCP(key:%v).err(%v)", key, err)
+		return
+	}
+	if p.Operation != model.OpAuth {
+		log.Warn("auth operation not valid: %d", p.Operation)
+		err = ErrOperation
+		return
+	}
+	if mid, key, rid, platform, accepts, err = s.Connect(ctx, p, ""); err != nil {
+		log.Error("authTCP.Connect(key:%v).err(%v)", key, err)
+		return
+	}
+	p.Body = []byte(`{"code":0,"message":"ok"}`)
+	p.Operation = model.OpAuthReply
+	if err = p.WriteTCP(wr); err != nil {
+		log.Error("authTCP.WriteTCP(key:%v).err(%v)", key, err)
+		return
+	}
+	err = wr.Flush()
+	return
+}

app/interface/main/broadcast/server/server_tcp_v1.go

@@ -0,0 +1,258 @@
+package server
+
+import (
+	"io"
+	"net"
+	"time"
+
+	"go-common/app/service/main/broadcast/libs/bufio"
+	"go-common/app/service/main/broadcast/libs/bytes"
+	itime "go-common/app/service/main/broadcast/libs/time"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/log"
+)
+
+// InitTCPV1 listen all tcp.bind and start accept connections.
+func InitTCPV1(server *Server, addrs []string, accept int) (err error) {
+	var (
+		bind     string
+		listener *net.TCPListener
+		addr     *net.TCPAddr
+	)
+	for _, bind = range addrs {
+		if addr, err = net.ResolveTCPAddr("tcp", bind); err != nil {
+			log.Error("net.ResolveTCPAddr(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		if listener, err = net.ListenTCP("tcp", addr); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start tcp listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptTCPV1(server, listener)
+		}
+	}
+	return
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptTCPV1(server *Server, lis *net.TCPListener) {
+	var (
+		conn *net.TCPConn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.AcceptTCP(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			time.Sleep(time.Second)
+			continue
+		}
+		if err = conn.SetKeepAlive(server.c.TCP.Keepalive); err != nil {
+			log.Error("conn.SetKeepAlive() error(%v)", err)
+			return
+		}
+		if err = conn.SetReadBuffer(server.c.TCP.Rcvbuf); err != nil {
+			log.Error("conn.SetReadBuffer() error(%v)", err)
+			return
+		}
+		if err = conn.SetWriteBuffer(server.c.TCP.Sndbuf); err != nil {
+			log.Error("conn.SetWriteBuffer() error(%v)", err)
+			return
+		}
+		go serveTCPV1(server, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+func serveTCPV1(s *Server, conn *net.TCPConn, r int) {
+	var (
+		// timer
+		tr = s.round.Timer(r)
+		rp = s.round.Reader(r)
+		wp = s.round.Writer(r)
+		// ip addr
+		lAddr = conn.LocalAddr().String()
+		rAddr = conn.RemoteAddr().String()
+	)
+	if s.c.Broadcast.Debug {
+		log.Info("start tcp serve \"%s\" with \"%s\"", lAddr, rAddr)
+	}
+	s.serveTCPV1(conn, rp, wp, tr)
+}
+
+// TODO linger close?
+func (s *Server) serveTCPV1(conn *net.TCPConn, rp, wp *bytes.Pool, tr *itime.Timer) {
+	var (
+		err    error
+		roomID string
+		hb     time.Duration // heartbeat
+		p      *model.Proto
+		b      *Bucket
+		trd    *itime.TimerData
+		rpt    *Report
+		rb     = rp.Get()
+		wb     = wp.Get()
+		ch     = NewChannel(s.c.ProtoSection.CliProto, s.c.ProtoSection.SvrProto)
+		rr     = &ch.Reader
+		wr     = &ch.Writer
+	)
+	ch.Reader.ResetBuffer(conn, rb.Bytes())
+	ch.Writer.ResetBuffer(conn, wb.Bytes())
+	// handshake
+	trd = tr.Add(time.Duration(s.c.ProtoSection.HandshakeTimeout), func() {
+		conn.Close()
+	})
+	ch.V1 = true
+	ch.IP, _, _ = net.SplitHostPort(conn.RemoteAddr().String())
+	// must not setadv, only used in auth
+	if p, err = ch.CliProto.Set(); err == nil {
+		if ch.Key, roomID, ch.Mid, hb, rpt, err = s.authTCPV1(rr, wr, p, ch.IP); err == nil {
+			b = s.Bucket(ch.Key)
+			err = b.Put(roomID, ch)
+		}
+	}
+	if err != nil {
+		if err != io.EOF {
+			log.Error("key: %s ip: %s handshake failed error(%v)", ch.Key, conn.RemoteAddr().String(), err)
+		}
+		conn.Close()
+		rp.Put(rb)
+		wp.Put(wb)
+		tr.Del(trd)
+		return
+	}
+	trd.Key = ch.Key
+	tr.Set(trd, hb)
+	var online int32
+	if ch.Room != nil {
+		online = ch.Room.OnlineNum()
+	}
+	report(actionConnect, rpt, online)
+	// hanshake ok start dispatch goroutine
+	go s.dispatchTCPV1(ch.Key, conn, wr, wp, wb, ch)
+	for {
+		if p, err = ch.CliProto.Set(); err != nil {
+			break
+		}
+		if err = p.ReadTCPV1(rr); err != nil {
+			break
+		}
+		if p.Operation == model.OpHeartbeat {
+			tr.Set(trd, hb)
+			p.Body = nil
+			p.Operation = model.OpHeartbeatReply
+		} else {
+			if err = s.Operate(p, ch, b); err != nil {
+				break
+			}
+		}
+		ch.CliProto.SetAdv()
+		ch.Signal()
+	}
+	if err != nil && err != io.EOF {
+		log.Error("key: %s server tcp failed error(%v)", ch.Key, err)
+	}
+	b.Del(ch)
+	tr.Del(trd)
+	rp.Put(rb)
+	conn.Close()
+	ch.Close()
+	//if err = s.Disconnect(context.Background(), ch.Mid, roomID); err != nil {
+	//	log.Error("key: %s operator do disconnect error(%v)", ch.Key, err)
+	//}
+	if ch.Room != nil {
+		online = ch.Room.OnlineNum()
+	}
+	report(actionDisconnect, rpt, online)
+}
+
+// dispatch accepts connections on the listener and serves requests
+// for each incoming connection.  dispatch blocks; the caller typically
+// invokes it in a go statement.
+func (s *Server) dispatchTCPV1(key string, conn *net.TCPConn, wr *bufio.Writer, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
+	var (
+		err    error
+		finish bool
+		online int32
+	)
+	for {
+		var p = ch.Ready()
+		switch p {
+		case model.ProtoFinish:
+			finish = true
+			goto failed
+		case model.ProtoReady:
+			// fetch message from svrbox(client send)
+			for {
+				if p, err = ch.CliProto.Get(); err != nil {
+					err = nil // must be empty error
+					break
+				}
+				if p.Operation == model.OpHeartbeatReply {
+					if ch.Room != nil {
+						online = ch.Room.OnlineNum()
+					}
+					if err = p.WriteTCPHeartV1(wr, online); err != nil {
+						goto failed
+					}
+				} else {
+					if err = p.WriteTCPV1(wr); err != nil {
+						goto failed
+					}
+				}
+				p.Body = nil // avoid memory leak
+				ch.CliProto.GetAdv()
+			}
+		default:
+			// server send
+			if err = p.WriteTCPV1(wr); err != nil {
+				goto failed
+			}
+		}
+		// only hungry flush response
+		if err = wr.Flush(); err != nil {
+			break
+		}
+	}
+failed:
+	if err != nil {
+		log.Error("key: %s dispatch tcp error(%v)", key, err)
+	}
+	conn.Close()
+	wp.Put(wb)
+	// must ensure all channel message discard, for reader won't blocking Signal
+	for !finish {
+		finish = (ch.Ready() == model.ProtoFinish)
+	}
+}
+
+// auth for goim handshake with client, use rsa & aes.
+func (s *Server) authTCPV1(rr *bufio.Reader, wr *bufio.Writer, p *model.Proto, ip string) (key, roomID string, userID int64, heartbeat time.Duration, rpt *Report, err error) {
+	if err = p.ReadTCPV1(rr); err != nil {
+		return
+	}
+	if p.Operation != model.OpAuth {
+		log.Warn("auth operation not valid: %d", p.Operation)
+		err = ErrOperation
+		return
+	}
+	if userID, roomID, key, rpt, err = s.NoAuth(int16(p.Ver), p.Body, ip); err != nil {
+		return
+	}
+	heartbeat = _clientHeartbeat
+	p.Body = nil
+	p.Operation = model.OpAuthReply
+	if err = p.WriteTCPV1(wr); err != nil {
+		return
+	}
+	err = wr.Flush()
+	return
+}

app/interface/main/broadcast/server/server_websocket.go

@@ -0,0 +1,439 @@
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	iModel "go-common/app/interface/main/broadcast/model"
+	"go-common/app/service/main/broadcast/libs/bytes"
+	itime "go-common/app/service/main/broadcast/libs/time"
+	"go-common/app/service/main/broadcast/libs/websocket"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+// InitWebsocket listen all tcp.bind and start accept connections.
+func InitWebsocket(server *Server, addrs []string, accept int) (err error) {
+	var (
+		bind     string
+		listener *net.TCPListener
+		addr     *net.TCPAddr
+	)
+	for _, bind = range addrs {
+		if addr, err = net.ResolveTCPAddr("tcp", bind); err != nil {
+			log.Error("net.ResolveTCPAddr(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		if listener, err = net.ListenTCP("tcp", addr); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start ws listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptWebsocket(server, listener)
+		}
+	}
+	return
+}
+
+// InitWebsocketWithTLS init websocket with tls.
+func InitWebsocketWithTLS(server *Server, addrs []string, certFile, privateFile string, accept int) (err error) {
+	var (
+		bind     string
+		listener net.Listener
+		cert     tls.Certificate
+		certs    []tls.Certificate
+	)
+	certFiles := strings.Split(certFile, ",")
+	privateFiles := strings.Split(privateFile, ",")
+	for i := range certFiles {
+		cert, err = tls.LoadX509KeyPair(certFiles[i], privateFiles[i])
+		if err != nil {
+			log.Error("Error loading certificate. error(%v)", err)
+			return
+		}
+		certs = append(certs, cert)
+	}
+	tlsCfg := &tls.Config{Certificates: certs}
+	tlsCfg.BuildNameToCertificate()
+	for _, bind = range addrs {
+		if listener, err = tls.Listen("tcp", bind, tlsCfg); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start wss listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptWebsocketWithTLS(server, listener)
+		}
+	}
+	return
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptWebsocket(server *Server, lis *net.TCPListener) {
+	var (
+		conn *net.TCPConn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.AcceptTCP(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			return
+		}
+		if err = conn.SetKeepAlive(server.c.TCP.Keepalive); err != nil {
+			log.Error("conn.SetKeepAlive() error(%v)", err)
+			return
+		}
+		if err = conn.SetReadBuffer(server.c.TCP.Rcvbuf); err != nil {
+			log.Error("conn.SetReadBuffer() error(%v)", err)
+			return
+		}
+		if err = conn.SetWriteBuffer(server.c.TCP.Sndbuf); err != nil {
+			log.Error("conn.SetWriteBuffer() error(%v)", err)
+			return
+		}
+		go serveWebsocket(server, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptWebsocketWithTLS(server *Server, lis net.Listener) {
+	var (
+		conn net.Conn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.Accept(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			return
+		}
+		go serveWebsocket(server, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+func serveWebsocket(s *Server, conn net.Conn, r int) {
+	var (
+		// timer
+		tr = s.round.Timer(r)
+		rp = s.round.Reader(r)
+		wp = s.round.Writer(r)
+	)
+	if s.c.Broadcast.Debug {
+		// ip addr
+		lAddr := conn.LocalAddr().String()
+		rAddr := conn.RemoteAddr().String()
+		log.Info("start tcp serve \"%s\" with \"%s\"", lAddr, rAddr)
+	}
+	s.ServeWebsocket(conn, rp, wp, tr)
+}
+
+// ServeWebsocket .
+func (s *Server) ServeWebsocket(conn net.Conn, rp, wp *bytes.Pool, tr *itime.Timer) {
+	var (
+		err     error
+		accepts []int32
+		rid     string
+		white   bool
+		p       *model.Proto
+		b       *Bucket
+		trd     *itime.TimerData
+		lastHB  = time.Now()
+		rb      = rp.Get()
+		ch      = NewChannel(s.c.ProtoSection.CliProto, s.c.ProtoSection.SvrProto)
+		rr      = &ch.Reader
+		wr      = &ch.Writer
+		ws      *websocket.Conn // websocket
+		req     *websocket.Request
+	)
+	// reader
+	ch.Reader.ResetBuffer(conn, rb.Bytes())
+	// handshake
+	step := 0
+	trd = tr.Add(time.Duration(s.c.ProtoSection.HandshakeTimeout), func() {
+		conn.SetDeadline(time.Now().Add(time.Millisecond * 100))
+		conn.Close()
+		log.Error("key: %s remoteIP: %s step: %d ws handshake timeout", ch.Key, conn.RemoteAddr().String(), step)
+	})
+	// websocket
+	ch.IP, _, _ = net.SplitHostPort(conn.RemoteAddr().String())
+	step = 1
+	if req, err = websocket.ReadRequest(rr); err != nil || req.RequestURI != "/sub" {
+		conn.Close()
+		tr.Del(trd)
+		rp.Put(rb)
+		if err != io.EOF {
+			log.Error("http.ReadRequest(rr) error(%v)", err)
+		}
+		return
+	}
+	// writer
+	wb := wp.Get()
+	ch.Writer.ResetBuffer(conn, wb.Bytes())
+	step = 2
+	if ws, err = websocket.Upgrade(conn, rr, wr, req); err != nil {
+		conn.Close()
+		tr.Del(trd)
+		rp.Put(rb)
+		wp.Put(wb)
+		if err != io.EOF {
+			log.Error("websocket.NewServerConn error(%v)", err)
+		}
+		return
+	}
+	// must not setadv, only used in auth
+	step = 3
+	md := metadata.MD{
+		metadata.RemoteIP: ch.IP,
+	}
+	ctx := metadata.NewContext(context.Background(), md)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	if p, err = ch.CliProto.Set(); err == nil {
+		if ch.Mid, ch.Key, rid, ch.Platform, accepts, err = s.authWebsocket(ctx, ws, p, req.Header.Get("Cookie")); err == nil {
+			ch.Watch(accepts...)
+			b = s.Bucket(ch.Key)
+			err = b.Put(rid, ch)
+			if s.c.Broadcast.Debug {
+				log.Info("websocket connnected key:%s mid:%d proto:%+v", ch.Key, ch.Mid, p)
+			}
+		}
+	}
+	step = 4
+	if err != nil {
+		ws.Close()
+		rp.Put(rb)
+		wp.Put(wb)
+		tr.Del(trd)
+		if err != io.EOF && err != websocket.ErrMessageClose {
+			log.Error("key: %s remoteIP: %s step: %d ws handshake failed error(%v)", ch.Key, conn.RemoteAddr().String(), step, err)
+		}
+		return
+	}
+	trd.Key = ch.Key
+	tr.Set(trd, _clientHeartbeat)
+	white = whitelist.Contains(ch.Mid)
+	if white {
+		whitelist.Printf("key: %s[%s] auth\n", ch.Key, rid)
+	}
+	// hanshake ok start dispatch goroutine
+	step = 5
+	reportCh(actionConnect, ch)
+	go s.dispatchWebsocket(ws, wp, wb, ch)
+	serverHeartbeat := s.RandServerHearbeat()
+	for {
+		if p, err = ch.CliProto.Set(); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s start read proto\n", ch.Key)
+		}
+		if err = p.ReadWebsocket(ws); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s read proto:%v\n", ch.Key, p)
+		}
+		if p.Operation == model.OpHeartbeat {
+			tr.Set(trd, _clientHeartbeat)
+			p.Body = nil
+			p.Operation = model.OpHeartbeatReply
+			// last server heartbeat
+			if now := time.Now(); now.Sub(lastHB) > serverHeartbeat {
+				if err = s.Heartbeat(ctx, ch.Mid, ch.Key); err == nil {
+					lastHB = now
+				} else {
+					err = nil
+				}
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("websocket heartbeat receive key:%s, mid:%d", ch.Key, ch.Mid)
+			}
+			step++
+		} else {
+			if err = s.Operate(p, ch, b); err != nil {
+				break
+			}
+		}
+		if white {
+			whitelist.Printf("key: %s process proto:%v\n", ch.Key, p)
+		}
+		ch.CliProto.SetAdv()
+		ch.Signal()
+		if white {
+			whitelist.Printf("key: %s signal\n", ch.Key)
+		}
+	}
+	if white {
+		whitelist.Printf("key: %s server tcp error(%v)\n", ch.Key, err)
+	}
+	if err != nil && err != io.EOF && err != websocket.ErrMessageClose && !strings.Contains(err.Error(), "closed") {
+		log.Error("key: %s server ws failed error(%v)", ch.Key, err)
+	}
+	b.Del(ch)
+	tr.Del(trd)
+	ws.Close()
+	ch.Close()
+	rp.Put(rb)
+	if err = s.Disconnect(ctx, ch.Mid, ch.Key); err != nil {
+		log.Error("key: %s operator do disconnect error(%v)", ch.Key, err)
+	}
+	if white {
+		whitelist.Printf("key: %s disconnect error(%v)\n", ch.Key, err)
+	}
+	reportCh(actionDisconnect, ch)
+	if s.c.Broadcast.Debug {
+		log.Info("websocket disconnected key: %s mid:%d", ch.Key, ch.Mid)
+	}
+}
+
+// dispatch accepts connections on the listener and serves requests
+// for each incoming connection.  dispatch blocks; the caller typically
+// invokes it in a go statement.
+func (s *Server) dispatchWebsocket(ws *websocket.Conn, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
+	var (
+		err    error
+		finish bool
+		online int32
+		white  = whitelist.Contains(ch.Mid)
+	)
+	if s.c.Broadcast.Debug {
+		log.Info("key: %s start dispatch tcp goroutine", ch.Key)
+	}
+	for {
+		if white {
+			whitelist.Printf("key: %s wait proto ready\n", ch.Key)
+		}
+		var p = ch.Ready()
+		if white {
+			whitelist.Printf("key: %s proto ready\n", ch.Key)
+		}
+		if s.c.Broadcast.Debug {
+			log.Info("key:%s dispatch msg:%s", ch.Key, p.Body)
+		}
+		switch p {
+		case model.ProtoFinish:
+			if white {
+				whitelist.Printf("key: %s receive proto finish\n", ch.Key)
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("key: %s wakeup exit dispatch goroutine", ch.Key)
+			}
+			finish = true
+			goto failed
+		case model.ProtoReady:
+			// fetch message from svrbox(client send)
+			for {
+				if p, err = ch.CliProto.Get(); err != nil {
+					err = nil // must be empty error
+					break
+				}
+				if white {
+					whitelist.Printf("key: %s start write client proto%v\n", ch.Key, p)
+				}
+				if p.Operation == model.OpHeartbeatReply {
+					if ch.Room != nil {
+						online = ch.Room.OnlineNum()
+						b := map[string]interface{}{"room": map[string]interface{}{"online": online, "room_id": ch.Room.ID}}
+						p.Body = iModel.Message(b, nil)
+					}
+					if err = p.WriteWebsocketHeart(ws); err != nil {
+						goto failed
+					}
+				} else {
+					if err = p.WriteWebsocket(ws); err != nil {
+						goto failed
+					}
+				}
+				if white {
+					whitelist.Printf("key: %s write client proto%v\n", ch.Key, p)
+				}
+				p.Body = nil // avoid memory leak
+				ch.CliProto.GetAdv()
+			}
+		default:
+			if white {
+				whitelist.Printf("key: %s start write server proto%v\n", ch.Key, p)
+			}
+			if err = p.WriteWebsocket(ws); err != nil {
+				goto failed
+			}
+			if white {
+				whitelist.Printf("key: %s write server proto%v\n", ch.Key, p)
+			}
+			if s.c.Broadcast.Debug {
+				log.Info("websocket sent a message key:%s mid:%d proto:%+v", ch.Key, ch.Mid, p)
+			}
+		}
+		if white {
+			whitelist.Printf("key: %s start flush \n", ch.Key)
+		}
+		// only hungry flush response
+		if err = ws.Flush(); err != nil {
+			break
+		}
+		if white {
+			whitelist.Printf("key: %s flush\n", ch.Key)
+		}
+	}
+failed:
+	if white {
+		whitelist.Printf("key: %s dispatch tcp error(%v)\n", ch.Key, err)
+	}
+	if err != nil && err != io.EOF && err != websocket.ErrMessageClose {
+		log.Error("key: %s dispatch ws error(%v)", ch.Key, err)
+	}
+	ws.Close()
+	wp.Put(wb)
+	// must ensure all channel message discard, for reader won't blocking Signal
+	for !finish {
+		finish = (ch.Ready() == model.ProtoFinish)
+	}
+	if s.c.Broadcast.Debug {
+		log.Info("key: %s dispatch goroutine exit", ch.Key)
+	}
+}
+
+// auth for goim handshake with client, use rsa & aes.
+func (s *Server) authWebsocket(ctx context.Context, ws *websocket.Conn, p *model.Proto, cookie string) (mid int64, key string, rid string, platform string, accepts []int32, err error) {
+	for {
+		if err = p.ReadWebsocket(ws); err != nil {
+			return
+		}
+		if p.Operation == model.OpAuth {
+			break
+		} else {
+			log.Error("ws request operation(%d) not auth", p.Operation)
+		}
+	}
+	if mid, key, rid, platform, accepts, err = s.Connect(ctx, p, cookie); err != nil {
+		return
+	}
+	p.Body = []byte(`{"code":0,"message":"ok"}`)
+	p.Operation = model.OpAuthReply
+	if err = p.WriteWebsocket(ws); err != nil {
+		return
+	}
+	err = ws.Flush()
+	return
+}

app/interface/main/broadcast/server/server_websocket_v1.go

@@ -0,0 +1,325 @@
+package server
+
+import (
+	"crypto/tls"
+	"io"
+	"net"
+	"time"
+
+	"go-common/app/service/main/broadcast/libs/bytes"
+	itime "go-common/app/service/main/broadcast/libs/time"
+	"go-common/app/service/main/broadcast/libs/websocket"
+	"go-common/app/service/main/broadcast/model"
+	"go-common/library/log"
+)
+
+// InitWebsocketV1 listen all tcp.bind and start accept connections.
+func InitWebsocketV1(s *Server, addrs []string, accept int) (err error) {
+	var (
+		bind     string
+		listener *net.TCPListener
+		addr     *net.TCPAddr
+	)
+	for _, bind = range addrs {
+		if addr, err = net.ResolveTCPAddr("tcp", bind); err != nil {
+			log.Error("net.ResolveTCPAddr(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		if listener, err = net.ListenTCP("tcp", addr); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start ws listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptWebsocketV1(s, listener)
+		}
+	}
+	return
+}
+
+// InitWebsocketWithTLSV1 .
+func InitWebsocketWithTLSV1(s *Server, addrs []string, certFile, privateFile string, accept int) (err error) {
+	var (
+		bind     string
+		listener net.Listener
+		cert     tls.Certificate
+	)
+	cert, err = tls.LoadX509KeyPair(certFile, privateFile)
+	if err != nil {
+		log.Error("Error loading certificate. error(%v)", err)
+		return
+	}
+	tlsCfg := &tls.Config{Certificates: []tls.Certificate{cert}}
+	for _, bind = range addrs {
+		if listener, err = tls.Listen("tcp", bind, tlsCfg); err != nil {
+			log.Error("net.ListenTCP(\"tcp\", \"%s\") error(%v)", bind, err)
+			return
+		}
+		log.Info("start wss listen: \"%s\"", bind)
+		// split N core accept
+		for i := 0; i < accept; i++ {
+			go acceptWebsocketWithTLSV1(s, listener)
+		}
+	}
+	return
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptWebsocketV1(s *Server, lis *net.TCPListener) {
+	var (
+		conn *net.TCPConn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.AcceptTCP(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			time.Sleep(time.Second)
+			continue
+		}
+		if err = conn.SetKeepAlive(s.c.TCP.Keepalive); err != nil {
+			log.Error("conn.SetKeepAlive() error(%v)", err)
+			return
+		}
+		if err = conn.SetReadBuffer(s.c.TCP.Rcvbuf); err != nil {
+			log.Error("conn.SetReadBuffer() error(%v)", err)
+			return
+		}
+		if err = conn.SetWriteBuffer(s.c.TCP.Sndbuf); err != nil {
+			log.Error("conn.SetWriteBuffer() error(%v)", err)
+			return
+		}
+		go serveWebsocketV1(s, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+// Accept accepts connections on the listener and serves requests
+// for each incoming connection.  Accept blocks; the caller typically
+// invokes it in a go statement.
+func acceptWebsocketWithTLSV1(server *Server, lis net.Listener) {
+	var (
+		conn net.Conn
+		err  error
+		r    int
+	)
+	for {
+		if conn, err = lis.Accept(); err != nil {
+			// if listener close then return
+			log.Error("listener.Accept(\"%s\") error(%v)", lis.Addr().String(), err)
+			return
+		}
+		go serveWebsocketV1(server, conn, r)
+		if r++; r == _maxInt {
+			r = 0
+		}
+	}
+}
+
+func serveWebsocketV1(server *Server, conn net.Conn, r int) {
+	var (
+		// timer
+		tr = server.round.Timer(r)
+		rp = server.round.Reader(r)
+		wp = server.round.Writer(r)
+	)
+	server.serveWebsocketV1(conn, rp, wp, tr)
+}
+
+// TODO linger close?
+func (s *Server) serveWebsocketV1(conn net.Conn, rp, wp *bytes.Pool, tr *itime.Timer) {
+	var (
+		err    error
+		roomID string
+		hb     time.Duration // heartbeat
+		p      *model.Proto
+		b      *Bucket
+		trd    *itime.TimerData
+		rb     = rp.Get()
+		ch     = NewChannel(s.c.ProtoSection.CliProto, s.c.ProtoSection.SvrProto)
+		rr     = &ch.Reader
+		wr     = &ch.Writer
+		ws     *websocket.Conn // websocket
+		req    *websocket.Request
+		rpt    *Report
+	)
+	// reader
+	ch.Reader.ResetBuffer(conn, rb.Bytes())
+	// handshake
+	trd = tr.Add(time.Duration(s.c.ProtoSection.HandshakeTimeout), func() {
+		conn.SetDeadline(time.Now().Add(time.Millisecond))
+		conn.Close()
+	})
+	// websocket
+	if req, err = websocket.ReadRequest(rr); err != nil || req.RequestURI != "/sub" {
+		conn.Close()
+		tr.Del(trd)
+		rp.Put(rb)
+		if err != io.EOF {
+			log.Error("http.ReadRequest(rr) error(%v)", err)
+		}
+		return
+	}
+	// writer
+	wb := wp.Get()
+	ch.Writer.ResetBuffer(conn, wb.Bytes())
+	if ws, err = websocket.Upgrade(conn, rr, wr, req); err != nil {
+		conn.Close()
+		tr.Del(trd)
+		rp.Put(rb)
+		wp.Put(wb)
+		if err != io.EOF {
+			log.Error("websocket.NewServerConn error(%v)", err)
+		}
+		return
+	}
+	ch.V1 = true
+	ch.IP, _, _ = net.SplitHostPort(conn.RemoteAddr().String())
+	// must not setadv, only used in auth
+	if p, err = ch.CliProto.Set(); err == nil {
+		if ch.Key, roomID, ch.Mid, hb, rpt, err = s.authWebsocketV1(ws, p, ch.IP); err == nil {
+			b = s.Bucket(ch.Key)
+			err = b.Put(roomID, ch)
+		}
+	}
+	if err != nil {
+		if err != io.EOF && err != websocket.ErrMessageClose {
+			log.Error("key: %s ip: %s handshake failed error(%v)", ch.Key, conn.RemoteAddr().String(), err)
+		}
+		ws.Close()
+		rp.Put(rb)
+		wp.Put(wb)
+		tr.Del(trd)
+		return
+	}
+	trd.Key = ch.Key
+	tr.Set(trd, hb)
+	var online int32
+	if ch.Room != nil {
+		online = ch.Room.OnlineNum()
+	}
+	report(actionConnect, rpt, online)
+	// hanshake ok start dispatch goroutine
+	go s.dispatchWebsocketV1(ch.Key, ws, wp, wb, ch)
+	for {
+		if p, err = ch.CliProto.Set(); err != nil {
+			break
+		}
+		if err = p.ReadWebsocketV1(ws); err != nil {
+			break
+		}
+		if p.Operation == model.OpHeartbeat {
+			tr.Set(trd, hb)
+			p.Operation = model.OpHeartbeatReply
+		} else {
+			if err = s.Operate(p, ch, b); err != nil {
+				break
+			}
+		}
+		ch.CliProto.SetAdv()
+		ch.Signal()
+	}
+	if err != nil && err != io.EOF && err != websocket.ErrMessageClose {
+		log.Error("key: %s server tcp failed error(%v)", ch.Key, err)
+	}
+	b.Del(ch)
+	tr.Del(trd)
+	ws.Close()
+	ch.Close()
+	rp.Put(rb)
+	//if err = s.Disconnect(context.Background(), ch.Mid, roomID); err != nil {
+	//	log.Error("key: %s operator do disconnect error(%v)", ch.Key, err)
+	//}
+	if ch.Room != nil {
+		online = ch.Room.OnlineNum()
+	}
+	report(actionDisconnect, rpt, online)
+}
+
+// dispatch accepts connections on the listener and serves requests
+// for each incoming connection.  dispatch blocks; the caller typically
+// invokes it in a go statement.
+func (s *Server) dispatchWebsocketV1(key string, ws *websocket.Conn, wp *bytes.Pool, wb *bytes.Buffer, ch *Channel) {
+	var (
+		err    error
+		finish bool
+		online int32
+	)
+	for {
+		var p = ch.Ready()
+		switch p {
+		case model.ProtoFinish:
+			finish = true
+			goto failed
+		case model.ProtoReady:
+			// fetch message from svrbox(client send)
+			for {
+				if p, err = ch.CliProto.Get(); err != nil {
+					err = nil // must be empty error
+					break
+				}
+				if p.Operation == model.OpHeartbeatReply {
+					if ch.Room != nil {
+						online = ch.Room.OnlineNum()
+					}
+					if err = p.WriteWebsocketHeartV1(ws, online); err != nil {
+						goto failed
+					}
+				} else {
+					if err = p.WriteWebsocketV1(ws); err != nil {
+						goto failed
+					}
+				}
+				p.Body = nil // avoid memory leak
+				ch.CliProto.GetAdv()
+			}
+		default:
+			// server send
+			if err = p.WriteWebsocketV1(ws); err != nil {
+				goto failed
+			}
+		}
+		// only hungry flush response
+		if err = ws.Flush(); err != nil {
+			break
+		}
+	}
+failed:
+	if err != nil && err != io.EOF && err != websocket.ErrMessageClose {
+		log.Error("key: %s dispatch tcp error(%v)", key, err)
+	}
+	ws.Close()
+	wp.Put(wb)
+	// must ensure all channel message discard, for reader won't blocking Signal
+	for !finish {
+		finish = (ch.Ready() == model.ProtoFinish)
+	}
+}
+
+// auth for goim handshake with client, use rsa & aes.
+func (s *Server) authWebsocketV1(ws *websocket.Conn, p *model.Proto, ip string) (key, roomID string, userID int64, heartbeat time.Duration, rpt *Report, err error) {
+	if err = p.ReadWebsocketV1(ws); err != nil {
+		return
+	}
+	if p.Operation != model.OpAuth {
+		err = ErrOperation
+		return
+	}
+	if userID, roomID, key, rpt, err = s.NoAuth(int16(p.Ver), p.Body, ip); err != nil {
+		return
+	}
+	heartbeat = _clientHeartbeat
+	p.Body = nil
+	p.Operation = model.OpAuthReply
+	if err = p.WriteWebsocketV1(ws); err != nil {
+		return
+	}
+	err = ws.Flush()
+	return
+}

app/interface/main/broadcast/server/whitelist.go

@@ -0,0 +1,45 @@
+package server
+
+import (
+	"go-common/app/interface/main/broadcast/conf"
+	"log"
+	"os"
+)
+
+var whitelist *Whitelist
+
+// Whitelist .
+type Whitelist struct {
+	log  *log.Logger
+	list map[int64]struct{} // whitelist for debug
+}
+
+// InitWhitelist a whitelist struct.
+func InitWhitelist(c *conf.Whitelist) (err error) {
+	var (
+		mid int64
+		f   *os.File
+	)
+	if f, err = os.OpenFile(c.WhiteLog, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0644); err == nil {
+		whitelist = new(Whitelist)
+		whitelist.log = log.New(f, "", log.LstdFlags)
+		whitelist.list = make(map[int64]struct{})
+		for _, mid = range c.Whitelist {
+			whitelist.list[mid] = struct{}{}
+		}
+	}
+	return
+}
+
+// Contains whitelist contains a mid or not.
+func (w *Whitelist) Contains(mid int64) (ok bool) {
+	if mid > 0 {
+		_, ok = w.list[mid]
+	}
+	return
+}
+
+// Printf calls l.Output to print to the logger.
+func (w *Whitelist) Printf(format string, v ...interface{}) {
+	w.log.Printf(format, v...)
+}