Create & Init Project...

app/service/live/broadcast-proxy/dispatch/BUILD

@@ -0,0 +1,46 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["matcher_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "matcher.go",
+        "min_heap.go",
+        "sinaip.go",
+    ],
+    importpath = "go-common/app/service/live/broadcast-proxy/dispatch",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/live/broadcast-proxy/conf:go_default_library",
+        "//app/service/live/broadcast-proxy/expr:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/ipipdotnet/ipdb-go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/live/broadcast-proxy/dispatch/matcher.go

@@ -0,0 +1,306 @@
+package dispatch
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/ipipdotnet/ipdb-go"
+	"go-common/app/service/live/broadcast-proxy/conf"
+	"go-common/app/service/live/broadcast-proxy/expr"
+	"go-common/library/log"
+	"math"
+	"math/rand"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type Matcher struct {
+	ipDataV4             *ipdb.City
+	ipDataV6             *ipdb.City
+	heapPool             sync.Pool
+	Config               string
+	MaxLimit             int    `json:"ip_max_limit"`
+	DefaultDomain        string `json:"default_domain"`
+	WildcardDomainSuffix string `json:"wildcard_domain_suffix"`
+	CommonDispatch       struct {
+		ChinaDispatch struct {
+			ChinaTelecom *CommonBucket `json:"china_telecom"`
+			ChinaUnicom  *CommonBucket `json:"china_unicom"`
+			CMCC         *CommonBucket `json:"cmcc"`
+			ChinaOther   *CommonBucket `json:"other"`
+		} `json:"china"`
+		OverseaDispatch     []*CommonRuleBucket `json:"oversea"`
+		UnknownAreaDispatch *CommonBucket       `json:"unknown"`
+	} `json:"danmaku_common_dispatch"`
+	VIPDispatch []*VIPRuleBucket    `json:"danmaku_vip_dispatch"`
+	ServerGroup map[string][]string `json:"danmaku_comet_group"`
+	ServerHost  map[string]string   `json:"danmaku_comet_host"`
+	IPBlack     []string            `json:"ip_black"`
+	TempV6      []string            `json:"temp_v6"`
+	forbiddenIP map[string]struct{}
+}
+
+type CommonBucket struct {
+	Master map[string]int `json:"master"`
+	Slave  map[string]int `json:"slave"`
+}
+
+type CommonRuleBucket struct {
+	CommonBucket
+	Rule     string `json:"rule"`
+	RuleExpr expr.Expr
+}
+
+type VIPRuleBucket struct {
+	Rule     string `json:"rule"`
+	RuleExpr expr.Expr
+	IP       []string `json:"ip"`
+	Group    []string `json:"group"`
+}
+
+func NewMatcher(matcherConfig []byte, ipDataV4 *ipdb.City, ipDataV6 *ipdb.City, dispatchConfig *conf.DispatchConfig) (*Matcher, error) {
+	matcher := new(Matcher)
+	matcher.heapPool = sync.Pool{
+		New: func() interface{} {
+			return NewMinHeap()
+		},
+	}
+	matcher.forbiddenIP = make(map[string]struct{})
+	if ipDataV4 == nil || ipDataV6 == nil {
+		return nil, errors.New("invalid IP database")
+	}
+	matcher.ipDataV4 = ipDataV4
+	matcher.ipDataV6 = ipDataV6
+	matcher.Config = string(matcherConfig)
+	if err := json.Unmarshal(matcherConfig, matcher); err != nil {
+		return nil, err
+	}
+	for _, ip := range matcher.IPBlack {
+		matcher.forbiddenIP[ip] = struct{}{}
+	}
+	parser := expr.NewExpressionParser()
+	for _, oversea := range matcher.CommonDispatch.OverseaDispatch {
+		if oversea.Rule == "" {
+			oversea.Rule = "true"
+		}
+		if err := parser.Parse(oversea.Rule); err != nil {
+			log.Error("[Matcher] Parse rule expr:%s, error:%+v", oversea.Rule, err)
+			return nil, err
+		}
+		for _, variable := range parser.GetVariable() {
+			if variable != "$lng" && variable != "$lat" {
+				return nil, errors.New("oversea dispatch only supports variable $lng and $lat")
+			}
+		}
+		oversea.RuleExpr = parser.GetExpr()
+	}
+	for _, vip := range matcher.VIPDispatch {
+		if err := parser.Parse(vip.Rule); err != nil {
+			log.Error("[Matcher] Parse rule expr:%s, error:%+v", vip.Rule, err)
+			return nil, err
+		}
+		for _, variable := range parser.GetVariable() {
+			if variable != "$uid" {
+				return nil, errors.New("vip dispatch only supports variable $uid")
+			}
+		}
+		if len(parser.GetVariable()) == 0 {
+			return nil, errors.New("vip dispatch must contains variable $uid")
+		}
+		vip.RuleExpr = parser.GetExpr()
+	}
+
+	if matcher.MaxLimit == 0 && dispatchConfig != nil {
+		matcher.MaxLimit = dispatchConfig.MaxLimit
+	}
+	if matcher.DefaultDomain == "" && dispatchConfig != nil {
+		matcher.DefaultDomain = dispatchConfig.DefaultDomain
+	}
+	if matcher.WildcardDomainSuffix == "" && dispatchConfig != nil {
+		matcher.WildcardDomainSuffix = dispatchConfig.WildcardDomainSuffix
+	}
+	return matcher, nil
+}
+
+func (matcher *Matcher) GetConfig() string {
+	return matcher.Config
+}
+
+func (matcher *Matcher) Dispatch(ip string, uid int64) ([]string, []string) {
+	danmakuIP := matcher.dispatchInternal(ip, uid)
+	danmakuHost := make([]string, 0, len(danmakuIP))
+	for _, singleDanmakuIP := range danmakuIP {
+		if host, ok := matcher.ServerHost[singleDanmakuIP]; ok {
+			danmakuHost = append(danmakuHost, host+matcher.WildcardDomainSuffix)
+		}
+	}
+	danmakuIP = append(danmakuIP, matcher.DefaultDomain)
+	danmakuHost = append(danmakuHost, matcher.DefaultDomain)
+	return danmakuIP, danmakuHost
+}
+
+func (matcher *Matcher) dispatchInternal(ip string, uid int64) []string {
+	if _, ok := matcher.forbiddenIP[ip]; ok {
+		return []string{}
+	}
+	// VIP Dispatch
+	vipDispatchEnv := make(map[expr.Var]interface{})
+	vipDispatchEnv[expr.Var("$uid")] = uid
+	for _, vip := range matcher.VIPDispatch {
+		if v, err := expr.SafetyEvalBool(vip.RuleExpr, vipDispatchEnv); v && err == nil {
+			return matcher.pickFromVIPRuleBucket(vip)
+		} else {
+			if err != nil {
+				log.Error("[Matcher] VIP dispatch, uid:%d, eval rule expr:%s error:%+v", uid, vip.Rule, err)
+			}
+		}
+	}
+	// Common Dispatch
+	var ipDatabase *ipdb.City
+	for i := 0; i < len(ip); i++ {
+		if ip[i] == '.' {
+			ipDatabase = matcher.ipDataV4
+			break
+		} else if ip[i] == ':' {
+			ipDatabase = matcher.ipDataV6
+			//break
+			//TODO: this is temp solution, replace this block with "break" here when all server supports IPv6
+			return matcher.randomPickN(matcher.TempV6, matcher.MaxLimit)
+		}
+	}
+	if ipDatabase == nil {
+		return matcher.pickFromCommonBucket(matcher.CommonDispatch.UnknownAreaDispatch)
+	}
+
+	detail, err := ipDatabase.FindMap(ip, "EN")
+	if err != nil {
+		return matcher.pickFromCommonBucket(matcher.CommonDispatch.UnknownAreaDispatch)
+	}
+	country := strings.TrimSpace(detail["country_name"])
+	province := strings.TrimSpace(detail["region_name"])
+	isp := strings.TrimSpace(detail["isp_domain"])
+	latitude, _ := strconv.ParseFloat(detail["latitude"], 64)
+	longitude, _ := strconv.ParseFloat(detail["longitude"], 64)
+
+	if country != "China" && country != "Reserved" && country != "LAN Address" && country != "Loopback" {
+		return matcher.pickFromCommonRuleBucket(matcher.CommonDispatch.OverseaDispatch, latitude, longitude)
+	} else if country == "China" {
+		if province == "Hong Kong" || province == "Macau" || province == "Taiwan" {
+			return matcher.pickFromCommonRuleBucket(matcher.CommonDispatch.OverseaDispatch, latitude, longitude)
+		} else {
+			switch isp {
+			case "ChinaTelecom":
+				return matcher.pickFromCommonBucket(matcher.CommonDispatch.ChinaDispatch.ChinaTelecom)
+			case "ChinaMobile":
+				return matcher.pickFromCommonBucket(matcher.CommonDispatch.ChinaDispatch.CMCC)
+			case "ChinaUnicom":
+				return matcher.pickFromCommonBucket(matcher.CommonDispatch.ChinaDispatch.ChinaUnicom)
+			default:
+				return matcher.pickFromCommonBucket(matcher.CommonDispatch.ChinaDispatch.ChinaOther)
+			}
+		}
+	} else {
+		return matcher.pickFromCommonBucket(matcher.CommonDispatch.UnknownAreaDispatch)
+	}
+}
+
+func (matcher *Matcher) pickFromCommonRuleBucket(overseaBucket []*CommonRuleBucket, latitude float64, longitude float64) []string {
+	overseaDispatchEnv := make(map[expr.Var]interface{})
+	overseaDispatchEnv[expr.Var("$lat")] = latitude
+	overseaDispatchEnv[expr.Var("$lng")] = longitude
+	for _, bucket := range overseaBucket {
+		if v, err := expr.SafetyEvalBool(bucket.RuleExpr, overseaDispatchEnv); v && err == nil {
+			return matcher.pickFromCommonBucket(&bucket.CommonBucket)
+		}
+	}
+	return []string{}
+}
+
+func (matcher *Matcher) pickOneFromWeightedGroup(groupWeightDict map[string]int) (string, string) {
+	var luckyKey float64
+	var luckyGroup string
+	for group, weight := range groupWeightDict {
+		if weight > 0 {
+			key := math.Pow(rand.Float64(), 1.0/float64(weight))
+			if key >= luckyKey {
+				luckyKey = key
+				luckyGroup = group
+			}
+		}
+	}
+	luckyIP := matcher.ServerGroup[luckyGroup]
+	if len(luckyIP) == 0 {
+		return "", ""
+	}
+	return matcher.randomPickOne(luckyIP), luckyGroup
+}
+
+func (matcher *Matcher) pickNFromWeightedGroup(groupWeightDict map[string]int, n int, groupIgnore string) []string {
+	h := matcher.heapPool.Get().(*MinHeap)
+	for group, weight := range groupWeightDict {
+		if group != groupIgnore && weight > 0 {
+			key := math.Pow(rand.Float64(), 1.0/float64(weight))
+			if h.HeapLength() < n {
+				h.HeapPush(group, key)
+			} else {
+				_, top, _ := h.HeapTop()
+				if key > top {
+					h.HeapPush(group, key)
+					h.HeapPop()
+				}
+			}
+		}
+	}
+	r := make([]string, 0, n)
+	for h.HeapLength() > 0 {
+		v, _, _ := h.HeapPop()
+		member := matcher.ServerGroup[v.(string)]
+		if len(member) > 0 {
+			r = append(r, matcher.randomPickOne(member))
+		}
+	}
+	matcher.heapPool.Put(h)
+	return r
+}
+
+func (matcher *Matcher) pickFromCommonBucket(b *CommonBucket) []string {
+	r := make([]string, 0, matcher.MaxLimit)
+	masterIP, masterGroup := matcher.pickOneFromWeightedGroup(b.Master)
+	if masterIP != "" {
+		r = append(r, masterIP)
+	}
+	for _, slaveIP := range matcher.pickNFromWeightedGroup(b.Slave, matcher.MaxLimit-len(r), masterGroup) {
+		r = append(r, slaveIP)
+	}
+	return r
+}
+
+func (matcher *Matcher) pickFromVIPRuleBucket(b *VIPRuleBucket) []string {
+	var length int
+	for _, group := range b.Group {
+		length += len(matcher.ServerGroup[group])
+	}
+	length += len(b.IP)
+	candidate := make([]string, length)
+	i := 0
+	for _, group := range b.Group {
+		i += copy(candidate[i:], matcher.ServerGroup[group])
+	}
+	i += copy(candidate[i:], b.IP)
+	return matcher.randomPickN(candidate, matcher.MaxLimit)
+}
+
+func (matcher *Matcher) randomPickOne(s []string) string {
+	return s[rand.Intn(len(s))]
+}
+
+func (matcher *Matcher) randomPickN(s []string, n int) []string {
+	var r []string
+	if n > len(s) {
+		n = len(s)
+	}
+	for _, v := range rand.Perm(len(s))[0:n] {
+		r = append(r, s[v])
+	}
+	return r
+}

app/service/live/broadcast-proxy/dispatch/matcher_test.go

@@ -0,0 +1,159 @@
+package dispatch
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestMatcher(t *testing.T) {
+	config := []byte(`{
+        "ip_max_limit": 2,
+        "default_domain" : "broadcastlv.chat.bilibili.com",
+        "danmaku_common_dispatch": {
+            "china" :{
+                "china_telecom": {
+                    "master": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                },
+                "china_unicom": {
+                    "master": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                },
+                "cmcc": {
+                    "master": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                },
+                "other": {
+                    "master": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                }
+            },
+            "oversea": [
+                {
+                    "rule":"($lng >= -20) && ($lng <= 160)",
+                    "master": {
+                        "tencent_siliconvalley": 10
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                },
+                {
+                    "master": {
+                        "tencent_siliconvalley": 10
+                    },
+                    "slave": {
+                        "tencent_shanghai": 10,
+                        "tencent_guangzhou": 6,
+                        "kingsoft": 24,
+                        "aliyun": 30
+                    }
+                }
+            ],
+            "unknown" : {
+                "master": {
+                    "tencent_shanghai": 10,
+                    "tencent_guangzhou": 6,
+                    "kingsoft": 24
+                },
+                "slave": {
+                    "tencent_shanghai": 10,
+                    "tencent_guangzhou": 6,
+                    "kingsoft": 24,
+                    "aliyun": 30
+                }
+            }
+        },
+        "danmaku_vip_dispatch" : [
+                {
+                    "rule":"$uid==120497668",
+                    "ip": ["118.89.14.174"]
+                },
+                {
+                    "rule":"$uid % 10 == 1",
+                    "group": ["tencent_guangzhou"]
+                },
+                {
+                    "rule":"$uid == 221122111"
+                }
+        ],
+        "danmaku_comet_group": {
+            "tencent_shanghai": [
+                "118.89.14.174",
+                "118.89.14.115",
+                "118.89.14.103",
+                "118.89.14.206",
+                "118.89.13.229"
+            ],
+            "tencent_guangzhou": [
+                "211.159.194.41",
+                "211.159.194.115",
+                "211.159.194.105"
+            ],
+            "tencent_hongkong": [
+                "119.28.56.183"
+            ],
+            "tencent_siliconvalley": [
+                "49.51.37.200"
+            ],
+            "kingsoft": [
+                "120.92.78.57",
+                "120.92.158.137",
+                "120.92.112.150"
+            ],
+            "aliyun": [
+                "101.132.195.89",
+                "47.104.64.120",
+                "59.110.167.237",
+                "47.92.112.162",
+                "47.96.139.69",
+                "119.23.41.85"
+            ]
+        }
+    }`)
+	m, err := NewMatcher(config, nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+	fmt.Println(m)
+}

app/service/live/broadcast-proxy/dispatch/min_heap.go

@@ -0,0 +1,79 @@
+package dispatch
+
+import (
+	"container/heap"
+	"errors"
+)
+
+type HeapData []*HeapDataItem
+
+type HeapDataItem struct {
+	value  interface{}
+	weight float64
+}
+
+func (d HeapData) Len() int {
+	return len(d)
+}
+
+func (d HeapData) Swap(i, j int) {
+	d[i], d[j] = d[j], d[i]
+}
+
+func (d *HeapData) Push(x interface{}) {
+	item := x.(*HeapDataItem)
+	*d = append(*d, item)
+}
+
+func (d *HeapData) Pop() interface{} {
+	old := *d
+	n := len(old)
+	item := old[n-1]
+	*d = old[0 : n-1]
+	return item
+}
+
+type MinHeapData struct {
+	HeapData
+}
+
+func (d MinHeapData) Less(i, j int) bool {
+	return d.HeapData[i].weight < d.HeapData[j].weight
+}
+
+type MinHeap struct {
+	data MinHeapData
+}
+
+func NewMinHeap() *MinHeap {
+	h := new(MinHeap)
+	heap.Init(&h.data)
+	return h
+}
+
+func (h *MinHeap) HeapPush(value interface{}, weight float64) {
+	heap.Push(&h.data, &HeapDataItem{
+		value:  value,
+		weight: weight,
+	})
+}
+
+func (h *MinHeap) HeapPop() (interface{}, float64, error) {
+	if h.data.Len() == 0 {
+		return nil, 0, errors.New("heap is empty")
+	}
+	item := heap.Pop(&h.data).(*HeapDataItem)
+	return item.value, item.weight, nil
+}
+
+func (h *MinHeap) HeapTop() (interface{}, float64, error) {
+	if h.data.Len() == 0 {
+		return nil, 0, errors.New("heap is empty")
+	}
+	item := h.data.HeapData[0]
+	return item.value, item.weight, nil
+}
+
+func (h *MinHeap) HeapLength() int {
+	return h.data.Len()
+}

app/service/live/broadcast-proxy/dispatch/sinaip.go

@@ -0,0 +1,372 @@
+package dispatch
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"os"
+	"syscall"
+)
+
+type SinaIP struct {
+	country  map[uint16]*Country
+	province map[uint16]*Province
+	city     map[uint16]*City
+	isp      map[uint16]*ISP
+	district map[uint16]*District
+	segment  []*IPSegment
+}
+
+type IPDetail struct {
+	Country   string
+	Province  string
+	City      string
+	ISP       string
+	District  string
+	Latitude  float64
+	Longitude float64
+}
+
+type IPSegment struct {
+	start     uint32
+	end       uint32
+	country   *Country
+	province  *Province
+	city      *City
+	isp       *ISP
+	district  *District
+	latitude  float64
+	longitude float64
+}
+
+type Country struct {
+	name string
+}
+
+type Province struct {
+	name string
+}
+
+type City struct {
+	name string
+}
+
+type ISP struct {
+	name string
+}
+
+type District struct {
+	name string
+}
+
+func NewSinaIP(file string) (*SinaIP, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	data, err := syscall.Mmap(int(f.Fd()), 0, int(info.Size()), syscall.PROT_READ, syscall.MAP_PRIVATE)
+	if err != nil {
+		return nil, err
+	}
+	defer syscall.Munmap(data)
+
+	var (
+		segmentOffset  uint32
+		countryOffset  uint32
+		provinceOffset uint32
+		cityOffset     uint32
+		ispOffset      uint32
+		districtOffset uint32
+	)
+	reader := bytes.NewReader(data[0:24])
+	if err := binary.Read(reader, binary.BigEndian, &segmentOffset); err != nil {
+		return nil, err
+	}
+	if err := binary.Read(reader, binary.BigEndian, &countryOffset); err != nil {
+		return nil, err
+	}
+	if err := binary.Read(reader, binary.BigEndian, &provinceOffset); err != nil {
+		return nil, err
+	}
+	if err := binary.Read(reader, binary.BigEndian, &cityOffset); err != nil {
+		return nil, err
+	}
+	if err := binary.Read(reader, binary.BigEndian, &ispOffset); err != nil {
+		return nil, err
+	}
+	if err := binary.Read(reader, binary.BigEndian, &districtOffset); err != nil {
+		return nil, err
+	}
+
+	biliIP := new(SinaIP)
+	if err := biliIP.loadCity(data[cityOffset:]); err != nil {
+		return nil, err
+	}
+	if err := biliIP.loadCountry(data[countryOffset:]); err != nil {
+		return nil, err
+	}
+	if err := biliIP.loadProvince(data[provinceOffset:]); err != nil {
+		return nil, err
+	}
+	if err := biliIP.loadISP(data[ispOffset:]); err != nil {
+		return nil, err
+	}
+	if err := biliIP.loadDistrict(data[districtOffset:]); err != nil {
+		return nil, err
+	}
+	if err := biliIP.loadIPSegment(data[segmentOffset:]); err != nil {
+		return nil, err
+	}
+	return biliIP, nil
+}
+
+func (b *SinaIP) DoQuery(ip uint32) *IPDetail {
+	left := 0
+	right := len(b.segment) - 1
+	var r *IPDetail
+	for left <= right {
+		middle := left + (right-left)/2
+		s := b.segment[middle]
+		if ip >= s.start && ip <= s.end {
+			r = new(IPDetail)
+			if s.country != nil {
+				r.Country = s.country.name
+			}
+			if s.province != nil {
+				r.Province = s.province.name
+			}
+			if s.city != nil {
+				r.City = s.city.name
+			}
+			if s.isp != nil {
+				r.ISP = s.isp.name
+			}
+			if s.district != nil {
+				r.District = s.district.name
+			}
+			r.Latitude = s.latitude
+			r.Longitude = s.longitude
+			break
+		} else if ip < s.start {
+			right = middle - 1
+		} else if ip > s.end {
+			left = middle + 1
+		}
+	}
+	return r
+}
+
+func (b *SinaIP) loadCountry(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.country = make(map[uint16]*Country, count)
+	for i := uint16(0); i < count; i++ {
+		var code uint16
+		if err := binary.Read(reader, binary.BigEndian, &code); err != nil {
+			return err
+		}
+		var length uint8
+		if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
+			return err
+		}
+		country := make([]byte, length)
+		if _, err := io.ReadFull(reader, country); err != nil {
+			return err
+		}
+		b.country[code] = &Country{
+			name: string(country),
+		}
+	}
+	return nil
+}
+
+func (b *SinaIP) loadProvince(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.province = make(map[uint16]*Province, count)
+	for i := uint16(0); i < count; i++ {
+		var code uint16
+		if err := binary.Read(reader, binary.BigEndian, &code); err != nil {
+			return err
+		}
+		var length uint8
+		if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
+			return err
+		}
+		province := make([]byte, length)
+		if _, err := io.ReadFull(reader, province); err != nil {
+			return err
+		}
+		b.province[code] = &Province{
+			name: string(province),
+		}
+	}
+	return nil
+}
+
+func (b *SinaIP) loadCity(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.city = make(map[uint16]*City, count)
+	for i := uint16(0); i < count; i++ {
+		var code uint16
+		if err := binary.Read(reader, binary.BigEndian, &code); err != nil {
+			return err
+		}
+		var length uint8
+		if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
+			return err
+		}
+		city := make([]byte, length)
+		if _, err := io.ReadFull(reader, city); err != nil {
+			return err
+		}
+		b.city[code] = &City{
+			name: string(city),
+		}
+	}
+	return nil
+}
+
+func (b *SinaIP) loadISP(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.isp = make(map[uint16]*ISP, count)
+	for i := uint16(0); i < count; i++ {
+		var code uint16
+		if err := binary.Read(reader, binary.BigEndian, &code); err != nil {
+			return err
+		}
+		var length uint8
+		if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
+			return err
+		}
+		isp := make([]byte, length)
+		if _, err := io.ReadFull(reader, isp); err != nil {
+			return err
+		}
+		b.isp[code] = &ISP{
+			name: string(isp),
+		}
+	}
+	return nil
+}
+
+func (b *SinaIP) loadDistrict(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint16
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.district = make(map[uint16]*District, count)
+	for i := uint16(0); i < count; i++ {
+		var code uint16
+		if err := binary.Read(reader, binary.BigEndian, &code); err != nil {
+			return err
+		}
+		var length uint8
+		if err := binary.Read(reader, binary.BigEndian, &length); err != nil {
+			return err
+		}
+		district := make([]byte, length)
+		if _, err := io.ReadFull(reader, district); err != nil {
+			return err
+		}
+		b.district[code] = &District{
+			name: string(district),
+		}
+	}
+	return nil
+}
+
+func (b *SinaIP) loadIPSegment(data []byte) error {
+	reader := bytes.NewReader(data)
+
+	var count uint32
+	if err := binary.Read(reader, binary.BigEndian, &count); err != nil {
+		return err
+	}
+	b.segment = make([]*IPSegment, 0, count)
+	for i := uint32(0); i < count; i++ {
+		segment := new(IPSegment)
+		if err := binary.Read(reader, binary.BigEndian, &segment.start); err != nil {
+			return err
+		}
+		if err := binary.Read(reader, binary.BigEndian, &segment.end); err != nil {
+			return err
+		}
+		var (
+			countryCode  uint16
+			provinceCode uint16
+			cityCode     uint16
+			ispCode      uint16
+			districtCode uint16
+			latitude     int32
+			longitude    int32
+		)
+		if err := binary.Read(reader, binary.BigEndian, &countryCode); err != nil {
+			return err
+		} else {
+			segment.country = b.country[countryCode]
+		}
+
+		if err := binary.Read(reader, binary.BigEndian, &provinceCode); err != nil {
+			return err
+		} else {
+			segment.province = b.province[provinceCode]
+		}
+
+		if err := binary.Read(reader, binary.BigEndian, &cityCode); err != nil {
+			return err
+		} else {
+			segment.city = b.city[cityCode]
+		}
+
+		if err := binary.Read(reader, binary.BigEndian, &ispCode); err != nil {
+			return err
+		} else {
+			segment.isp = b.isp[ispCode]
+		}
+
+		if err := binary.Read(reader, binary.BigEndian, &districtCode); err != nil {
+			return err
+		} else {
+			segment.district = b.district[districtCode]
+		}
+
+		if err := binary.Read(reader, binary.BigEndian, &latitude); err != nil {
+			return err
+		} else {
+			segment.latitude = float64(latitude) / float64(10000)
+		}
+		if err := binary.Read(reader, binary.BigEndian, &longitude); err != nil {
+			return err
+		} else {
+			segment.longitude = float64(longitude) / float64(10000)
+		}
+		b.segment = append(b.segment, segment)
+	}
+	return nil
+}