Create & Init Project...

app/service/main/location/dao/BUILD

@@ -0,0 +1,61 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "redis_test.go",
+        "zlimit_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/location/conf:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "redis.go",
+        "zlimit.go",
+    ],
+    importpath = "go-common/app/service/main/location/dao",
+    tags = ["automanaged"],
+    deps = [
+        "//app/service/main/location/conf:go_default_library",
+        "//app/service/main/location/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/xstr:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/location/dao/dao.go

@@ -0,0 +1,259 @@
+package dao
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"go-common/app/service/main/location/conf"
+	"go-common/app/service/main/location/model"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	httpx "go-common/library/net/http/blademaster"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_anonym  = "/app/geoip_download"
+	_checkip = "/ipip/checkipipnetversion"
+)
+
+// Dao dao.
+type Dao struct {
+	// mysql
+	c       *conf.Config
+	db      *sql.DB
+	client  *http.Client
+	client2 *httpx.Client
+	// redis
+	redis  *redis.Pool
+	expire int32
+	// host
+	anonym  string
+	checkip string
+}
+
+// New new a dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:      c,
+		db:     sql.NewMySQL(c.DB.Zlimit),
+		redis:  redis.NewPool(c.Redis.Zlimit.Config),
+		expire: int32(time.Duration(c.Redis.Zlimit.Expire) / time.Second),
+		client: &http.Client{
+			Timeout: time.Second * 30,
+		},
+		anonym:  c.Host.Maxmind + _anonym,
+		checkip: c.Host.Bvcip + _checkip,
+		client2: httpx.NewClient(c.HTTPClient),
+	}
+	return
+}
+
+// Ping ping a dao.
+func (d *Dao) Ping(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}
+
+// Close close a dao.
+func (d *Dao) Close() (err error) {
+	if d.redis != nil {
+		if err1 := d.redis.Close(); err1 != nil {
+			err = err1
+		}
+	}
+	if d.db != nil {
+		if err1 := d.db.Close(); err1 != nil {
+			err = err1
+		}
+	}
+	return
+}
+
+// DownloadAnonym download anonym file.
+func (d *Dao) DownloadAnonym() (err error) {
+	// get file
+	var (
+		req  *http.Request
+		resp *http.Response
+	)
+	params := url.Values{}
+	params.Set("edition_id", "GeoIP2-Anonymous-IP")
+	params.Set("date", "")
+	params.Set("license_key", d.c.AnonymKey)
+	params.Set("suffix", "tar.gz")
+	enc := params.Encode()
+	if req, err = http.NewRequest(http.MethodGet, d.anonym+"?"+enc, nil); err != nil {
+		log.Error("http.NewRequest(%v) error(%v)", d.anonym+"?"+enc, err)
+		return
+	}
+	if resp, err = d.client.Do(req); err != nil {
+		log.Error("d.client.Do error(%v)", d.anonym+"?"+enc, err)
+		return
+	}
+	if resp.StatusCode >= http.StatusBadRequest {
+		err = errors.New(fmt.Sprintf("incorrect http status:%d host:%s, url:%s", resp.StatusCode, d.anonym, enc))
+		log.Error("%v", err)
+		return
+	}
+	defer resp.Body.Close()
+	// get md5
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, resp.Body); err != nil {
+		log.Error("io.Copy error(%v)", err)
+		return
+	}
+	var md5Str string
+	md5Bs := md5.Sum(buf.Bytes())
+	if md5Str, err = d.downloadAnonymMd5(); err != nil {
+		log.Error("downloadAnonymMd5 error(%v)", err)
+		return
+	}
+	if md5Str != hex.EncodeToString(md5Bs[:]) {
+		err = errors.New("md5 not matched")
+		return
+	}
+	var gr *gzip.Reader
+	if gr, err = gzip.NewReader(buf); err != nil {
+		log.Error("gzip.NewReader error(%v)", err)
+		return
+	}
+	defer gr.Close()
+	tr := tar.NewReader(gr)
+	var hdr *tar.Header
+	for {
+		if hdr, err = tr.Next(); err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			} else {
+				log.Error("DownloadAnonym error(%v)", err)
+				return
+			}
+		}
+		if strings.Index(hdr.Name, d.c.AnonymFileName) > -1 {
+			var f *os.File
+			downfile := path.Join(d.c.FilePath, d.c.AnonymFileName)
+			if f, err = os.OpenFile(downfile, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666); err != nil {
+				log.Error(" os.OpenFile(%v) error(%v)", downfile, err)
+				return
+			}
+			defer f.Close()
+			if _, err = io.Copy(f, tr); err != nil {
+				log.Error("io.Copy error(%v)", err)
+				return
+			}
+			break
+		}
+	}
+	return
+}
+
+// downloadAnonymMd5 get anonym file md5.
+func (d *Dao) downloadAnonymMd5() (md5 string, err error) {
+	var (
+		req  *http.Request
+		resp *http.Response
+		body []byte
+	)
+	params := url.Values{}
+	params.Set("edition_id", "GeoIP2-Anonymous-IP")
+	params.Set("date", "")
+	params.Set("license_key", d.c.AnonymKey)
+	params.Set("suffix", "tar.gz.md5")
+	enc := params.Encode()
+	if req, err = http.NewRequest(http.MethodGet, d.anonym+"?"+enc, nil); err != nil {
+		log.Error("DownloadAnonym http.NewRequest(%v) error(%v)", d.anonym+"?"+enc, err)
+		return
+	}
+	if resp, err = d.client.Do(req); err != nil {
+		log.Error("DownloadAnonym d.client.Do error(%v)", d.anonym+"?"+enc, err)
+		return
+	}
+	if resp.StatusCode >= http.StatusBadRequest {
+		err = errors.New(fmt.Sprintf("incorrect http status:%d host:%s, url:%s", resp.StatusCode, d.anonym, enc))
+		log.Error("%v", err)
+		return
+	}
+	defer resp.Body.Close()
+	if body, err = ioutil.ReadAll(resp.Body); err != nil {
+		log.Error("DownloadAnonymMd5 ioutil.ReadAll() error(%v)", err)
+		return
+	}
+	md5 = string(body)
+	return
+}
+
+func (d *Dao) CheckVersion(c context.Context) (version *model.Version, err error) {
+	var (
+		req *http.Request
+		ip  = metadata.String(c, metadata.RemoteIP)
+	)
+	if req, err = d.client2.NewRequest("GET", d.checkip, ip, nil); err != nil {
+		log.Error("%v", err)
+		return
+	}
+	var res struct {
+		Code int            `json:"code"`
+		Msg  string         `json:"msg"`
+		Data *model.Version `json:"data"`
+	}
+	if err = d.client2.Do(c, req, &res); err != nil {
+		return
+	}
+	if res.Code != ecode.OK.Code() {
+		err = errors.New(fmt.Sprintf("checkVersion falid err_code(%v)", res.Code))
+		return
+	}
+	version = res.Data
+	return
+}
+
+func (d *Dao) DownIPLibrary(c context.Context, version, file string) (err error) {
+	var (
+		req  *http.Request
+		resp *http.Response
+	)
+	if req, err = http.NewRequest(http.MethodGet, version, nil); err != nil {
+		log.Error("http.NewRequest(%v) error(%v)", version, err)
+		return
+	}
+	if resp, err = d.client.Do(req); err != nil {
+		log.Error("d.client.Do error(%v)", version, err)
+		return
+	}
+	if resp.StatusCode >= http.StatusBadRequest {
+		err = errors.New(fmt.Sprintf("incorrect http status:%d url:%s", resp.StatusCode, version))
+		log.Error("%v", err)
+		return
+	}
+	defer resp.Body.Close()
+	var f *os.File
+	if f, err = os.OpenFile(file, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666); err != nil {
+		log.Error(" os.OpenFile(%v) error(%v)", file, err)
+		return
+	}
+	defer f.Close()
+	if _, err = io.Copy(f, resp.Body); err != nil {
+		log.Error("io.Copy error(%v)", err)
+	}
+	return
+}

app/service/main/location/dao/dao_test.go

@@ -0,0 +1,52 @@
+package dao
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/service/main/location/conf"
+	"go-common/library/cache/redis"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "main.app-svr.location-service")
+		flag.Set("conf_token", "bdc7976c3e1dbf8adeb1cdb7b1e823af")
+		flag.Set("tree_id", "3170")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/location-example.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}
+
+func CleanCache() {
+	pool := redis.NewPool(conf.Conf.Redis.Zlimit.Config)
+	pool.Get(context.TODO()).Do("FLUSHDB")
+}
+
+func WithDao(f func(d *Dao)) func() {
+	return func() {
+		convey.Reset(func() { CleanCache() })
+		f(d)
+	}
+}

app/service/main/location/dao/redis.go

@@ -0,0 +1,93 @@
+package dao
+
+import (
+	"context"
+	"strconv"
+
+	"go-common/library/cache/redis"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_prefixBlackList = "zl_"
+)
+
+func keyZlimit(aid int64) (key string) {
+	key = _prefixBlackList + strconv.FormatInt(aid, 10)
+	return
+}
+
+// ExistsAuth if existes ruls in redis.
+func (d *Dao) ExistsAuth(c context.Context, aid int64) (ok bool, err error) {
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if ok, err = redis.Bool(conn.Do("EXISTS", keyZlimit(aid))); err != nil {
+		err = errors.Wrapf(err, "EXISTS %s", keyZlimit(aid))
+	}
+	return
+}
+
+// Auth get zone rule from redis
+func (d *Dao) Auth(c context.Context, aid int64, zoneids []int64) (res []int64, err error) {
+	var playauth int64
+	key := keyZlimit(aid)
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	for _, v := range zoneids {
+		if err = conn.Send("HGET", key, v); err != nil {
+			err = errors.Wrapf(err, "HGET %s %d", key, v)
+			return
+		}
+	}
+	if err = conn.Flush(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	for range zoneids {
+		if playauth, err = redis.Int64(conn.Receive()); err != nil {
+			if err != redis.ErrNil {
+				err = errors.WithStack(err)
+				return
+			}
+			err = nil
+		}
+		res = append(res, playauth)
+	}
+	return
+}
+
+// AddAuth add zone rule from redis
+func (d *Dao) AddAuth(c context.Context, zoneids map[int64]map[int64]int64) (err error) {
+	var key string
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	count := 0
+	for aid, zids := range zoneids {
+		if key == "" {
+			key = keyZlimit(aid)
+		}
+		for zid, auth := range zids {
+			if err = conn.Send("HSET", key, zid, auth); err != nil {
+				err = errors.Wrapf(err, "HGET %s %d", key, zid)
+				return
+			}
+			count++
+		}
+	}
+	if err = conn.Send("EXPIRE", key, d.expire); err != nil {
+		err = errors.Wrapf(err, "EXPIRE %s %d", key, d.expire)
+		return
+	}
+	if err = conn.Flush(); err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	for i := 0; i <= count; i++ {
+		if _, err = conn.Receive(); err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+	}
+	return
+}

app/service/main/location/dao/redis_test.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaokeyZlimit(t *testing.T) {
+	var (
+		aid = int64(123456)
+	)
+	convey.Convey("keyZlimit", t, func(ctx convey.C) {
+		key := keyZlimit(aid)
+		ctx.Convey("key should not be equal to 123456", func(ctx convey.C) {
+			ctx.So(key, convey.ShouldEqual, "zl_123456")
+		})
+	})
+}
+
+func TestDaoExistsAuth(t *testing.T) {
+	var (
+		c       = context.TODO()
+		aid     = int64(123456)
+		zoneids = map[int64]map[int64]int64{int64(123456): {int64(234567): int64(345678)}}
+	)
+	convey.Convey("ExistsAuth", t, WithDao(func(d *Dao) {
+		d.AddAuth(c, zoneids)
+		ok, err := d.ExistsAuth(c, aid)
+		convey.Convey("Error should be nil, ok should be true", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(ok, convey.ShouldBeTrue)
+		})
+	}))
+}
+
+func TestDaoAuth(t *testing.T) {
+	var (
+		c       = context.TODO()
+		aid     = int64(0)
+		zoneid  = []int64{int64(0)}
+		zoneids = map[int64]map[int64]int64{int64(123456): {int64(234567): int64(345678)}}
+	)
+	convey.Convey("Auth", t, WithDao(func(d *Dao) {
+		d.AddAuth(c, zoneids)
+		res, err := d.Auth(c, aid, zoneid)
+		convey.Convey("Error should be nil, res should not be empty", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	}))
+}
+
+func TestDaoAddAuth(t *testing.T) {
+	var (
+		c       = context.TODO()
+		zoneids = map[int64]map[int64]int64{int64(123456): {int64(234567): int64(345678)}}
+	)
+	convey.Convey("AddAuth", t, WithDao(func(d *Dao) {
+		err := d.AddAuth(c, zoneids)
+		convey.Convey("Error should be nil", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	}))
+}

app/service/main/location/dao/zlimit.go

@@ -0,0 +1,151 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/service/main/location/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	_getPolicySQL       = "SELECT id, play_auth, down_auth, zone_id FROM policy_item WHERE zone_id <> '' AND state=1"
+	_getRelationSQL     = "SELECT policy_id FROM archive_relation WHERE aid=?"
+	_getGolbalPolicySQL = "SELECT group_id,group_concat(id) FROM policy_item WHERE zone_id <> '' AND state=1 GROUP BY group_id"
+	_getGroupZone       = "SELECT a.group_id,a.play_auth,a.zone_id FROM policy_item AS a,policy_group AS b WHERE a.zone_id <> '' AND a.group_id=b.id AND b.type=2 AND a.state=1 AND b.state=1"
+)
+
+// Policies get policy data from db
+func (d *Dao) Policies(c context.Context) (res map[int64]map[int64]int64, err error) {
+	var (
+		tmpres map[int64]int64
+		ok     bool
+	)
+	rows, err := d.db.Query(c, _getPolicySQL)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]map[int64]int64)
+	for rows.Next() {
+		var (
+			pid, playAuth, downAuth int64
+			zoneID                  string
+			zoneIDs                 []int64
+		)
+		if err = rows.Scan(&pid, &playAuth, &downAuth, &zoneID); err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+		if zoneIDs, err = xstr.SplitInts(zoneID); err != nil {
+			log.Error("xstr.SplitInts(%s) error(%v)", zoneID, err)
+			continue
+		}
+		for _, zoneid := range zoneIDs {
+			if tmpres, ok = res[pid]; !ok {
+				tmpres = make(map[int64]int64)
+				res[pid] = tmpres
+			}
+			resCode := playAuth<<8 | downAuth
+			tmpres[zoneid] = resCode
+		}
+	}
+	err = errors.WithStack(err)
+	return
+}
+
+// GroupPolicies get policy data from db group by group_id
+func (d *Dao) GroupPolicies(c context.Context) (res map[int64][]int64, err error) {
+	rows, err := d.db.Query(c, _getGolbalPolicySQL)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64][]int64)
+	for rows.Next() {
+		var (
+			groupID int64
+			pids    string
+			zoneIDs []int64
+		)
+		if err = rows.Scan(&groupID, &pids); err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+		if zoneIDs, err = xstr.SplitInts(pids); err != nil {
+			log.Error("xstr.SplitInts(%s) error(%v)", pids, err)
+			continue
+		}
+		res[groupID] = zoneIDs
+	}
+	err = errors.WithStack(err)
+	return
+}
+
+// Groupid get gid from db by aid
+func (d *Dao) Groupid(c context.Context, aid int64) (gid int64, err error) {
+	row := d.db.QueryRow(c, _getRelationSQL, aid)
+	if err = row.Scan(&gid); err != nil {
+		if err == sql.ErrNoRows {
+			gid = 0
+			err = nil
+		} else {
+			err = errors.WithStack(err)
+		}
+	}
+	return
+}
+
+// GroupAuthZone zone_id by group_id.
+func (d *Dao) GroupAuthZone(c context.Context) (res map[int64]map[int64]map[int64]int64, err error) {
+	var (
+		tmpAres map[int64]map[int64]int64
+		tmpZres map[int64]int64
+		ok      bool
+	)
+	rows, err := d.db.Query(c, _getGroupZone)
+	if err != nil {
+		err = errors.WithStack(err)
+		return
+	}
+	defer rows.Close()
+	res = make(map[int64]map[int64]map[int64]int64)
+	for rows.Next() {
+		var (
+			gid, playAuth int64
+			zoneID        string
+			zoneIDs       []int64
+		)
+		if err = rows.Scan(&gid, &playAuth, &zoneID); err != nil {
+			err = errors.WithStack(err)
+			return
+		}
+		if playAuth != model.Forbidden && playAuth != model.Allow {
+			playAuth = model.Allow
+		}
+		if zoneIDs, err = xstr.SplitInts(zoneID); err != nil {
+			log.Error("xstr.SplitInts(%s) error(%v)", zoneID, err)
+			continue
+		}
+		for _, zoneid := range zoneIDs {
+			if tmpAres, ok = res[gid]; !ok {
+				tmpAres = make(map[int64]map[int64]int64)
+				res[gid] = tmpAres
+			}
+			if tmpZres, ok = tmpAres[playAuth]; !ok {
+				tmpZres = make(map[int64]int64)
+				tmpAres[playAuth] = tmpZres
+			}
+			if _, ok = tmpZres[zoneid]; !ok {
+				tmpZres[zoneid] = zoneid
+			}
+		}
+	}
+	err = errors.WithStack(err)
+	return
+}

app/service/main/location/dao/zlimit_test.go

@@ -0,0 +1,47 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoPolicies(t *testing.T) {
+	convey.Convey("Policies", t, func(ctx convey.C) {
+		res, err := d.Policies(context.Background())
+		ctx.Convey("Error should be nil, res should not be empty", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	})
+}
+
+func TestDaoGroupPolicies(t *testing.T) {
+	convey.Convey("GroupPolicies", t, func(ctx convey.C) {
+		res, err := d.GroupPolicies(context.Background())
+		ctx.Convey("Error should be nil, res should not be empty", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	})
+}
+
+func TestDaoGroupid(t *testing.T) {
+	convey.Convey("should get group_id", t, func(ctx convey.C) {
+		_, err := d.Groupid(context.Background(), 11424224)
+		ctx.Convey("Error should be nil", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestDaoGroupAuthZone(t *testing.T) {
+	convey.Convey("GroupAuthZone", t, func(ctx convey.C) {
+		res, err := d.GroupAuthZone(context.Background())
+		ctx.Convey("Error should be nil, res should not be empty", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeEmpty)
+		})
+	})
+}