Create & Init Project...

app/admin/main/bfs/dao/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "http.go",
+        "zk.go",
+    ],
+    importpath = "go-common/app/admin/main/bfs/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/main/bfs/conf:go_default_library",
+        "//app/admin/main/bfs/model:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//vendor/github.com/samuel/go-zookeeper/zk:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "http_test.go",
+        "zk_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/main/bfs/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/admin/main/bfs/dao/dao.go

@@ -0,0 +1,48 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/admin/main/bfs/conf"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/samuel/go-zookeeper/zk"
+)
+
+// Dao dao
+type Dao struct {
+	c       *conf.Config
+	zkcs    map[string]*zk.Conn
+	httpCli *bm.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	zkcs := make(map[string]*zk.Conn)
+	for name, zkCfg := range c.Zookeepers {
+		zkc, _, err := zk.Connect(zkCfg.Addrs, time.Duration(zkCfg.Timeout))
+		if err != nil {
+			panic(err)
+		}
+		zkcs[name] = zkc
+	}
+	dao = &Dao{
+		c:       c,
+		zkcs:    zkcs,
+		httpCli: bm.NewClient(c.HTTPClient),
+	}
+	return
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	return nil
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	for _, zkc := range d.zkcs {
+		zkc.Close()
+	}
+}

app/admin/main/bfs/dao/dao_test.go

@@ -0,0 +1,36 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/main/bfs/conf"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("conf_appid", "main.common-arch.bfs-admin")
+		flag.Set("conf_token", "117c7ca29fdb846f73d96122491bb75d")
+		flag.Set("conf_env", "uat")
+		flag.Set("tree_id", "73143")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	os.Exit(m.Run())
+}

app/admin/main/bfs/dao/http.go

@@ -0,0 +1,97 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+)
+
+const (
+	_addVolume     = "/pitchfork/group/add_volume"
+	_addFreeVolume = "/pitchfork/group/add_free_volume"
+	_compact       = "/pitchfork/group/compact"
+	_groupStatus   = "/pitchfork/group/status"
+)
+
+func (d *Dao) addVolumeURI() string {
+	return d.c.Host.Pitchfork + _addVolume
+}
+
+func (d *Dao) addFreeVolumeURI() string {
+	return d.c.Host.Pitchfork + _addFreeVolume
+}
+
+func (d *Dao) compactURI() string {
+	return d.c.Host.Pitchfork + _compact
+}
+
+func (d *Dao) groupStatusURI() string {
+	return d.c.Host.Pitchfork + _groupStatus
+}
+
+// AddVolume add volumes.
+func (d *Dao) AddVolume(c context.Context, group string, num int64) (err error) {
+	var (
+		params = url.Values{}
+		uri    = d.addVolumeURI()
+		ip     = metadata.String(c, metadata.RemoteIP)
+	)
+	params.Set("group", group)
+	params.Set("num", fmt.Sprint(num))
+	if err = d.httpCli.Post(c, uri, ip, params, nil); err != nil {
+		log.Error("add volume error(%v)", err)
+	}
+	return
+}
+
+// AddFreeVolume add free volume.
+func (d *Dao) AddFreeVolume(c context.Context, group, dir string, num int64) (err error) {
+	var (
+		params = url.Values{}
+		uri    = d.addFreeVolumeURI()
+		ip     = metadata.String(c, metadata.RemoteIP)
+	)
+	params.Set("group", group)
+	params.Set("idir", dir)
+	params.Set("bdir", dir)
+	params.Set("num", fmt.Sprint(num))
+	if err = d.httpCli.Post(c, uri, ip, params, nil); err != nil {
+		log.Error("add free volume error(%v)", err)
+	}
+	return
+}
+
+// Compact compact store disk by group.
+func (d *Dao) Compact(c context.Context, group string, vid int64) (err error) {
+	var (
+		params = url.Values{}
+		uri    = d.compactURI()
+		ip     = metadata.String(c, metadata.RemoteIP)
+	)
+	params.Set("group", group)
+	if vid > 0 {
+		params.Set("vid", fmt.Sprint(vid))
+	}
+	if err = d.httpCli.Post(c, uri, ip, params, nil); err != nil {
+		log.Error("compact group error(%v)", err)
+	}
+	return
+}
+
+// SetGroupStatus set store status by group id.
+func (d *Dao) SetGroupStatus(c context.Context, group, status string) (err error) {
+	var (
+		params = url.Values{}
+		uri    = d.groupStatusURI()
+		ip     = metadata.String(c, metadata.RemoteIP)
+	)
+	params.Set("group", group)
+	params.Set("status", status)
+	if err = d.httpCli.Post(c, uri, ip, params, nil); err != nil {
+		log.Error("set group status error(%v)", err)
+	}
+	return
+}

app/admin/main/bfs/dao/http_test.go

@@ -0,0 +1,121 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoaddVolumeURI(t *testing.T) {
+	convey.Convey("addVolumeURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := d.addVolumeURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoaddFreeVolumeURI(t *testing.T) {
+	convey.Convey("addFreeVolumeURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := d.addFreeVolumeURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaocompactURI(t *testing.T) {
+	convey.Convey("compactURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := d.compactURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaogroupStatusURI(t *testing.T) {
+	convey.Convey("groupStatusURI", t, func(ctx convey.C) {
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			p1 := d.groupStatusURI()
+			ctx.Convey("Then p1 should not be nil.", func(ctx convey.C) {
+				ctx.So(p1, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+// func TestDaoAddVolume(t *testing.T) {
+// 	convey.Convey("AddVolume", t, func(ctx convey.C) {
+// 		var (
+// 			c     = context.Background()
+// 			group = "1"
+// 			num   = int64(1)
+// 		)
+// 		ctx.Convey("add one volume to group 1", func(ctx convey.C) {
+// 			err := d.AddVolume(c, group, num)
+// 			if strings.Contains(err.Error(), "store response status code:7001") {
+// 				t.Log("store have no free volume")
+// 				err = nil // NOTE ignore error
+// 			}
+// 			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+// 				ctx.So(err, convey.ShouldBeNil)
+// 			})
+// 		})
+// 	})
+// }
+
+// func TestDaoAddFreeVolume(t *testing.T) {
+// 	convey.Convey("AddFreeVolume", t, func(ctx convey.C) {
+// 		var (
+// 			c     = context.Background()
+// 			group = "1"
+// 			dir   = "/mnt/storage00/bfsdata"
+// 			num   = int64(1)
+// 		)
+// 		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+// 			err := d.AddFreeVolume(c, group, dir, num)
+// 			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+// 				ctx.So(err, convey.ShouldBeNil)
+// 			})
+// 		})
+// 	})
+// }
+
+func TestDaoCompact(t *testing.T) {
+	convey.Convey("Compact", t, func(ctx convey.C) {
+		var (
+			c     = context.Background()
+			group = "1"
+			vid   = int64(1)
+		)
+		ctx.Convey("compact group:1 vid:1", func(ctx convey.C) {
+			err := d.Compact(c, group, vid)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoSetGroupStatus(t *testing.T) {
+	convey.Convey("SetGroupStatus", t, func(ctx convey.C) {
+		var (
+			c      = context.Background()
+			group  = "1"
+			status = "health"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			err := d.SetGroupStatus(c, group, status)
+			ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+			})
+		})
+	})
+}

app/admin/main/bfs/dao/zk.go

@@ -0,0 +1,127 @@
+package dao
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+
+	"go-common/app/admin/main/bfs/model"
+	"go-common/library/log"
+)
+
+// Racks get rack infos.
+func (d *Dao) Racks(cluster string) (racks map[string]*model.Rack, err error) {
+	zkConn, ok := d.zkcs[cluster]
+	if !ok {
+		err = fmt.Errorf("not support cluster:%s", cluster)
+		return
+	}
+	root := d.c.Zookeepers[cluster].RackRoot
+	// get all rocks
+	rackNames, _, err := zkConn.Children(root)
+	if err != nil {
+		log.Error("get from:%s children error(%v)", root, err)
+		return
+	}
+	racks = make(map[string]*model.Rack)
+	for _, rackName := range rackNames {
+		// get all the store
+		rackPath := path.Join(root, rackName)
+		storeNames, _, err := zkConn.Children(rackPath)
+		if err != nil {
+			log.Error("cannot get /rack/%s info error(%v)", rackName, err)
+			continue
+		}
+		log.Info("rack:%s got store:%v", rackName, storeNames)
+		rack := &model.Rack{Stores: make(map[string]*model.Store)}
+		racks[rackName] = rack
+		// get the volume of store
+		for _, storeName := range storeNames {
+			storePath := path.Join(rackPath, storeName)
+			storeMeta, _, err := zkConn.Get(storePath)
+			if err != nil {
+				log.Error("cannot get from:%s info error(%v)", storePath, err)
+				continue
+			}
+			var data model.Store
+			if err = json.Unmarshal(storeMeta, &data); err != nil {
+				log.Error("cannot Unmarshal %s info get from:%s error(%v)", storeMeta, storePath, err)
+				continue
+			}
+			volumes, _, err := zkConn.Children(storePath)
+			if err != nil {
+				log.Error("cannot list %s info error(%v)", storePath, err)
+				continue
+			}
+			data.Volumes = volumes
+			log.Info("rack:%s store:%s got volume:%v", rackName, storeName, volumes)
+			rack.Stores[storeName] = &data
+		}
+	}
+	return
+}
+
+// Volumes get volume infos.
+func (d *Dao) Volumes(cluster string) (volumes map[string]*model.VolumeState, err error) {
+	zkConn, ok := d.zkcs[cluster]
+	if !ok {
+		err = fmt.Errorf("not support cluster:%s", cluster)
+		return
+	}
+	root := d.c.Zookeepers[cluster].VolumeRoot
+	// get all volumes
+	volumeIDs, _, err := zkConn.Children(root)
+	if err != nil {
+		log.Error("cannot get from:%s info error(%v)", root, err)
+		return
+	}
+	log.Info("got volumes: %s", volumeIDs)
+	volumes = make(map[string]*model.VolumeState)
+	for _, id := range volumeIDs {
+		// get all the volumes
+		volumePath := path.Join(root, id)
+		volumeState, _, err := zkConn.Get(volumePath)
+		if err != nil {
+			log.Error("cannot get %s info error(%v)", volumePath, err)
+			continue
+		}
+		var data model.VolumeState
+		if err := json.Unmarshal(volumeState, &data); err != nil {
+			log.Error("cannot Unmarshal %s info get from:%s error(%v)", volumeState, volumePath, err)
+			continue
+		}
+		volumes[id] = &data
+	}
+	return
+}
+
+// Groups get group infos.
+func (d *Dao) Groups(cluster string) (groups map[string]*model.Group, err error) {
+	zkConn, ok := d.zkcs[cluster]
+	if !ok {
+		err = fmt.Errorf("not support cluster:%s", cluster)
+		return
+	}
+	root := d.c.Zookeepers[cluster].GroupRoot
+	// get all groups
+	groupNames, _, err := zkConn.Children(root)
+	if err != nil {
+		log.Error("cannot get from:%s info error(%v)", root, err)
+		return
+	}
+	log.Info("got groups: %s", groupNames)
+	groups = make(map[string]*model.Group)
+	for _, groupName := range groupNames {
+		// get all the volumes
+		groupPath := path.Join(root, groupName)
+		storeNames, _, err := zkConn.Children(groupPath)
+		if err != nil {
+			log.Error("cannot get from:%s info error(%v)", groupPath, err)
+			continue
+		}
+		groups[groupName] = &model.Group{
+			Stores: storeNames,
+		}
+	}
+	return
+}

app/admin/main/bfs/dao/zk_test.go

@@ -0,0 +1,52 @@
+package dao
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDaoRacks(t *testing.T) {
+	convey.Convey("Racks", t, func(ctx convey.C) {
+		var (
+			cluster = "uat"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			racks, err := d.Racks(cluster)
+			ctx.Convey("Then err should be nil.racks should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(racks, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoVolumes(t *testing.T) {
+	convey.Convey("Volumes", t, func(ctx convey.C) {
+		var (
+			cluster = "uat"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			volumes, err := d.Volumes(cluster)
+			ctx.Convey("Then err should be nil.volumes should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(volumes, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}
+
+func TestDaoGroups(t *testing.T) {
+	convey.Convey("Groups", t, func(ctx convey.C) {
+		var (
+			cluster = "uat"
+		)
+		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+			groups, err := d.Groups(cluster)
+			ctx.Convey("Then err should be nil.groups should not be nil.", func(ctx convey.C) {
+				ctx.So(err, convey.ShouldBeNil)
+				ctx.So(groups, convey.ShouldNotBeNil)
+			})
+		})
+	})
+}