Create & Init Project...

app/service/ops/log-agent/pipeline/dockerlogcollector/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "file.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pipeline/dockerlogcollector",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/pipeline:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/docker/docker/api/types:go_default_library",
+        "//vendor/github.com/docker/docker/client:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pipeline/dockerlogcollector/config.go

@@ -0,0 +1,39 @@
+package dockerlogcollector
+
+import (
+	"errors"
+	"time"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	ConfigEnv    string        `toml:"configEnv"`
+	ConfigSuffix string        `toml:"configSuffix"`
+	MetaPath     string        `toml:"metaPath"`
+	ScanInterval xtime.Duration `toml:"scanInterval"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of docker log collector can't be nil")
+	}
+
+	if c.ConfigEnv == "" {
+		c.ConfigEnv = "LogCollectorConf"
+	}
+
+	if c.MetaPath == "" {
+		c.MetaPath = "/data/log-agent/meta"
+	}
+
+	if c.ConfigSuffix == "" {
+		c.ConfigSuffix = ".conf"
+	}
+
+	if c.ScanInterval == 0 {
+		c.ScanInterval = xtime.Duration(time.Second * 10)
+	}
+
+	return nil
+}

app/service/ops/log-agent/pipeline/dockerlogcollector/file.go

@@ -0,0 +1,116 @@
+package dockerlogcollector
+
+import (
+	"context"
+	"time"
+	"strings"
+	"path"
+	"io/ioutil"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pipeline"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+type DockerLogCollector struct {
+	c      *Config
+	client *client.Client
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+type configItem struct {
+	configPath string
+	MergedDir  string
+}
+
+func InitDockerLogCollector(ctx context.Context, c *Config) (err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return err
+	}
+	collector := new(DockerLogCollector)
+	collector.c = c
+
+	collector.ctx, collector.cancel = context.WithCancel(ctx)
+
+	// init docker client
+	collector.client, err = client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+	go collector.scan()
+
+	return nil
+}
+
+func (collector *DockerLogCollector) getConfigs() ([]*configItem, error) {
+	var (
+		configItems = make([]*configItem, 0)
+		mergedDir   string
+		ok          bool
+	)
+	containers, err := collector.client.ContainerList(collector.ctx, types.ContainerListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, container := range containers {
+		info, err := collector.client.ContainerInspect(collector.ctx, container.ID)
+		if err != nil {
+			log.Error("failed to inspect container: %s", container.ID)
+			continue
+		}
+		// get overlay2 info
+		if info.GraphDriver.Name != "overlay2" {
+			log.Error("only overlay2 is supported")
+			continue
+		}
+
+		mergedDir, ok = info.GraphDriver.Data["MergedDir"]
+		if !ok {
+			log.Error("failed to get MergedDir of container:%s", container.ID)
+		}
+
+		for _, env := range info.Config.Env {
+			if strings.HasPrefix(env, collector.c.
+
+				ConfigEnv) {
+				for _, path := range strings.Split(strings.TrimPrefix(env, collector.c.ConfigEnv+"="), ",") {
+					configItems = append(configItems, &configItem{path, mergedDir})
+				}
+			}
+		}
+	}
+
+	return configItems, nil
+}
+
+func (collector *DockerLogCollector) scan() {
+	ticker := time.Tick(time.Duration(collector.c.ScanInterval))
+	for {
+		select {
+		case <-ticker:
+			configItems, err := collector.getConfigs()
+			if err != nil {
+				log.Error("failed to scan hostlogcollector config file list: %s", err)
+				continue
+			}
+			for _, item := range configItems {
+				configPath := path.Join(item.MergedDir, item.configPath)
+				config, err := ioutil.ReadFile(configPath)
+				if err != nil {
+					log.Error("filed to read hostlogcollector config file %s: %s", configPath, err)
+					continue
+				}
+				if !pipeline.PipelineManagement.PipelineExisted(configPath) {
+					ctx := context.WithValue(collector.ctx, "MergedDir", item.MergedDir)
+					go pipeline.PipelineManagement.StartPipeline(ctx, configPath, string(config))
+				}
+			}
+		case <-collector.ctx.Done():
+			return
+		}
+	}
+}