Create & Init Project...

app/interface/main/mcn/tool/worker/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pool.go"],
+    importpath = "go-common/app/interface/main/mcn/tool/worker",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/log:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pool_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)

app/interface/main/mcn/tool/worker/pool.go

@@ -0,0 +1,137 @@
+package worker
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	_ratio = float32(0.8)
+)
+
+var (
+	_default = &Conf{
+		QueueSize:     1024,
+		WorkerProcMax: 32,
+		WorkerNumber:  runtime.NumCPU() - 1,
+	}
+)
+
+// Conf .
+type Conf struct {
+	QueueSize     int
+	WorkerProcMax int
+	WorkerNumber  int
+}
+
+// Pool .
+type Pool struct {
+	c            *Conf
+	queue        chan func()
+	workerNumber int
+	close        chan struct{}
+	wg           sync.WaitGroup
+}
+
+// New .
+func New(conf *Conf) (w *Pool) {
+	if conf == nil {
+		conf = _default
+	}
+	w = &Pool{
+		c:            conf,
+		queue:        make(chan func(), conf.QueueSize),
+		workerNumber: conf.WorkerNumber,
+		close:        make(chan struct{}),
+	}
+	w.start()
+	go w.moni()
+	return
+}
+
+func (w *Pool) start() {
+	for i := 0; i < w.workerNumber; i++ {
+		w.wg.Add(1)
+		go w.workerRoutine()
+	}
+}
+
+func (w *Pool) moni() {
+	var conf = w.c
+	for {
+		time.Sleep(time.Second * 5)
+		var ratio = float32(len(w.queue)) / float32(conf.QueueSize)
+		if ratio >= _ratio {
+			if w.workerNumber >= conf.WorkerProcMax {
+				log.Warn("work thread more than max(%d)", conf.WorkerProcMax)
+				return
+			}
+			var next = minInt(w.workerNumber<<1, w.c.WorkerProcMax)
+			var diff = next - w.workerNumber
+			log.Info("current thread count=%d, queue ratio=%f, create new thread number=(%d)", w.workerNumber, ratio, diff)
+			for i := 0; i < diff; i++ {
+				w.wg.Add(1)
+				go w.workerRoutine()
+			}
+			w.workerNumber = next
+		}
+	}
+}
+
+// Close .
+func (w *Pool) Close() {
+	close(w.close)
+}
+
+// Wait .
+func (w *Pool) Wait() {
+	w.wg.Wait()
+}
+
+func (w *Pool) workerRoutine() {
+	defer func() {
+		w.wg.Done()
+		if x := recover(); x != nil {
+			const size = 64 << 10
+			buf := make([]byte, size)
+			buf = buf[:runtime.Stack(buf, false)]
+			log.Error("w.workerRoutine panic(%+v) :\n %s", x, buf)
+			w.wg.Add(1)
+			go w.workerRoutine()
+		}
+	}()
+loop:
+	for {
+		select {
+		case f := <-w.queue:
+			f()
+		case <-w.close:
+			log.Info("workerRoutine close()")
+			break loop
+		}
+	}
+	for f := range w.queue {
+		f()
+	}
+}
+
+// Add .
+func (w *Pool) Add(f func()) error {
+	select {
+	case w.queue <- f:
+	default:
+		return fmt.Errorf("task channel is full")
+	}
+	return nil
+}
+
+func minInt(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

app/interface/main/mcn/tool/worker/pool_test.go

@@ -0,0 +1,61 @@
+package worker
+
+import (
+	"testing"
+	"time"
+)
+
+func TestIncrease(t *testing.T) {
+	var (
+		conf = &Conf{
+			QueueSize:     10,
+			WorkerProcMax: 10,
+			WorkerNumber:  1,
+		}
+		workerPool = New(conf)
+	)
+
+	for i := 0; i < 10; i++ {
+		workerPool.Add(longtime)
+	}
+	time.Sleep(6 * time.Second)
+	var expect = minInt(conf.WorkerNumber<<1, conf.WorkerProcMax)
+	if workerPool.workerNumber != expect {
+		t.Logf("worker number=%d, expect=%d", workerPool.workerNumber, expect)
+		t.FailNow()
+	}
+
+	for i := 0; i < 10; i++ {
+		workerPool.Add(longtime)
+	}
+	time.Sleep(6 * time.Second)
+	expect = minInt(conf.WorkerNumber<<2, conf.WorkerProcMax)
+	if workerPool.workerNumber != expect {
+		t.Logf("worker number=%d, expect=%d", workerPool.workerNumber, expect)
+		t.FailNow()
+	}
+
+	for i := 0; i < 10; i++ {
+		workerPool.Add(longtime)
+	}
+	time.Sleep(6 * time.Second)
+	expect = minInt(conf.WorkerNumber<<3, conf.WorkerProcMax)
+	if workerPool.workerNumber != expect {
+		t.Logf("worker number=%d, expect=%d", workerPool.workerNumber, expect)
+		t.FailNow()
+	}
+
+	for i := 0; i < 10; i++ {
+		workerPool.Add(longtime)
+	}
+	time.Sleep(6 * time.Second)
+	expect = minInt(conf.WorkerNumber<<4, conf.WorkerProcMax)
+	if workerPool.workerNumber != expect {
+		t.Logf("worker number=%d, expect=%d", workerPool.workerNumber, expect)
+		t.FailNow()
+	}
+}
+
+func longtime() {
+	time.Sleep(20 * time.Second)
+}