Create & Init Project...

library/sync/pipeline/fanout/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "example_test.go",
+        "fanout_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["fanout.go"],
+    importpath = "go-common/library/sync/pipeline/fanout",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/stat/prom:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

library/sync/pipeline/fanout/CHANGELOG.md

@@ -0,0 +1,4 @@
+### pipeline/fanout
+
+#### Version 1.0.0
+> 1. library/cache包改为fanout

library/sync/pipeline/fanout/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+# Author
+wangxu01
+ 
+# Reviewer
+maojian
+zhapuyu

library/sync/pipeline/fanout/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- wangxu01
+reviewers:
+- maojian
+- wangxu01
+- zhapuyu

library/sync/pipeline/fanout/README.md

@@ -0,0 +1,19 @@
+# go-common/sync/pipeline/fanout
+
+以前的library/cache包改为pipeline/fanout
+
+增加使用范围 不止由于异步增加缓存 也可以用在其他地方
+
+功能:
+
+* 支持定义Worker 数量的goroutine，进行消费
+* 内部支持的元数据传递（library/net/metadata）
+* 后续会作废library/cache，以及统一收敛Go并行里面的扇出模型
+
+示例:
+```golang
+//名称为cache 执行线程为1 buffer长度为1024
+cache := fanout.New("cache", fanout.Worker(1), fanout.Buffer(1024))
+cache.Do(c, func(c context.Context) { SomeFunc(c, args...) })
+cache.Close()
+```

library/sync/pipeline/fanout/example_test.go

@@ -0,0 +1,22 @@
+package fanout
+
+import "context"
+
+// addCache 加缓存的例子
+func addCache(c context.Context, id, value int) {
+	// some thing...
+}
+
+func Example() {
+	// 这里只是举个例子 真正使用的时候 应该用bm/rpc 传过来的context
+	var c = context.Background()
+	// 新建一个fanout 对象 名称为cache 名称主要用来上报监控和打日志使用 最好不要重复
+	// (可选参数) worker数量为1 表示后台只有1个线程在工作
+	// (可选参数) buffer 为1024 表示缓存chan长度为1024 如果chan慢了 再调用Do方法就会报错 设定长度主要为了防止OOM
+	cache := New("cache", Worker(1), Buffer(1024))
+	// 需要异步执行的方法
+	// 这里传进来的c里面的meta信息会被复制 超时会忽略 addCache拿到的context已经没有超时信息了
+	cache.Do(c, func(c context.Context) { addCache(c, 0, 0) })
+	// 程序结束的时候关闭fanout 会等待后台线程完成后返回
+	cache.Close()
+}

library/sync/pipeline/fanout/fanout.go

@@ -0,0 +1,151 @@
+package fanout
+
+import (
+	"context"
+	"errors"
+	"runtime"
+	"sync"
+
+	"go-common/library/log"
+	"go-common/library/net/metadata"
+	"go-common/library/net/trace"
+	"go-common/library/stat/prom"
+)
+
+var (
+	// ErrFull chan full.
+	ErrFull   = errors.New("fanout: chan full")
+	stats     = prom.BusinessInfoCount
+	traceTags = []trace.Tag{
+		trace.Tag{Key: trace.TagSpanKind, Value: "background"},
+		trace.Tag{Key: trace.TagComponent, Value: "sync/pipeline/fanout"},
+	}
+)
+
+type options struct {
+	worker int
+	buffer int
+}
+
+// Option fanout option
+type Option func(*options)
+
+// Worker specifies the worker of fanout
+func Worker(n int) Option {
+	if n <= 0 {
+		panic("fanout: worker should > 0")
+	}
+	return func(o *options) {
+		o.worker = n
+	}
+}
+
+// Buffer specifies the buffer of fanout
+func Buffer(n int) Option {
+	if n <= 0 {
+		panic("fanout: buffer should > 0")
+	}
+	return func(o *options) {
+		o.buffer = n
+	}
+}
+
+type item struct {
+	f   func(c context.Context)
+	ctx context.Context
+}
+
+// Fanout async consume data from chan.
+type Fanout struct {
+	name    string
+	ch      chan item
+	options *options
+	waiter  sync.WaitGroup
+
+	ctx    context.Context
+	cancel func()
+}
+
+// New new a fanout struct.
+func New(name string, opts ...Option) *Fanout {
+	if name == "" {
+		name = "fanout"
+	}
+	o := &options{
+		worker: 1,
+		buffer: 1024,
+	}
+	for _, op := range opts {
+		op(o)
+	}
+	c := &Fanout{
+		ch:      make(chan item, o.buffer),
+		name:    name,
+		options: o,
+	}
+	c.ctx, c.cancel = context.WithCancel(context.Background())
+	c.waiter.Add(o.worker)
+	for i := 0; i < o.worker; i++ {
+		go c.proc()
+	}
+	return c
+}
+
+func (c *Fanout) proc() {
+	defer c.waiter.Done()
+	for {
+		select {
+		case t := <-c.ch:
+			wrapFunc(t.f)(t.ctx)
+			stats.State(c.name+"_channel", int64(len(c.ch)))
+		case <-c.ctx.Done():
+			return
+		}
+	}
+}
+
+func wrapFunc(f func(c context.Context)) (res func(context.Context)) {
+	res = func(ctx context.Context) {
+		defer func() {
+			if r := recover(); r != nil {
+				buf := make([]byte, 64*1024)
+				buf = buf[:runtime.Stack(buf, false)]
+				log.Error("panic in fanout proc, err: %s, stack: %s", r, buf)
+			}
+		}()
+		f(ctx)
+		if tr, ok := trace.FromContext(ctx); ok {
+			tr.Finish(nil)
+		}
+	}
+	return
+}
+
+// Do save a callback func.
+func (c *Fanout) Do(ctx context.Context, f func(ctx context.Context)) (err error) {
+	if f == nil || c.ctx.Err() != nil {
+		return c.ctx.Err()
+	}
+	nakeCtx := metadata.WithContext(ctx)
+	if tr, ok := trace.FromContext(ctx); ok {
+		tr = tr.Fork("", "Fanout:Do").SetTag(traceTags...)
+		nakeCtx = trace.NewContext(nakeCtx, tr)
+	}
+	select {
+	case c.ch <- item{f: f, ctx: nakeCtx}:
+	default:
+		err = ErrFull
+	}
+	stats.State(c.name+"_channel", int64(len(c.ch)))
+	return
+}
+
+// Close close fanout
+func (c *Fanout) Close() error {
+	if err := c.ctx.Err(); err != nil {
+		return err
+	}
+	c.cancel()
+	c.waiter.Wait()
+	return nil
+}

library/sync/pipeline/fanout/fanout_test.go

@@ -0,0 +1,30 @@
+package fanout
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestFanout_Do(t *testing.T) {
+	ca := New("cache", Worker(1), Buffer(1024))
+	var run bool
+	ca.Do(context.Background(), func(c context.Context) {
+		run = true
+		panic("error")
+	})
+	time.Sleep(time.Millisecond * 50)
+	t.Log("not panic")
+	if !run {
+		t.Fatal("expect run be true")
+	}
+}
+
+func TestFanout_Close(t *testing.T) {
+	ca := New("cache", Worker(1), Buffer(1024))
+	ca.Close()
+	err := ca.Do(context.Background(), func(c context.Context) {})
+	if err == nil {
+		t.Fatal("expect get err")
+	}
+}