Create & Init Project...

library/sync/errgroup/BUILD.bazel

@@ -0,0 +1,39 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["errgroup.go"],
+    importpath = "go-common/library/sync/errgroup",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "errgroup_test.go",
+        "example_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = ["@org_golang_x_net//context:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

library/sync/errgroup/CHANGELOG.md

@@ -0,0 +1,7 @@
+### errgroup
+
+#### Version 1.1.0
+> 1.支持 MaxProc 限制并发执行数
+
+#### Version 1.0.0
+> 1.提供带recover的errgroup，Wait()返回的err包含完整的堆栈信息  

library/sync/errgroup/CONTRIBUTORS.md

@@ -0,0 +1,5 @@
+# Author
+peiyifei
+ 
+# Reviewer
+haoguanwei

library/sync/errgroup/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- peiyifei
+reviewers:
+- haoguanwei
+- peiyifei

library/sync/errgroup/README.md

@@ -0,0 +1,3 @@
+# go-common/errgroup
+
+提供带recover的errgroup，err中包含详细堆栈信息  

library/sync/errgroup/errgroup.go

@@ -0,0 +1,114 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package errgroup provides synchronization, error propagation, and Context
+// cancelation for groups of goroutines working on subtasks of a common task.
+package errgroup
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"sync"
+)
+
+// A Group is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero Group is valid and does not cancel on error.
+type Group struct {
+	err     error
+	wg      sync.WaitGroup
+	errOnce sync.Once
+
+	workerOnce sync.Once
+	ch         chan func() error
+	chs        []func() error
+
+	cancel func()
+}
+
+// WithContext returns a new Group and an associated Context derived from ctx.
+//
+// The derived Context is canceled the first time a function passed to Go
+// returns a non-nil error or the first time Wait returns, whichever occurs
+// first.
+func WithContext(ctx context.Context) (*Group, context.Context) {
+	ctx, cancel := context.WithCancel(ctx)
+	return &Group{cancel: cancel}, ctx
+}
+
+func (g *Group) do(f func() error) {
+	var err error
+	defer func() {
+		if r := recover(); r != nil {
+			buf := make([]byte, 64<<10)
+			buf = buf[:runtime.Stack(buf, false)]
+			err = fmt.Errorf("errgroup: panic recovered: %s\n%s", r, buf)
+		}
+		if err != nil {
+			g.errOnce.Do(func() {
+				g.err = err
+				if g.cancel != nil {
+					g.cancel()
+				}
+			})
+		}
+		g.wg.Done()
+	}()
+	err = f()
+}
+
+// GOMAXPROCS set max goroutine to work.
+func (g *Group) GOMAXPROCS(n int) {
+	if n <= 0 {
+		panic("errgroup: GOMAXPROCS must great than 0")
+	}
+	g.workerOnce.Do(func() {
+		g.ch = make(chan func() error, n)
+		for i := 0; i < n; i++ {
+			go func() {
+				for f := range g.ch {
+					g.do(f)
+				}
+			}()
+		}
+	})
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error cancels the group; its error will be
+// returned by Wait.
+func (g *Group) Go(f func() error) {
+	g.wg.Add(1)
+	if g.ch != nil {
+		select {
+		case g.ch <- f:
+		default:
+			g.chs = append(g.chs, f)
+		}
+		return
+	}
+	go g.do(f)
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the first non-nil error (if any) from them.
+func (g *Group) Wait() error {
+	if g.ch != nil {
+		for _, f := range g.chs {
+			g.ch <- f
+		}
+	}
+
+	g.wg.Wait()
+	if g.ch != nil {
+		close(g.ch) // let all receiver exit
+	}
+	if g.cancel != nil {
+		g.cancel()
+	}
+	return g.err
+}

library/sync/errgroup/errgroup_test.go

@@ -0,0 +1,288 @@
+package errgroup
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+type ABC struct {
+	CBA int
+}
+
+func TestNormal(t *testing.T) {
+	var (
+		abcs = make(map[int]*ABC)
+		g    Group
+		err  error
+	)
+	for i := 0; i < 10; i++ {
+		abcs[i] = &ABC{CBA: i}
+	}
+	g.Go(func() (err error) {
+		abcs[1].CBA++
+		return
+	})
+	g.Go(func() (err error) {
+		abcs[2].CBA++
+		return
+	})
+	if err = g.Wait(); err != nil {
+		t.Log(err)
+	}
+	t.Log(abcs)
+}
+
+func sleep1s() error {
+	time.Sleep(time.Second)
+	return nil
+}
+
+func TestGOMAXPROCS(t *testing.T) {
+	// 没有并发数限制
+	g := Group{}
+	now := time.Now()
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Go(sleep1s)
+	g.Wait()
+	sec := math.Round(time.Since(now).Seconds())
+	if sec != 1 {
+		t.FailNow()
+	}
+	// 限制并发数
+	g2 := Group{}
+	g2.GOMAXPROCS(2)
+	now = time.Now()
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Go(sleep1s)
+	g2.Wait()
+	sec = math.Round(time.Since(now).Seconds())
+	if sec != 2 {
+		t.FailNow()
+	}
+	// context canceled
+	var canceled bool
+	g3, ctx := WithContext(context.Background())
+	g3.GOMAXPROCS(2)
+	g3.Go(func() error {
+		return fmt.Errorf("error for testing errgroup context")
+	})
+	g3.Go(func() error {
+		time.Sleep(time.Second)
+		select {
+		case <-ctx.Done():
+			canceled = true
+		default:
+		}
+		return nil
+	})
+	g3.Wait()
+	if !canceled {
+		t.FailNow()
+	}
+}
+
+func TestRecover(t *testing.T) {
+	var (
+		abcs = make(map[int]*ABC)
+		g    Group
+		err  error
+	)
+	g.Go(func() (err error) {
+		abcs[1].CBA++
+		return
+	})
+	g.Go(func() (err error) {
+		abcs[2].CBA++
+		return
+	})
+	if err = g.Wait(); err != nil {
+		t.Logf("error:%+v", err)
+		return
+	}
+	t.FailNow()
+}
+
+func TestRecover2(t *testing.T) {
+	var (
+		g   Group
+		err error
+	)
+	g.Go(func() (err error) {
+		panic("2233")
+	})
+	if err = g.Wait(); err != nil {
+		t.Logf("error:%+v", err)
+		return
+	}
+	t.FailNow()
+}
+
+var (
+	Web   = fakeSearch("web")
+	Image = fakeSearch("image")
+	Video = fakeSearch("video")
+)
+
+type Result string
+type Search func(ctx context.Context, query string) (Result, error)
+
+func fakeSearch(kind string) Search {
+	return func(_ context.Context, query string) (Result, error) {
+		return Result(fmt.Sprintf("%s result for %q", kind, query)), nil
+	}
+}
+
+// JustErrors illustrates the use of a Group in place of a sync.WaitGroup to
+// simplify goroutine counting and error handling. This example is derived from
+// the sync.WaitGroup example at https://golang.org/pkg/sync/#example_WaitGroup.
+func ExampleGroup_justErrors() {
+	var g Group
+	var urls = []string{
+		"http://www.golang.org/",
+		"http://www.google.com/",
+		"http://www.somestupidname.com/",
+	}
+	for _, url := range urls {
+		// Launch a goroutine to fetch the URL.
+		url := url // https://golang.org/doc/faq#closures_and_goroutines
+		g.Go(func() error {
+			// Fetch the URL.
+			resp, err := http.Get(url)
+			if err == nil {
+				resp.Body.Close()
+			}
+			return err
+		})
+	}
+	// Wait for all HTTP fetches to complete.
+	if err := g.Wait(); err == nil {
+		fmt.Println("Successfully fetched all URLs.")
+	}
+}
+
+// Parallel illustrates the use of a Group for synchronizing a simple parallel
+// task: the "Google Search 2.0" function from
+// https://talks.golang.org/2012/concurrency.slide#46, augmented with a Context
+// and error-handling.
+func ExampleGroup_parallel() {
+	Google := func(ctx context.Context, query string) ([]Result, error) {
+		g, ctx := WithContext(ctx)
+
+		searches := []Search{Web, Image, Video}
+		results := make([]Result, len(searches))
+		for i, search := range searches {
+			i, search := i, search // https://golang.org/doc/faq#closures_and_goroutines
+			g.Go(func() error {
+				result, err := search(ctx, query)
+				if err == nil {
+					results[i] = result
+				}
+				return err
+			})
+		}
+		if err := g.Wait(); err != nil {
+			return nil, err
+		}
+		return results, nil
+	}
+
+	results, err := Google(context.Background(), "golang")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		return
+	}
+	for _, result := range results {
+		fmt.Println(result)
+	}
+
+	// Output:
+	// web result for "golang"
+	// image result for "golang"
+	// video result for "golang"
+}
+
+func TestZeroGroup(t *testing.T) {
+	err1 := errors.New("errgroup_test: 1")
+	err2 := errors.New("errgroup_test: 2")
+
+	cases := []struct {
+		errs []error
+	}{
+		{errs: []error{}},
+		{errs: []error{nil}},
+		{errs: []error{err1}},
+		{errs: []error{err1, nil}},
+		{errs: []error{err1, nil, err2}},
+	}
+
+	for _, tc := range cases {
+		var g Group
+
+		var firstErr error
+		for i, err := range tc.errs {
+			err := err
+			g.Go(func() error { return err })
+
+			if firstErr == nil && err != nil {
+				firstErr = err
+			}
+
+			if gErr := g.Wait(); gErr != firstErr {
+				t.Errorf("after g.Go(func() error { return err }) for err in %v\n"+
+					"g.Wait() = %v; want %v", tc.errs[:i+1], err, firstErr)
+			}
+		}
+	}
+}
+
+func TestWithContext(t *testing.T) {
+	errDoom := errors.New("group_test: doomed")
+
+	cases := []struct {
+		errs []error
+		want error
+	}{
+		{want: nil},
+		{errs: []error{nil}, want: nil},
+		{errs: []error{errDoom}, want: errDoom},
+		{errs: []error{errDoom, nil}, want: errDoom},
+	}
+
+	for _, tc := range cases {
+		g, ctx := WithContext(context.Background())
+
+		for _, err := range tc.errs {
+			err := err
+			g.Go(func() error { return err })
+		}
+
+		if err := g.Wait(); err != tc.want {
+			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+				"g.Wait() = %v; want %v",
+				g, tc.errs, err, tc.want)
+		}
+
+		canceled := false
+		select {
+		case <-ctx.Done():
+			canceled = true
+		default:
+		}
+		if !canceled {
+			t.Errorf("after %T.Go(func() error { return err }) for err in %v\n"+
+				"ctx.Done() was not closed",
+				g, tc.errs)
+		}
+	}
+}

library/sync/errgroup/example_test.go

@@ -0,0 +1,65 @@
+package errgroup
+
+import (
+	"context"
+	"sync"
+)
+
+func fakeRunTask(ctx context.Context) error {
+	return nil
+}
+
+func ExampleGroup_group() {
+	g := Group{}
+	g.Go(func() error {
+		return fakeRunTask(context.Background())
+	})
+	g.Go(func() error {
+		return fakeRunTask(context.Background())
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_ctx() {
+	g, ctx := WithContext(context.Background())
+	g.Go(func() error {
+		return fakeRunTask(ctx)
+	})
+	g.Go(func() error {
+		return fakeRunTask(ctx)
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_maxproc() {
+	g := Group{}
+	// set max concurrency
+	g.GOMAXPROCS(2)
+	g.Go(func() error {
+		return fakeRunTask(context.Background())
+	})
+	g.Go(func() error {
+		return fakeRunTask(context.Background())
+	})
+	if err := g.Wait(); err != nil {
+		// handle err
+	}
+}
+
+func ExampleGroup_waitgroup() {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		// do something
+		wg.Done()
+	}()
+	go func() {
+		// do something
+		wg.Done()
+	}()
+	wg.Wait()
+}