Create & Init Project...

app/admin/ep/saga/service/BUILD

@@ -0,0 +1,87 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "pipeline_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+        "//vendor/github.com/xanzy/go-gitlab:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "basic.go",
+        "branch.go",
+        "collect_project.go",
+        "commit.go",
+        "config.go",
+        "data.go",
+        "job.go",
+        "member.go",
+        "mr.go",
+        "pipeline.go",
+        "project.go",
+        "runner.go",
+        "service.go",
+        "statistics.go",
+        "sync.go",
+        "task.go",
+        "user.go",
+        "wechat.go",
+    ],
+    importpath = "go-common/app/admin/ep/saga/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/dao:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//app/admin/ep/saga/service/gitlab:go_default_library",
+        "//app/admin/ep/saga/service/utils:go_default_library",
+        "//app/admin/ep/saga/service/wechat:go_default_library",
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+        "//vendor/github.com/xanzy/go-gitlab:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/admin/ep/saga/service/gitlab:all-srcs",
+        "//app/admin/ep/saga/service/mail:all-srcs",
+        "//app/admin/ep/saga/service/utils:all-srcs",
+        "//app/admin/ep/saga/service/wechat:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/ep/saga/service/basic.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+)
+
+// QueryProjectStatus ...
+func (s *Service) QueryProjectStatus(c context.Context, req *model.ProjectDataReq) (resp []string) {
+	return conf.Conf.Property.DefaultProject.Status
+}
+
+// QueryProjectTypes ...
+func (s *Service) QueryProjectTypes(c context.Context, req *model.ProjectDataReq) (resp []*model.QueryTypeItem) {
+	queryTypes := conf.Conf.Property.DefaultProject.Types
+
+	for _, queryType := range queryTypes {
+		item := &model.QueryTypeItem{}
+		switch queryType {
+		case model.LastYearPerMonth:
+			item.Name = queryType
+			item.Value = model.LastYearPerMonthNote
+		case model.LastMonthPerDay:
+			item.Name = queryType
+			item.Value = model.LastMonthPerDayNote
+		case model.LastYearPerDay:
+			item.Name = queryType
+			item.Value = model.LastYearPerDayNote
+		default:
+			log.Warn("QueryProjectCommit Type is not in range")
+			return
+		}
+		resp = append(resp, item)
+	}
+
+	return
+}

app/admin/ep/saga/service/branch.go

@@ -0,0 +1,445 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// QueryProjectBranchList query project commit info according to project id.
+func (s *Service) QueryProjectBranchList(c context.Context, req *model.ProjectDataReq) (resp []*gitlab.Branch, err error) {
+	var (
+		branch     []*gitlab.Branch
+		response   *gitlab.Response
+		branchItem []*gitlab.Branch
+	)
+
+	if branch, response, err = s.gitlab.ListProjectBranch(req.ProjectID, 1); err != nil {
+		return
+	}
+
+	page := 2
+	for page <= response.TotalPages {
+		if branchItem, _, err = s.gitlab.ListProjectBranch(req.ProjectID, page); err != nil {
+			return
+		}
+		branch = append(branch, branchItem...)
+		page++
+	}
+	resp = branch
+	return
+}
+
+// QueryBranchDiffWith ...
+func (s *Service) QueryBranchDiffWith(c context.Context, req *model.BranchDiffWithRequest) (resp []*model.BranchDiffWithResponse, err error) {
+	var (
+		branchDiff []*model.BranchDiffWithResponse
+	)
+
+	// 默认主分支为master
+	if req.Master == "" {
+		req.Master = "master"
+	}
+
+	if branchDiff, err = s.AllBranchDiffWithMaster(c, req.ProjectID, req.Master); err != nil {
+		return
+	}
+	if req.Branch != "" {
+		for _, branch := range branchDiff {
+			if branch.Branch == req.Branch {
+				resp = append(resp, branch)
+				return
+			}
+		}
+	}
+	switch req.SortBy {
+	case "update":
+		sort.Slice(branchDiff, func(i, j int) bool {
+			if branchDiff[i].LatestUpdateTime.After(*branchDiff[j].LatestUpdateTime) {
+				return false
+			}
+			if branchDiff[i].LatestUpdateTime.Before(*branchDiff[j].LatestUpdateTime) {
+				return true
+			}
+			return true
+		})
+	case "sync":
+		sort.Slice(branchDiff, func(i, j int) bool {
+			if branchDiff[i].LatestSyncTime.After(*branchDiff[j].LatestSyncTime) {
+				return false
+			}
+			if branchDiff[i].LatestSyncTime.Before(*branchDiff[j].LatestSyncTime) {
+				return true
+			}
+			return true
+		})
+	case "ahead":
+		sort.Slice(branchDiff, func(i, j int) bool {
+			if branchDiff[i].Ahead > branchDiff[j].Ahead {
+				return true
+			}
+			if branchDiff[i].Ahead < branchDiff[j].Ahead {
+				return false
+			}
+			return true
+		})
+	case "behind":
+		sort.Slice(branchDiff, func(i, j int) bool {
+			if branchDiff[i].Behind > branchDiff[j].Behind {
+				return true
+			}
+			if branchDiff[i].Behind < branchDiff[j].Behind {
+				return false
+			}
+			return true
+		})
+	}
+
+	resp = branchDiff[:100]
+
+	return
+}
+
+// AllBranchDiffWithMaster ...
+func (s *Service) AllBranchDiffWithMaster(c context.Context, project int, master string) (branchDiff []*model.BranchDiffWithResponse, err error) {
+	var (
+		branches map[string]*model.CommitTreeNode
+		tree     []*model.CommitTreeNode
+		cacheKey string
+	)
+
+	log.Info("sync Branch diff start => %s", cacheKey)
+
+	if branches, err = s.AllProjectBranchInfo(c, project); err != nil {
+		return
+	}
+	if tree, err = s.BuildCommitTree(c, project); err != nil {
+		return
+	}
+
+	for k, v := range branches {
+		var (
+			base   *gitlab.Commit
+			ahead  int
+			behind int
+		)
+		if base, _, err = s.gitlab.MergeBase(c, project, []string{k, master}); err != nil {
+			return
+		}
+
+		// 计算领先commit
+		ahead = s.ComputeCommitNum(v, base, &tree)
+
+		// 计算落后commit
+		behind = s.ComputeCommitNum(branches[master], base, &tree)
+
+		log.Info("%s: ahead: %d behind:%d\n", k, ahead, behind)
+		branchDiff = append(branchDiff, &model.BranchDiffWithResponse{Branch: k, Behind: behind, Ahead: ahead, LatestUpdateTime: v.CreatedAt, LatestSyncTime: base.CreatedAt})
+	}
+
+	log.Info("Redis: Save Branch Diff Info Successfully!")
+
+	return
+}
+
+// ComputeCommitNum ...
+func (s *Service) ComputeCommitNum(head *model.CommitTreeNode, base *gitlab.Commit, tree *[]*model.CommitTreeNode) (num int) {
+	var (
+		visitedQueue []string
+		commitTree   = *tree
+		i            int
+	)
+
+	if head.CommitID == base.ID {
+		return
+	}
+	visitedQueue = append(visitedQueue, head.CommitID)
+	for i = 0; ; i++ {
+		pointer := visitedQueue[i]
+		if pointer == base.ID {
+			break
+		}
+		for _, node := range commitTree {
+			if node.CommitID == pointer {
+				if utils.InSlice(base.ID, node.Parents) {
+					visitedQueue = append(visitedQueue, base.ID)
+					break
+				}
+				for _, p := range node.Parents {
+					if !utils.InSlice(p, visitedQueue) {
+						visitedQueue = append(visitedQueue, p)
+					}
+				}
+				break
+			}
+		}
+		if i > 999 {
+			break
+		}
+		if i == len(visitedQueue)-1 {
+			break
+		}
+	}
+
+	num = i
+
+	return
+}
+
+// AllMergeBase ...
+func (s *Service) AllMergeBase(c context.Context, project int, branches []string, master string) (mergeBase map[string]*gitlab.Commit, err error) {
+	var base *gitlab.Commit
+
+	mergeBase = make(map[string]*gitlab.Commit)
+	for _, branch := range branches {
+		if base, _, err = s.gitlab.MergeBase(c, project, []string{master, branch}); err != nil {
+			return
+		}
+		mergeBase[branch] = base
+	}
+
+	return
+}
+
+//AllProjectBranchInfo 获取所有分支信息
+func (s *Service) AllProjectBranchInfo(c context.Context, projectID int) (branches map[string]*model.CommitTreeNode, err error) {
+	var projectBranches []*model.StatisticsBranches
+
+	branches = make(map[string]*model.CommitTreeNode)
+	if projectBranches, err = s.dao.QueryProjectBranch(c, projectID); err != nil {
+		return
+	}
+
+	for _, b := range projectBranches {
+		var (
+			commitInfo    *model.StatisticsCommits
+			commitParents []string
+		)
+		if commitInfo, err = s.dao.QueryCommitByID(projectID, b.CommitID); err != nil {
+			log.Error("AllProjectBranchInfo QueryCommitByID err(%+v)", err)
+			err = nil
+			continue
+		}
+		if commitInfo.ParentIDs != "" {
+			if err = json.Unmarshal([]byte(commitInfo.ParentIDs), &commitParents); err != nil {
+				return
+			}
+		}
+		branches[b.BranchName] = &model.CommitTreeNode{CommitID: b.CommitID, Parents: commitParents, CreatedAt: commitInfo.CreatedAt, Author: commitInfo.AuthorName}
+	}
+	return
+}
+
+// BuildCommitTree 获取到所有的Commits
+func (s *Service) BuildCommitTree(c context.Context, project int) (tree []*model.CommitTreeNode, err error) {
+	var projectCommits []*model.StatisticsCommits
+	tree = []*model.CommitTreeNode{}
+	if projectCommits, err = s.dao.QueryProjectCommits(c, project); err != nil {
+		return
+	}
+
+	for _, c := range projectCommits {
+		var commitParents []string
+		commitParentsString := c.ParentIDs
+		if commitParentsString != "" {
+			if err = json.Unmarshal([]byte(commitParentsString), &commitParents); err != nil {
+				log.Error("解析commit parents报错(%+v)", err)
+			}
+		}
+		tree = append(tree, &model.CommitTreeNode{CommitID: c.CommitID, Parents: commitParents, CreatedAt: c.CreatedAt, Author: c.AuthorName})
+	}
+	return
+}
+
+/*-------------------------------------- sync branch ----------------------------------------*/
+
+// SyncProjectBranch ...
+func (s *Service) SyncProjectBranch(c context.Context, projectID int) (result *model.SyncResult, err error) {
+	var (
+		branches    []*gitlab.Branch
+		resp        *gitlab.Response
+		projectInfo *model.ProjectInfo
+	)
+	result = &model.SyncResult{}
+	if err = s.dao.DeleteProjectBranch(c, projectID); err != nil {
+		return
+	}
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+	for page := 1; ; page++ {
+		result.TotalPage++
+		if branches, resp, err = s.gitlab.ListProjectBranch(projectID, page); err != nil {
+			return
+		}
+		for _, branch := range branches {
+			branchDB := &model.StatisticsBranches{
+				ProjectID:          projectID,
+				ProjectName:        projectInfo.Name,
+				CommitID:           branch.Commit.ID,
+				BranchName:         branch.Name,
+				Protected:          branch.Protected,
+				Merged:             branch.Merged,
+				DevelopersCanPush:  branch.DevelopersCanPush,
+				DevelopersCanMerge: branch.DevelopersCanMerge,
+			}
+			if err = s.SaveDatabaseBranch(c, branchDB); err != nil {
+				log.Error("Branch 存数据库报错(%+V)", err)
+				err = nil
+
+				errData := &model.FailData{
+					ChildIDStr: branch.Name,
+				}
+				result.FailData = append(result.FailData, errData)
+				continue
+			}
+			result.TotalNum++
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// SaveDatabaseBranch ...
+func (s *Service) SaveDatabaseBranch(c context.Context, branchDB *model.StatisticsBranches) (err error) {
+	var (
+		total  int
+		branch *model.StatisticsBranches
+	)
+
+	if total, err = s.dao.HasBranch(c, branchDB.ProjectID, branchDB.BranchName); err != nil {
+		log.Error("SaveDatabaseBranch HasBranch(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if branch, err = s.dao.QueryFirstBranch(c, branchDB.ProjectID, branchDB.BranchName); err != nil {
+			log.Error("SaveDatabaseBranch QueryFirstBranch(%+v)", err)
+			return
+		}
+		branchDB.ID = branch.ID
+		if err = s.dao.UpdateBranch(c, branchDB.ProjectID, branchDB.BranchName, branchDB); err != nil {
+			log.Error("SaveDatabaseBranch UpdateBranch(%+v)", err)
+			return
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseBranch Branch has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateBranch(c, branchDB); err != nil {
+		log.Error("SaveDatabaseBranch CreateBranch(%+v)", err)
+		return
+	}
+
+	return
+}
+
+/*-------------------------------------- agg branch ----------------------------------------*/
+
+// AggregateProjectBranch ...
+func (s *Service) AggregateProjectBranch(c context.Context, projectID int, master string) (err error) {
+	var (
+		branches    map[string]*model.CommitTreeNode
+		tree        []*model.CommitTreeNode
+		projectInfo *model.ProjectInfo
+	)
+	if err = s.dao.DeleteAggregateBranch(c, projectID); err != nil {
+		return
+	}
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+	if branches, err = s.AllProjectBranchInfo(c, projectID); err != nil {
+		return
+	}
+	if tree, err = s.BuildCommitTree(c, projectID); err != nil {
+		return
+	}
+
+	for k, v := range branches {
+		var (
+			base   *gitlab.Commit
+			ahead  int
+			behind int
+		)
+		if base, _, err = s.gitlab.MergeBase(c, projectID, []string{k, master}); err != nil {
+			return
+		}
+
+		// 计算领先commit
+		ahead = s.ComputeCommitNum(v, base, &tree)
+
+		// 计算落后commit
+		behind = s.ComputeCommitNum(branches[master], base, &tree)
+		branch := &model.AggregateBranches{
+			ProjectID:        projectID,
+			ProjectName:      projectInfo.Name,
+			BranchName:       k,
+			BranchUserName:   v.Author,
+			BranchMaster:     master,
+			Behind:           behind,
+			Ahead:            ahead,
+			LatestUpdateTime: v.CreatedAt,
+			LatestSyncTime:   base.CreatedAt,
+		}
+
+		if err = s.SaveAggregateBranchDatabase(c, branch); err != nil {
+			log.Error("AggregateBranch 存数据库报错(%+V)", err)
+			continue
+		}
+	}
+	return
+}
+
+// SaveAggregateBranchDatabase ...
+func (s *Service) SaveAggregateBranchDatabase(c context.Context, branchDB *model.AggregateBranches) (err error) {
+	var (
+		total           int
+		aggregateBranch *model.AggregateBranches
+	)
+
+	if total, err = s.dao.HasAggregateBranch(c, branchDB.ProjectID, branchDB.BranchName); err != nil {
+		log.Error("SaveAggregateBranchDatabase HasAggregateBranch(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if aggregateBranch, err = s.dao.QueryFirstAggregateBranch(c, branchDB.ProjectID, branchDB.BranchName); err != nil {
+			log.Error("SaveDatabaseBranch QueryFirstAggregateBranch(%+v)", err)
+			return
+		}
+		branchDB.ID = aggregateBranch.ID
+		if err = s.dao.UpdateAggregateBranch(c, branchDB.ProjectID, branchDB.BranchName, branchDB); err != nil {
+			log.Error("SaveAggregateBranchDatabase UpdateAggregateBranch(%+v)", err)
+			return
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveAggregateBranchDatabase AggregateBranch has more rows(%d)", total)
+		return
+
+	}
+
+	// insert row now
+	if err = s.dao.CreateAggregateBranch(c, branchDB); err != nil {
+		log.Error("SaveAggregateBranchDatabase CreateAggregateBranch(%+v)", err)
+		return
+	}
+
+	return
+}

app/admin/ep/saga/service/collect_project.go

@@ -0,0 +1,198 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// collectprojectproc cron func
+func (s *Service) collectprojectproc() {
+	/*defer func() {
+		if x := recover(); x != nil {
+			log.Error("collectprojectproc panic(%v)", errors.WithStack(fmt.Errorf("%v", x)))
+			go s.collectprojectproc()
+			log.Info("collectprojectproc recover")
+		}
+	}()*/
+	var err error
+	if err = s.CollectProject(context.TODO()); err != nil {
+		log.Error("s.CollectProject err (%+v)", err)
+	}
+}
+
+// CollectProject collect project information
+func (s *Service) CollectProject(c context.Context) (err error) {
+	var (
+		projects []*gitlab.Project
+		total    = 0
+		page     = 1
+	)
+
+	log.Info("Collect Project start")
+	for page <= 1000 {
+
+		if projects, err = s.gitlab.ListProjects(page); err != nil {
+			return
+		}
+
+		num := len(projects)
+		if num <= 0 {
+			break
+		}
+		total = total + num
+
+		for _, p := range projects {
+			if err = s.insertDB(p); err != nil {
+				return
+			}
+		}
+
+		page = page + 1
+	}
+	log.Info("Collect Project end, find %d projects", total)
+
+	return
+}
+
+// insertDB
+func (s *Service) insertDB(project *gitlab.Project) (err error) {
+	var (
+		b           bool
+		parseFail   bool
+		projectInfo = &model.ProjectInfo{
+			ProjectID:     project.ID,
+			Name:          project.Name,
+			Description:   project.Description,
+			WebURL:        project.WebURL,
+			Repo:          project.SSHURLToRepo,
+			DefaultBranch: project.DefaultBranch,
+			//Owner:         project.Owner.Name,
+			SpaceName:  project.Namespace.Name,
+			SpaceKind:  project.Namespace.Kind,
+			Saga:       false,
+			Runner:     false,
+			Department: "",
+			Business:   "",
+			Language:   "",
+		}
+	)
+
+	if project.Namespace.Kind == "user" {
+		return
+	}
+
+	if b, err = s.dao.HasProjectInfo(project.ID); err != nil {
+		return
+	}
+
+	if len(project.Description) > 6 {
+		projectInfo.Department, projectInfo.Business, projectInfo.Language, parseFail = parseDes(project.Description)
+	}
+
+	if parseFail {
+		projectInfo.Department, projectInfo.Business = parseGroup(project.Namespace.Name)
+	}
+
+	if b {
+		/*if err = s.update(project.ID, projectInfo); err != nil {
+			return
+		}*/
+		if err = s.dao.UpdateProjectInfo(project.ID, projectInfo); err != nil {
+			log.Warn("UpdateProjectInfo ProjectID(%d), Description: (%s)", projectInfo.ProjectID, projectInfo.Description)
+			if strings.Contains(err.Error(), "Incorrect string value") {
+				projectInfo.Description = strconv.QuoteToASCII(projectInfo.Description)
+			}
+			if err = s.dao.UpdateProjectInfo(project.ID, projectInfo); err != nil {
+				return
+			}
+		}
+	} else {
+
+		if err = s.dao.AddProjectInfo(projectInfo); err != nil {
+			log.Warn("AddProjectInfo ProjectID(%d), Description: (%s)", projectInfo.ProjectID, projectInfo.Description)
+			if strings.Contains(err.Error(), "Incorrect string value") {
+				projectInfo.Description = strconv.QuoteToASCII(projectInfo.Description)
+			}
+			if err = s.dao.AddProjectInfo(projectInfo); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// update database
+/*func (s *Service) update(projectID int, projectSrc *model.ProjectInfo) (err error) {
+	var (
+		projectDes *model.ProjectInfo
+	)
+
+	if projectDes, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if *projectSrc == *projectDes {
+		return
+	}
+
+	s.dao.UpdateProjectInfo(projectID, projectSrc)
+
+	return
+}*/
+
+// parseDes get info from project description
+func parseDes(s string) (department, business, language string, parseFail bool) {
+	//[主站 android java]
+	ids := strings.Index(s, "[")
+	idx := strings.LastIndex(s, "]")
+	if ids == -1 || idx == -1 {
+		parseFail = true
+		return
+	}
+	str := s[ids+1 : idx]
+
+	fields := strings.Fields(str)
+	if len(fields) < 3 {
+		parseFail = true
+		return
+	}
+	department = fields[0]
+	business = fields[1]
+	language = fields[2]
+
+	for _, de := range conf.Conf.Property.DeInfo {
+		if department == de.Label {
+			department = de.Value
+		}
+	}
+	for _, bu := range conf.Conf.Property.BuInfo {
+		if business == bu.Label {
+			business = bu.Value
+		}
+	}
+
+	return
+}
+
+func parseGroup(s string) (department, business string) {
+
+	group := strings.Fields(conf.Conf.Property.Group.Name)
+	de := strings.Fields(conf.Conf.Property.Group.Department)
+	bu := strings.Fields(conf.Conf.Property.Group.Business)
+
+	for i := 0; i < len(group); i++ {
+
+		if s == group[i] {
+			department, business = de[i], bu[i]
+		}
+	}
+	return
+}

app/admin/ep/saga/service/commit.go

@@ -0,0 +1,234 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// QueryProjectCommit query project commit info according to project id.
+func (s *Service) QueryProjectCommit(c context.Context, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) {
+
+	if resp, err = s.QueryProject(c, "commit", req); err != nil {
+		return
+	}
+	return
+}
+
+// QueryTeamCommit query team commit info according to department and business
+func (s *Service) QueryTeamCommit(c context.Context, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) {
+
+	if resp, err = s.QueryTeam(c, "commit", req); err != nil {
+		return
+	}
+	return
+}
+
+// QueryCommit query commit info according to department、 business and time.
+func (s *Service) QueryCommit(c context.Context, req *model.CommitRequest) (resp *model.CommitResp, err error) {
+	var (
+		layout        = "2006-01-02"
+		projectInfo   []*model.ProjectInfo
+		reqProject    = &model.ProjectInfoRequest{}
+		ProjectCommit []*model.ProjectCommit
+		respCommit    *gitlab.Response
+		since         time.Time
+		until         time.Time
+		commitNum     int
+	)
+
+	if len(req.Department) <= 0 && len(req.Business) <= 0 {
+		log.Warn("query department and business are empty!")
+		return
+	}
+
+	reqProject.Department = req.Department
+	reqProject.Business = req.Business
+	reqProject.Username = req.Username
+	if _, projectInfo, err = s.dao.QueryProjectInfo(false, reqProject); err != nil {
+		return
+	}
+
+	if len(projectInfo) <= 0 {
+		log.Warn("Found no project!")
+		return
+	}
+
+	//since, err = time.Parse("2006-01-02 15:04:05", "2018-08-13 00:00:00")
+	if since, err = time.ParseInLocation(layout, req.Since, time.Local); err != nil {
+		return
+	}
+	if until, err = time.ParseInLocation(layout, req.Until, time.Local); err != nil {
+		return
+	}
+
+	log.Info("query commit start!")
+	for _, project := range projectInfo {
+
+		if _, respCommit, err = s.gitlab.ListProjectCommit(project.ProjectID, 1, &since, &until); err != nil {
+			return
+		}
+		//log.Info("query: %s, result: %+v", project.Name, respCommit)
+
+		CommitPer := &model.ProjectCommit{
+			ProjectID: project.ProjectID,
+			Name:      project.Name,
+			CommitNum: respCommit.TotalItems,
+		}
+		ProjectCommit = append(ProjectCommit, CommitPer)
+		commitNum = commitNum + respCommit.TotalItems
+	}
+	log.Info("query commit end!")
+
+	resp = &model.CommitResp{
+		Total:         commitNum,
+		ProjectCommit: ProjectCommit,
+	}
+	return
+}
+
+/*-------------------------------------- sync commit ----------------------------------------*/
+
+// SyncProjectCommit ...
+func (s *Service) SyncProjectCommit(projectID int) (result *model.SyncResult, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		commits     []*gitlab.Commit
+		resp        *gitlab.Response
+		since       *time.Time
+		until       *time.Time
+		projectInfo *model.ProjectInfo
+	)
+	result = &model.SyncResult{}
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+	log.Info("sync project(%d) commit time since: %v, until: %v", projectID, since, until)
+
+	for page := 1; ; page++ {
+		result.TotalPage++
+		if commits, resp, err = s.gitlab.ListProjectCommit(projectID, page, since, until); err != nil {
+			return
+		}
+
+		for _, commit := range commits {
+			var (
+				statsAdditions int
+				statsDeletions int
+				parentIDs      string
+				commitStatus   string
+			)
+
+			if commit.Stats != nil {
+				statsAdditions = commit.Stats.Additions
+				statsDeletions = commit.Stats.Deletions
+			}
+			if commit.Status != nil {
+				commitStatusByte, _ := json.Marshal(commit.Status)
+				commitStatus = string(commitStatusByte)
+			}
+			parentIDsByte, _ := json.Marshal(commit.ParentIDs)
+			parentIDs = string(parentIDsByte)
+
+			commitDB := &model.StatisticsCommits{
+				CommitID:       commit.ID,
+				ProjectID:      projectID,
+				ProjectName:    projectInfo.Name,
+				ShortID:        commit.ShortID,
+				Title:          commit.Title,
+				AuthorName:     commit.AuthorName,
+				AuthoredDate:   commit.AuthoredDate,
+				CommitterName:  commit.CommitterName,
+				CommittedDate:  commit.CommittedDate,
+				CreatedAt:      commit.CreatedAt,
+				Message:        commit.Message,
+				ParentIDs:      parentIDs,
+				StatsAdditions: statsAdditions,
+				StatsDeletions: statsDeletions,
+				Status:         commitStatus,
+			}
+			if len(commitDB.Message) > model.MessageMaxLen {
+				commitDB.Message = commitDB.Message[0 : model.MessageMaxLen-1]
+			}
+
+			if err = s.SaveDatabaseCommit(commitDB); err != nil {
+				log.Error("Commit Save Database err: projectID(%d), commitID(%s)", projectID, commit.ID)
+				err = nil
+
+				errData := &model.FailData{
+					ChildIDStr: commit.ID,
+				}
+				result.FailData = append(result.FailData, errData)
+
+				continue
+			}
+			result.TotalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// SaveDatabaseCommit ...
+func (s *Service) SaveDatabaseCommit(commitDB *model.StatisticsCommits) (err error) {
+	var total int
+
+	if total, err = s.dao.HasCommit(commitDB.ProjectID, commitDB.CommitID); err != nil {
+		log.Error("SaveDatabaseCommit HasCommit(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if err = s.dao.UpdateCommit(commitDB.ProjectID, commitDB.CommitID, commitDB); err != nil {
+			if strings.Contains(err.Error(), model.DatabaseErrorText) {
+				commitDB.Title = strconv.QuoteToASCII(commitDB.Title)
+				commitDB.Message = strconv.QuoteToASCII(commitDB.Message)
+				commitDB.Title = utils.Unicode2Chinese(commitDB.Title)
+				commitDB.Message = utils.Unicode2Chinese(commitDB.Message)
+			}
+			if err = s.dao.UpdateCommit(commitDB.ProjectID, commitDB.CommitID, commitDB); err != nil {
+				log.Error("SaveDatabaseCommit UpdateCommit err(%+v)", err)
+				return
+			}
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseCommit commit has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateCommit(commitDB); err != nil {
+		if strings.Contains(err.Error(), model.DatabaseErrorText) {
+			commitDB.Title = strconv.QuoteToASCII(commitDB.Title)
+			commitDB.Message = strconv.QuoteToASCII(commitDB.Message)
+			commitDB.Title = utils.Unicode2Chinese(commitDB.Title)
+			commitDB.Message = utils.Unicode2Chinese(commitDB.Message)
+		}
+		if err = s.dao.CreateCommit(commitDB); err != nil {
+			log.Error("SaveDatabaseCommit CreateCommit err(%+v)", err)
+			return
+		}
+	}
+
+	return
+}

app/admin/ep/saga/service/config.go

@@ -0,0 +1,518 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	_configFlag     = "\r\n"
+	_sagaConfigFlag = "[[property.repos]]"
+)
+
+const (
+	_svenConfigAppName   = "app_name"
+	_svenConfigEnv       = "env"
+	_svenConfigZone      = "zone"
+	_svenConfigTreeID    = "tree_id"
+	_svenConfigToken     = "token"
+	_svenConfigBuild     = "build"
+	_svenConfigUser      = "user"
+	_svenConfigData      = "data"
+	_svenConfigNames     = "names"
+	_svenConfigMark      = "mark"
+	_svenConfigConfigIDs = "config_ids"
+	_svenConfigForce     = "force"
+	_svenConfigIncrement = "increment"
+)
+
+const (
+	_formatStr    = `        %s=%s`
+	_formatStrQuo = `        %s="%s"`
+	_formatValue  = `        %s=%v`
+	_formatInt    = `        %s=%d`
+)
+
+const (
+	_defaultBranch      = "master"
+	_defaultLockTimeout = 600
+)
+
+const (
+	_repoURL            = "URL"
+	_repoGroup          = "Group"
+	_repoName           = "Name"
+	_repoLanguage       = "Language"
+	_repoLockTimeout    = "LockTimeout"
+	_repoAuthBranches   = "AuthBranches"
+	_repoTargetBranches = "TargetBranches"
+)
+
+var (
+	sagaConfigCnName = []string{
+		"仓库地址",
+		"仓库组名",
+		"仓库名称",
+		"仓库别名",
+		"开发语言",
+		"权限分支",
+		"目标分支",
+		"MR锁定超时时间(s)",
+		"最少review人数",
+		"是否关联pipeline",
+		"自动合并",
+		"权限限制",
+		"准入标签",
+		"超级权限用户",
+	}
+
+	sagaConfigMark = []string{
+		"仓库地址",
+		"仓库组名",
+		"仓库名称",
+		"仓库别名",
+		"仓库使用语言",
+		"saga的权限管控将以此分支配置的CONTRIBUTORS.md为准，即使CONTRIBUTORS.md在其他分支上更改了，也都会以此分支的鉴权信息为准。",
+		"配置的分支可以触发saga行为，如配置 targetBranches为master、release分支，则MR的目标分支为master或release时，都能触发saga行为。支持通配",
+		"每个仓库在每个时间点只能允许一个MR在合并。MR合并时会取得一个锁并在其合并结束后将其释放；如果获取的锁MR在lockTimeout时间内都未能结束，则会认为超时并将锁自动释放，这样其他MR才能有机会获取到独享锁及合并的机会。",
+		"最终合并前除了需要owner点赞外，还需通过权限文件配置的Reviewer中minReviewer数量的人点赞后可合并。",
+		"配置后saga将会检查pipeline执行结果，并会在最后merge前再次retry pipeline。不配置的话saga不会对pipeline执行结果进行判断。",
+		"此配置以 relatePipeline 为基础，打开后saga在MR最终合并前将不再retry pipeline，并且 +mr 时如果pipeline还在运行中，待pipeline运行通过后MR将会自动合并。",
+		"打开后owner的权限将只限定在当前目录，即如果子目录配置了owner等信息，根目录的owner等将不能再管控子目录。",
+		"如果配置了标签，saga只合入打了此label的MR。",
+		"如果有配置，原来的鉴权文件CONTRIBUTORS.md将会失效，需要合并的MR都必须通过super users的review。super users本身也拥有+mr直接合并的权利。",
+	}
+)
+
+// SagaUserList ...
+func (s *Service) SagaUserList(c context.Context) (resp []string, err error) {
+	resp = conf.Conf.Property.Sven.SagaConfigsParam.UserList
+	return
+}
+
+// QueryAllConfigFile ...
+func (s *Service) QueryAllConfigFile(c context.Context, sessionID string, isSaga bool) (resp *model.ConfigData, err error) {
+	var (
+		url          = conf.Conf.Property.Sven.Configs + "?app_name=%s&tree_id=%s&env=%s&zone=%s&build_id=%s"
+		sagaConfig   = conf.Conf.Property.Sven.SagaConfigsParam
+		runnerConfig = conf.Conf.Property.Sven.ConfigsParam
+	)
+
+	if isSaga {
+		url = fmt.Sprintf(url, sagaConfig.AppName, strconv.Itoa(sagaConfig.TreeID), sagaConfig.Env, sagaConfig.Zone, strconv.Itoa(sagaConfig.BuildId))
+	} else {
+		url = fmt.Sprintf(url, runnerConfig.AppName, strconv.Itoa(runnerConfig.TreeID), runnerConfig.Env, runnerConfig.Zone, strconv.Itoa(runnerConfig.BuildId))
+	}
+
+	return s.dao.QueryAllConfigFile(c, sessionID, url)
+}
+
+// QueryConfigFileContent ...
+func (s *Service) QueryConfigFileContent(c context.Context, sessionID string) (content string, err error) {
+	var (
+		url      = conf.Conf.Property.Sven.ConfigValue
+		fileName = conf.Conf.Property.Sven.SagaConfigsParam.FileName
+		configs  *model.ConfigData
+	)
+
+	if configs, err = s.QueryAllConfigFile(c, sessionID, true); err != nil {
+		return
+	}
+	for _, confValue := range configs.BuildFiles {
+		if confValue.Name == fileName {
+			log.Info("QueryConfigFileContent get config name: %s", fileName)
+			id := strconv.Itoa(confValue.ID)
+			url = fmt.Sprintf(url+"?config_id=%s", id)
+			if content, err = s.dao.QueryConfigFileContent(c, sessionID, url); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// QueryProjectSagaConfig ...
+func (s *Service) QueryProjectSagaConfig(c context.Context, sessionID string, projectID int) (sagaConfig *model.RepoConfig, err error) {
+	var (
+		projectInfo *model.ProjectInfo
+		content     string
+	)
+
+	if content, err = s.QueryConfigFileContent(c, sessionID); err != nil {
+		return
+	}
+
+	index := strings.Index(content, _sagaConfigFlag)
+	if index < 0 {
+		return
+	}
+
+	content = content[index:]
+	log.Info("QueryProjectSagaConfig content: %s", content)
+
+	Conf := &model.Config{}
+	if _, err = toml.Decode(content, &Conf); err != nil {
+		log.Error("QueryProjectSagaConfig Decode err(%+v)", err)
+		return
+	}
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		log.Error("QueryProjectSagaConfig ProjectInfoByID err(%+v)", err)
+		return
+	}
+	projectUrl := strings.Replace(projectInfo.Repo, "git-test", "git", 1)
+	for _, r := range Conf.Property.Repos {
+		if r.URL == projectUrl {
+			sagaConfig = r
+			return
+		}
+	}
+	return
+}
+
+// UpdateConfig ...
+func (s *Service) UpdateConfig(c context.Context, sessionID, user, configFileName, configContent, mark string, isSaga bool) (resp *model.CommonResp, err error) {
+	var (
+		reqUrl = conf.Conf.Property.Sven.ConfigUpdate
+		params = url.Values{}
+	)
+
+	if isSaga {
+		params.Set(_svenConfigAppName, conf.Conf.Property.Sven.SagaConfigsParam.AppName)
+		params.Set(_svenConfigEnv, conf.Conf.Property.Sven.SagaConfigsParam.Env)
+		params.Set(_svenConfigZone, conf.Conf.Property.Sven.SagaConfigsParam.Zone)
+		params.Set(_svenConfigTreeID, strconv.Itoa(conf.Conf.Property.Sven.SagaConfigsParam.TreeID))
+		params.Set(_svenConfigToken, conf.Conf.Property.Sven.SagaConfigsParam.Token)
+	} else {
+		params.Set(_svenConfigAppName, conf.Conf.Property.Sven.ConfigsParam.AppName)
+		params.Set(_svenConfigEnv, conf.Conf.Property.Sven.ConfigsParam.Env)
+		params.Set(_svenConfigZone, conf.Conf.Property.Sven.ConfigsParam.Zone)
+		params.Set(_svenConfigTreeID, strconv.Itoa(conf.Conf.Property.Sven.ConfigsParam.TreeID))
+		params.Set(_svenConfigToken, conf.Conf.Property.Sven.ConfigsParam.Token)
+	}
+
+	data := `[{"name":"%s","comment":"%s","mark":"%s"}]`
+	data = fmt.Sprintf(data, configFileName, configContent, mark)
+	params.Set(_svenConfigData, data)
+	params.Set(_svenConfigUser, user)
+
+	log.Info("UpdateConfig params:%v", params)
+
+	if resp, err = s.dao.RequestConfig(c, sessionID, reqUrl, params); err != nil {
+		return
+	}
+	if resp.Code == ecode.OK.Code() && resp.Message == "0" {
+		log.Info("RequestConfig success")
+		resp = nil
+	}
+	return
+}
+
+// PublicConfig ...
+func (s *Service) PublicConfig(c context.Context, sessionID, user, configFileName, mark string, isSaga bool) (resp *model.CommonResp, err error) {
+	var (
+		reqUrl = conf.Conf.Property.Sven.TagUpdate
+		params = url.Values{}
+	)
+
+	if isSaga {
+		params.Set(_svenConfigAppName, conf.Conf.Property.Sven.SagaConfigsParam.AppName)
+		params.Set(_svenConfigEnv, conf.Conf.Property.Sven.SagaConfigsParam.Env)
+		params.Set(_svenConfigZone, conf.Conf.Property.Sven.SagaConfigsParam.Zone)
+		params.Set(_svenConfigTreeID, strconv.Itoa(conf.Conf.Property.Sven.SagaConfigsParam.TreeID))
+		params.Set(_svenConfigToken, conf.Conf.Property.Sven.SagaConfigsParam.Token)
+		params.Set(_svenConfigBuild, conf.Conf.Property.Sven.SagaConfigsParam.Build)
+
+	} else {
+		params.Set(_svenConfigAppName, conf.Conf.Property.Sven.ConfigsParam.AppName)
+		params.Set(_svenConfigEnv, conf.Conf.Property.Sven.ConfigsParam.Env)
+		params.Set(_svenConfigZone, conf.Conf.Property.Sven.ConfigsParam.Zone)
+		params.Set(_svenConfigTreeID, strconv.Itoa(conf.Conf.Property.Sven.ConfigsParam.TreeID))
+		params.Set(_svenConfigToken, conf.Conf.Property.Sven.ConfigsParam.Token)
+		params.Set(_svenConfigBuild, conf.Conf.Property.Sven.ConfigsParam.Build)
+	}
+	params.Set(_svenConfigForce, strconv.Itoa(conf.Conf.Property.Sven.SagaConfigsParam.Force))
+	params.Set(_svenConfigIncrement, strconv.Itoa(conf.Conf.Property.Sven.SagaConfigsParam.Increment))
+	params.Set(_svenConfigMark, mark)
+	params.Set(_svenConfigUser, user)
+	params.Set(_svenConfigNames, configFileName)
+	params.Set(_svenConfigConfigIDs, "")
+
+	if resp, err = s.dao.RequestConfig(c, sessionID, reqUrl, params); err != nil {
+		return
+	}
+	if resp.Code == ecode.OK.Code() && resp.Message == "0" {
+		log.Info("RequestConfig success")
+		resp = nil
+	}
+	return
+}
+
+// ParseRequestConfig ...
+func (s *Service) ParseRequestConfig(projectInfo *model.ProjectInfo, configs []model.ConfigSagaItem) (requestConfig *model.RepoConfig, requestConfigStr string, err error) {
+	var (
+		configStr string
+		content   []byte
+	)
+
+	configStr = doBasicDefault(projectInfo)
+
+	for _, config := range configs {
+
+		rv := reflect.ValueOf(config.Value)
+		if !rv.IsValid() {
+			configStr, _ = doDefault(config.Name, configStr)
+			continue
+		}
+
+		if rv.Kind() == reflect.Slice {
+			log.Info("%s is slice", config.Name)
+			if rv.IsNil() {
+				configStr, _ = doDefault(config.Name, configStr)
+				continue
+			}
+			if content, err = json.Marshal(config.Value); err != nil {
+				log.Error("ParseRequestConfig err(%+v)", err)
+				return
+			}
+
+			configTr := fmt.Sprintf(_formatStr, config.Name, string(content))
+			configStr = configStr + configTr + _configFlag
+
+		} else {
+			configTr := fmt.Sprintf(_formatValue, config.Name, config.Value)
+			configStr = configStr + configTr + _configFlag
+		}
+	}
+	log.Info("ParseRequestConfig: %s", configStr)
+	requestConfigStr = configStr
+
+	requestConfig = &model.RepoConfig{}
+	if _, err = toml.Decode(configStr, &requestConfig); err != nil {
+		log.Error("ParseRequestConfig toml decode err(%+v)", err)
+		return
+	}
+	return
+}
+
+// doBasicDefault ...
+func doBasicDefault(projectInfo *model.ProjectInfo) (configStr string) {
+	var configTr string
+
+	configStr = _configFlag
+
+	configTr = fmt.Sprintf(_formatStrQuo, _repoURL, projectInfo.Repo)
+	configStr = configStr + configTr + _configFlag
+
+	configTr = fmt.Sprintf(_formatStrQuo, _repoGroup, projectInfo.SpaceName)
+	configStr = configStr + configTr + _configFlag
+
+	configTr = fmt.Sprintf(_formatStrQuo, _repoName, projectInfo.Name)
+	configStr = configStr + configTr + _configFlag
+
+	return configStr
+}
+
+// doDefault ...
+func doDefault(name, config string) (configStr string, err error) {
+	var (
+		content   []byte
+		defaultBr = []string{_defaultBranch}
+	)
+	configStr = config
+
+	if strings.ToLower(name) == strings.ToLower(_repoLockTimeout) {
+
+		configTr := fmt.Sprintf(_formatInt, name, _defaultLockTimeout)
+		configStr = configStr + configTr + _configFlag
+	}
+
+	if strings.ToLower(name) == strings.ToLower(_repoAuthBranches) || strings.ToLower(name) == strings.ToLower(_repoTargetBranches) {
+
+		if content, err = json.Marshal(defaultBr); err != nil {
+			log.Error("Marshal err(%+v)", err)
+			return
+		}
+		configTr := fmt.Sprintf(_formatStr, name, string(content))
+		configStr = configStr + configTr + _configFlag
+	}
+	return
+}
+
+// ParseSvenConfig ...
+func (s *Service) ParseSvenConfig(c context.Context, sessionID, projectUrl string) (fileContent, svenConfig string, err error) {
+	var (
+		content        string
+		projectConfigs []string
+	)
+
+	if fileContent, err = s.QueryConfigFileContent(c, sessionID); err != nil {
+		return
+	}
+	log.Info("ParseSvenConfig fileContent : %s", fileContent)
+
+	index := strings.Index(fileContent, _sagaConfigFlag)
+	if index < 0 {
+		log.Warn("ParseSvenConfig not found any config flag: %s", projectUrl)
+		return
+	}
+	content = fileContent[index:]
+
+	projectConfigs = strings.Split(content, _sagaConfigFlag)
+	for i := 0; i < len(projectConfigs); i++ {
+
+		if strings.Contains(projectConfigs[i], projectUrl) {
+			svenConfig = projectConfigs[i]
+			return
+		}
+	}
+	return
+}
+
+// ReplaceConfig ...
+func (s *Service) ReplaceConfig(c context.Context, username, sessionID string, projectInfo *model.ProjectInfo, req *model.ConfigList) (newConfig string, err error) {
+	var (
+		requestConfig    *model.RepoConfig
+		requestConfigStr string
+		fileContent      string
+		svenConfig       string
+	)
+
+	if requestConfig, requestConfigStr, err = s.ParseRequestConfig(projectInfo, req.Configs); err != nil {
+		return
+	}
+
+	if fileContent, svenConfig, err = s.ParseSvenConfig(c, sessionID, projectInfo.Repo); err != nil {
+		return
+	}
+	if len(svenConfig) <= 0 {
+		return
+	}
+
+	index := strings.Index(svenConfig, "#")
+	if index > 0 {
+		annotate := svenConfig[index:]
+		requestConfigStr = requestConfigStr + _configFlag + "    " + annotate
+	}
+
+	log.Info("ReplaceConfig requestConfig: %v", requestConfig)
+	log.Info("ReplaceConfig requestConfigStr: %s", requestConfigStr)
+	log.Info("ReplaceConfig svenConfig: %s", svenConfig)
+
+	newConfig = strings.Replace(fileContent, svenConfig, requestConfigStr, 1)
+	log.Info("ReplaceConfig newConfig: %s", newConfig)
+
+	return
+}
+
+// ReleaseSagaConfig ...
+func (s *Service) ReleaseSagaConfig(c context.Context, username, sessionID string, req *model.ConfigList) (resp *model.CommonResp, err error) {
+	var (
+		configFileName   = conf.Conf.Property.Sven.SagaConfigsParam.FileName
+		sagaConfig       *model.RepoConfig
+		projectInfo      *model.ProjectInfo
+		projectID        = req.ProjectID
+		newConfigContent string
+	)
+
+	if sagaConfig, err = s.QueryProjectSagaConfig(c, sessionID, req.ProjectID); err != nil || sagaConfig == nil {
+		log.Error("ReleaseSagaConfig  err(%+v)", err)
+		return
+	}
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		log.Error("ProjectInfoByID err(%+v)", err)
+		return
+	}
+	projectInfo.Repo = strings.Replace(projectInfo.Repo, "git-test", "git", 1)
+
+	log.Info("ReleaseSagaConfig query project: %s, sagaConfig: %v", projectInfo.Name, sagaConfig)
+
+	if sagaConfig.Name == projectInfo.Name {
+
+		if newConfigContent, err = s.ReplaceConfig(c, username, sessionID, projectInfo, req); err != nil {
+			return
+		}
+
+		year, month, day := time.Now().Date()
+		monthInt := int(month)
+		hour := time.Now().Hour()
+		updateMark := fmt.Sprintf("%s-%d-%d-%d-%d | from saga-admin", username, year, monthInt, day, hour)
+
+		newConfigContent = strconv.Quote(newConfigContent)[1 : len(strconv.Quote(newConfigContent))-1]
+		log.Info("ReleaseSagaConfig newConfig: %s", newConfigContent)
+
+		if _, err = s.UpdateConfig(c, sessionID, username, configFileName, newConfigContent, updateMark, true); err != nil {
+			log.Error("UpdateConfig err(%+v)", err)
+			return
+		}
+		if _, err = s.PublicConfig(c, sessionID, username, configFileName, updateMark, true); err != nil {
+			log.Error("PublicConfig err(%+v)", err)
+			return
+		}
+	}
+
+	return
+}
+
+// OptionSaga ...
+func (s *Service) OptionSaga(c context.Context, projectID, sessionID string) (resp []*model.OptionSagaItem, err error) {
+
+	var sagaConfig *model.RepoConfig
+
+	projectIDInt, _ := strconv.Atoi(projectID)
+	if sagaConfig, err = s.QueryProjectSagaConfig(c, sessionID, projectIDInt); err != nil || sagaConfig == nil {
+		log.Error("QueryProjectSagaConfig  err(%+v)", err)
+		return
+	}
+
+	t := reflect.TypeOf(sagaConfig)
+	if t.Kind() != reflect.Ptr {
+		log.Info("OptionSaga the object is not a Ptr, but it is : %v", t.Kind())
+		return
+	}
+
+	t = reflect.TypeOf(sagaConfig).Elem()
+	if t.Kind() != reflect.Struct {
+		log.Info("OptionSaga the object is not a struct, but it is : %v", t.Kind())
+		return
+	}
+
+	v := reflect.ValueOf(sagaConfig).Elem()
+	for i := 0; i < t.NumField(); i++ {
+
+		f := t.Field(i)
+		if f.Name == _repoURL || f.Name == _repoGroup || f.Name == _repoName || f.Name == _repoLanguage {
+			continue
+		}
+
+		val := v.Field(i).Interface()
+		log.Info("OptionSaga === %s: %v = %v", f.Name, f.Type, val)
+
+		sagaItem := &model.OptionSagaItem{}
+
+		sagaItem.Name = f.Name
+		sagaItem.Value = val
+		sagaItem.CNName = sagaConfigCnName[i]
+		sagaItem.Remark = sagaConfigMark[i]
+
+		configTr := fmt.Sprintf(`%v`, f.Type)
+		sagaItem.Type = configTr
+
+		resp = append(resp, sagaItem)
+	}
+
+	return
+}

app/admin/ep/saga/service/data.go

@@ -0,0 +1,485 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+/*-------------------------------------- sync issue ----------------------------------------*/
+
+// SyncAllIssues ...
+func (s *Service) SyncAllIssues(projectID int) (totalPage, totalNum int, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		issues      []*gitlab.Issue
+		resp        *gitlab.Response
+		since       *time.Time
+		until       *time.Time
+	)
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+
+	for page := 1; ; page++ {
+		totalPage++
+		if issues, resp, err = s.gitlab.ListProjectIssues(projectID, page, since, until); err != nil {
+			return
+		}
+		for _, issue := range issues {
+			var (
+				issueAssignees string
+				issueLabels    string
+				issueTimeStats string
+				milestoneID    int
+			)
+			if issue.Milestone != nil {
+				milestoneID = issue.Milestone.ID
+			}
+			issueAssigneesByte, _ := json.Marshal(issue.Assignees)
+			issueAssignees = string(issueAssigneesByte)
+			issueLabelsByte, _ := json.Marshal(issue.Labels)
+			issueLabels = string(issueLabelsByte)
+			issueTimeStatsByte, _ := json.Marshal(issue.TimeStats)
+			issueTimeStats = string(issueTimeStatsByte)
+			issueDB := &model.StatisticsIssues{
+				ProjectID:        projectID,
+				IssueID:          issue.ID,
+				IssueIID:         issue.IID,
+				MilestoneID:      milestoneID,
+				Description:      issue.Description,
+				State:            issue.State,
+				Assignees:        issueAssignees,
+				Upvotes:          issue.Upvotes,
+				Downvotes:        issue.Downvotes,
+				Labels:           issueLabels,
+				Title:            issue.Title,
+				UpdatedAt:        issue.UpdatedAt,
+				CreatedAt:        issue.CreatedAt,
+				ClosedAt:         issue.ClosedAt,
+				Subscribed:       issue.Subscribed,
+				UserNotesCount:   issue.UserNotesCount,
+				DueDate:          issue.DueDate,
+				WebURL:           issue.WebURL,
+				TimeStats:        issueTimeStats,
+				Confidential:     issue.Confidential,
+				Weight:           issue.Weight,
+				DiscussionLocked: issue.DiscussionLocked,
+				IssueLinkID:      issue.IssueLinkID,
+			}
+
+			if issue.Author != nil {
+				issueDB.AuthorID = issue.Author.ID
+				issueDB.AuthorName = issue.Author.Name
+			}
+			if issue.Assignee != nil {
+				issueDB.AssigneeID = issue.Assignee.ID
+				issueDB.AssigneeName = issue.Assignee.Name
+			}
+
+			if len(issueDB.Description) > model.MessageMaxLen {
+				issueDB.Description = issueDB.Description[0 : model.MessageMaxLen-1]
+			}
+
+			if err = s.SaveDatabaseIssue(issueDB); err != nil {
+				log.Error("issue Save Database err: projectID(%d), IssueID(%d)", projectID, issue.ID)
+				err = nil
+				continue
+			}
+			totalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// SaveDatabaseIssue ...
+func (s *Service) SaveDatabaseIssue(issueDB *model.StatisticsIssues) (err error) {
+	var total int
+
+	if total, err = s.dao.HasIssue(issueDB.ProjectID, issueDB.IssueID); err != nil {
+		log.Error("SaveDatabaseIssue HasIssue(%+v)", err)
+		return
+	}
+
+	// update found row
+	if total == 1 {
+		if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
+			if strings.Contains(err.Error(), model.DatabaseErrorText) {
+				issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
+				issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
+				issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
+				issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
+			}
+			if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
+				log.Error("SaveDatabaseIssue UpdateIssue(%+v)", err)
+				return
+			}
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseIssue issue has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateIssue(issueDB); err != nil {
+		if strings.Contains(err.Error(), model.DatabaseErrorText) {
+			issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
+			issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
+			issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
+			issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
+		}
+		if err = s.dao.CreateIssue(issueDB); err != nil {
+			log.Error("SaveDatabaseIssue CreateIssue(%+v)", err)
+			return
+		}
+	}
+
+	return
+}
+
+/*-------------------------------------- sync note ----------------------------------------*/
+
+// SyncProjectNotes ...
+func (s *Service) SyncProjectNotes(c context.Context, projectID int) (totalPage, totalNum int, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		resp        *gitlab.Response
+		projectInfo *model.ProjectInfo
+		mrs         []*model.StatisticsMrs
+		notes       []*gitlab.Note
+		since       *time.Time
+		until       *time.Time
+	)
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+
+	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
+		return
+	}
+
+	for _, mr := range mrs {
+		for page := 1; ; page++ {
+			totalPage++
+			if notes, resp, err = s.gitlab.ListMRNotes(c, projectID, mr.MRIID, page); err != nil {
+				return
+			}
+			for _, note := range notes {
+				var (
+					notePosition     string
+					notePositionByte []byte
+				)
+				if notePositionByte, err = json.Marshal(note.Position); err != nil {
+					return
+				}
+				notePosition = string(notePositionByte)
+				noteDB := &model.StatisticsNotes{
+					ProjectID:      projectID,
+					ProjectName:    projectInfo.Name,
+					MrIID:          mr.MRIID,
+					NoteID:         note.ID,
+					Body:           note.Body,
+					Attachment:     note.Attachment,
+					Title:          note.Title,
+					FileName:       note.FileName,
+					AuthorID:       note.Author.ID,
+					AuthorName:     note.Author.Name,
+					System:         note.System,
+					ExpiresAt:      note.ExpiresAt,
+					UpdatedAt:      note.UpdatedAt,
+					CreatedAt:      note.CreatedAt,
+					NoteableID:     note.NoteableID,
+					NoteableType:   note.NoteableType,
+					Position:       notePosition,
+					Resolvable:     note.Resolvable,
+					Resolved:       note.Resolved,
+					ResolvedByID:   note.ResolvedBy.ID,
+					ResolvedByName: note.ResolvedBy.Username,
+					NoteableIID:    note.NoteableIID,
+				}
+				if err = s.SaveDatabaseNote(c, noteDB); err != nil {
+					log.Error("note Save Database err: projectID(%d), NoteID(%d)", projectID, note.ID)
+					err = nil
+					continue
+				}
+				totalNum++
+			}
+			if resp.NextPage == 0 {
+				break
+			}
+		}
+	}
+	return
+}
+
+// HandleNoteDBError ...
+func (s *Service) HandleNoteDBError(c context.Context, noteDB *model.StatisticsNotes, errText string) (note *model.StatisticsNotes) {
+	if strings.Contains(errText, model.DatabaseErrorText) {
+		noteDB.Title = strconv.QuoteToASCII(noteDB.Title)
+		noteDB.Body = strconv.QuoteToASCII(noteDB.Body)
+		noteDB.Title = utils.Unicode2Chinese(noteDB.Title)
+		noteDB.Body = utils.Unicode2Chinese(noteDB.Body)
+		if len(noteDB.Body) > model.MessageMaxLen {
+			noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
+		}
+	} else if strings.Contains(errText, model.DatabaseMaxLenthErrorText) {
+		noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
+	}
+	return noteDB
+}
+
+// SaveDatabaseNote ...
+func (s *Service) SaveDatabaseNote(c context.Context, noteDB *model.StatisticsNotes) (err error) {
+	var total int
+
+	if total, err = s.dao.HasNote(c, noteDB.ProjectID, noteDB.NoteID); err != nil {
+		log.Error("SaveDatabaseNote HasNote(%+v)", err)
+		return
+	}
+
+	// update found row
+	if total == 1 {
+		if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
+			noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
+			if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
+				log.Error("SaveDatabaseNote UpdateNote(%+v)", err)
+				return
+			}
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseNote Note has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateNote(c, noteDB); err != nil {
+		noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
+		if err = s.dao.CreateNote(c, noteDB); err != nil {
+			log.Error("SaveDatabaseNote CreateNote(%+v)", err)
+			return
+		}
+	}
+
+	return
+}
+
+/*-------------------------------------- sync emoji ----------------------------------------*/
+
+// SyncProjectAwardEmoji ...
+func (s *Service) SyncProjectAwardEmoji(c context.Context, projectID int) (totalPage, totalNum int, err error) {
+
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		resp        *gitlab.Response
+		projectInfo *model.ProjectInfo
+		mrs         []*model.StatisticsMrs
+		awardEmojis []*gitlab.AwardEmoji
+		since       *time.Time
+		until       *time.Time
+	)
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+
+	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
+		return
+	}
+
+	for _, mr := range mrs {
+		for page := 1; ; page++ {
+			totalPage++
+			if awardEmojis, resp, err = s.gitlab.ListMRAwardEmoji(projectID, mr.MRIID, page); err != nil {
+				return
+			}
+			for _, awardEmoji := range awardEmojis {
+				awardEmojiDB := &model.StatisticsMRAwardEmojis{
+					ProjectID:     projectID,
+					ProjectName:   projectInfo.Name,
+					MrIID:         mr.MRIID,
+					AwardEmojiID:  awardEmoji.ID,
+					Name:          awardEmoji.Name,
+					UserID:        awardEmoji.User.ID,
+					UserName:      awardEmoji.User.Name,
+					CreatedAt:     awardEmoji.CreatedAt,
+					UpdatedAt:     awardEmoji.UpdatedAt,
+					AwardableID:   awardEmoji.AwardableID,
+					AwardableType: awardEmoji.AwardableType,
+				}
+				if err = s.SaveDatabaseAwardEmoji(c, awardEmojiDB); err != nil {
+					log.Error("awardEmoji Save Database err: projectID(%d), AwardEmojiID(%d)", projectID, awardEmoji.ID)
+					err = nil
+					continue
+				}
+				totalNum++
+			}
+			if resp.NextPage == 0 {
+				break
+			}
+		}
+	}
+	return
+}
+
+// SaveDatabaseAwardEmoji ...
+func (s *Service) SaveDatabaseAwardEmoji(c context.Context, awardEmojiDB *model.StatisticsMRAwardEmojis) (err error) {
+	var total int
+
+	if total, err = s.dao.HasMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID); err != nil {
+		log.Error("SaveDatabaseAwardEmoji HasAwardEmoji(%+v)", err)
+		return
+	}
+
+	// update found row
+	if total == 1 {
+		if err = s.dao.UpdateMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID, awardEmojiDB); err != nil {
+			log.Error("SaveDatabaseAwardEmoji UpdateAwardEmoji(%+v)", err)
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseAwardEmoji Note has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateMRAwardEmoji(c, awardEmojiDB); err != nil {
+		log.Error("SaveDatabaseAwardEmoji CreateAwardEmoji(%+v)", err)
+		return
+
+	}
+
+	return
+}
+
+/*-------------------------------------- sync Discussion ----------------------------------------*/
+
+// SyncProjectDiscussion ...
+func (s *Service) SyncProjectDiscussion(c context.Context, projectID int) (totalPage, totalNum int, err error) {
+
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		resp        *gitlab.Response
+		projectInfo *model.ProjectInfo
+		mrs         []*model.StatisticsMrs
+		discussions []*gitlab.Discussion
+		since       *time.Time
+		until       *time.Time
+	)
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+
+	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
+		return
+	}
+
+	for _, mr := range mrs {
+		for page := 1; ; page++ {
+			totalPage++
+			if discussions, resp, err = s.gitlab.ListMRDiscussions(projectID, mr.MRIID, page); err != nil {
+				return
+			}
+			for _, discussion := range discussions {
+				var (
+					notesArray []int
+					notesByte  []byte
+					notes      string
+				)
+				if discussion.Notes != nil {
+					for _, note := range discussion.Notes {
+						notesArray = append(notesArray, note.ID)
+					}
+				}
+				if notesByte, err = json.Marshal(notesArray); err != nil {
+					notes = model.JsonMarshalErrorText
+				} else {
+					notes = string(notesByte)
+				}
+				discussionDB := &model.StatisticsDiscussions{
+					ProjectID:      projectID,
+					ProjectName:    projectInfo.Name,
+					MrIID:          mr.MRIID,
+					DiscussionID:   discussion.ID,
+					IndividualNote: discussion.IndividualNote,
+					Notes:          notes,
+				}
+				if err = s.SaveDatabaseDiscussion(c, discussionDB); err != nil {
+					log.Error("discussion Save Database err: projectID(%d), DiscussionID(%s)", projectID, discussion.ID)
+					err = nil
+					continue
+				}
+				totalNum++
+			}
+			if resp.NextPage == 0 {
+				break
+			}
+		}
+	}
+	return
+}
+
+// SaveDatabaseDiscussion ...
+func (s *Service) SaveDatabaseDiscussion(c context.Context, discussionDB *model.StatisticsDiscussions) (err error) {
+	var total int
+
+	if total, err = s.dao.HasDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID); err != nil {
+		log.Error("SaveDatabaseDiscussion HasDiscussion(%+v)", err)
+		return
+	}
+
+	// update found row
+	if total == 1 {
+		if err = s.dao.UpdateDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID, discussionDB); err != nil {
+			log.Error("SaveDatabaseDiscussion UpdateDiscussion(%+v)", err)
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseDiscussion Note has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateDiscussion(c, discussionDB); err != nil {
+		log.Error("SaveDatabaseDiscussion CreateDiscussion(%+v)", err)
+		return
+
+	}
+
+	return
+}

app/admin/ep/saga/service/gitlab/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["gitlab.go"],
+    importpath = "go-common/app/admin/ep/saga/service/gitlab",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/xanzy/go-gitlab:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["gitlab_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/admin/ep/saga/service/gitlab/gitlab.go

@@ -0,0 +1,239 @@
+package gitlab
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/xanzy/go-gitlab"
+)
+
+const (
+	_defaultPerPage = 20
+	_maxPerPage     = 100
+)
+
+// Gitlab def
+type Gitlab struct {
+	url    string
+	token  string
+	client *gitlab.Client
+}
+
+// New new gitlab structure
+func New(url string, token string) (g *Gitlab) {
+	g = &Gitlab{
+		url:    url,
+		token:  token,
+		client: gitlab.NewClient(nil, token),
+	}
+	g.client.SetBaseURL(url)
+	return
+}
+
+// ListProjects list all project
+func (g *Gitlab) ListProjects(page int) (projects []*gitlab.Project, err error) {
+
+	opt := &gitlab.ListProjectsOptions{}
+	opt.ListOptions.Page = page
+	opt.ListOptions.PerPage = _defaultPerPage
+
+	if projects, _, err = g.client.Projects.ListProjects(opt); err != nil {
+		err = errors.Wrapf(err, "ListProjects err(%+v)", err)
+		return
+	}
+	return
+}
+
+// ListProjectCommit ...
+func (g *Gitlab) ListProjectCommit(projectID, page int, since, until *time.Time) (commits []*gitlab.Commit, resp *gitlab.Response, err error) {
+	var (
+		opt = &gitlab.ListCommitsOptions{}
+		all = true
+	)
+	opt.Page = page
+	opt.PerPage = _maxPerPage
+	opt.All = &all
+
+	if since != nil {
+		opt.Since = since
+	}
+	if until != nil {
+		opt.Until = until
+	}
+
+	if commits, resp, err = g.client.Commits.ListCommits(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "ListCommits projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListProjectMergeRequests list all MR
+func (g *Gitlab) ListProjectMergeRequests(projectID int, since, until *time.Time, page int) (mrs []*gitlab.MergeRequest, resp *gitlab.Response, err error) {
+
+	opt := &gitlab.ListProjectMergeRequestsOptions{}
+	opt.UpdatedAfter = since
+	opt.UpdatedBefore = until
+	if page != -1 {
+		opt.PerPage = _defaultPerPage
+		opt.Page = page
+	}
+	if mrs, resp, err = g.client.MergeRequests.ListProjectMergeRequests(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectMergeRequests projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListProjectPipelines list all pipelines
+func (g *Gitlab) ListProjectPipelines(page, projectID int, status gitlab.BuildStateValue) (pipelineList gitlab.PipelineList, resp *gitlab.Response, err error) {
+
+	opt := &gitlab.ListProjectPipelinesOptions{}
+	opt.ListOptions.Page = page
+	if status != "" {
+		opt.Status = gitlab.BuildState(status)
+	}
+	if pipelineList, resp, err = g.client.Pipelines.ListProjectPipelines(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectPipelines projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListPipelineJobs list all pipeline jobs
+func (g *Gitlab) ListPipelineJobs(opt *gitlab.ListJobsOptions, projectID, pipelineID int) (jobList []*gitlab.Job, resp *gitlab.Response, err error) {
+	if jobList, resp, err = g.client.Jobs.ListPipelineJobs(projectID, pipelineID, opt); err != nil {
+		err = errors.Wrapf(err, "ListPipelineJobs projectId(%d), pipelineId(%d), err(%+v)", projectID, pipelineID, err)
+		return
+	}
+	return
+}
+
+// GetPipeline get a pipeline info
+func (g *Gitlab) GetPipeline(projectID, pipelineID int) (pipeline *gitlab.Pipeline, resp *gitlab.Response, err error) {
+
+	if pipeline, resp, err = g.client.Pipelines.GetPipeline(projectID, pipelineID); err != nil {
+		err = errors.Wrapf(err, "GetPipeline projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListProjectBranch ...
+func (g *Gitlab) ListProjectBranch(projectID, page int) (branches []*gitlab.Branch, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListBranchesOptions{Page: page, PerPage: 20}
+	if branches, resp, err = g.client.Branches.ListBranches(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "ListBranches projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListMRNotes get notes of the MR
+func (g *Gitlab) ListMRNotes(c context.Context, projectID, mrID, page int) (notes []*gitlab.Note, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListMergeRequestNotesOptions{Page: page, PerPage: 20}
+
+	if notes, resp, err = g.client.Notes.ListMergeRequestNotes(projectID, mrID, opt); err != nil {
+		err = errors.Wrapf(err, "ListMergeRequestNotes projectId(%d), mrId(%d), err(%+v)", projectID, mrID, err)
+		return
+	}
+	return
+}
+
+// MergeBase 获取两个分支最近的合并commit
+func (g *Gitlab) MergeBase(c context.Context, projectID int, refs []string) (commit *gitlab.Commit, resp *gitlab.Response, err error) {
+	var opt = &gitlab.MergeBaseOptions{Ref: refs}
+	if commit, resp, err = g.client.Repositories.MergeBase(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "MergeBase projectId(%d), refs(%v), err(%+v)", projectID, refs, err)
+		return
+	}
+	return
+}
+
+// ListMRAwardEmoji ...
+func (g *Gitlab) ListMRAwardEmoji(projectID, mrIID, page int) (emojis []*gitlab.AwardEmoji, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListAwardEmojiOptions{Page: page, PerPage: _defaultPerPage}
+	if emojis, resp, err = g.client.AwardEmoji.ListMergeRequestAwardEmoji(projectID, mrIID, opt); err != nil {
+		err = errors.Wrapf(err, "ListMergeRequestAwardEmoji projectId(%d), mrIID(%d), err(%+v)", projectID, mrIID, err)
+		return
+	}
+	return
+}
+
+//ListMRDiscussions ...
+func (g *Gitlab) ListMRDiscussions(projectID, mrIID, page int) (result []*gitlab.Discussion, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListMergeRequestDiscussionsOptions{Page: page, PerPage: _defaultPerPage}
+	if result, resp, err = g.client.Discussions.ListMergeRequestDiscussions(projectID, mrIID, opt); err != nil {
+		err = errors.Wrapf(err, "ListMergeRequestDiscussions projectId(%d), mrIID(%d), err(%+v)", projectID, mrIID, err)
+		return
+	}
+	return
+}
+
+// GetMergeRequestDiff ...
+func (g *Gitlab) GetMergeRequestDiff(projectID, mrID int) (mr *gitlab.MergeRequest, resp *gitlab.Response, err error) {
+	if mr, resp, err = g.client.MergeRequests.GetMergeRequestChanges(projectID, mrID); err != nil {
+		err = errors.Wrapf(err, "GetMergeRequestChanges projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListProjectJobs get jobs of the project
+func (g *Gitlab) ListProjectJobs(projID, page int) (jobs []gitlab.Job, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListJobsOptions{}
+	opt.Page = page
+	// perPage max is 100
+	opt.PerPage = _maxPerPage
+
+	if jobs, resp, err = g.client.Jobs.ListProjectJobs(projID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectJobs projectId(%d), err(%+v)", projID, err)
+		return
+	}
+	return
+}
+
+// ListProjectMembers ...
+func (g *Gitlab) ListProjectMembers(projectID, page int) (members []*gitlab.ProjectMember, resp *gitlab.Response, err error) {
+	opt := &gitlab.ListProjectMembersOptions{}
+	opt.Page = page
+	opt.PerPage = _defaultPerPage
+	if members, resp, err = g.client.ProjectMembers.ListProjectMembers(projectID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectMembers projectId(%d), err(%+v)", projectID, err)
+		return
+	}
+	return
+}
+
+// ListProjectIssues ...
+func (g *Gitlab) ListProjectIssues(projID, page int, since, until *time.Time) (issues []*gitlab.Issue, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListProjectIssuesOptions{}
+	opt.Page = page
+	// perPage max is 100
+	opt.PerPage = _maxPerPage
+	if since != nil {
+		opt.CreatedBefore = until
+	}
+	if until != nil {
+		opt.CreatedAfter = since
+	}
+
+	if issues, resp, err = g.client.Issues.ListProjectIssues(projID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectIssues projectId(%d), err(%+v)", projID, err)
+		return
+	}
+	return
+}
+
+// ListProjectRunners ...
+func (g *Gitlab) ListProjectRunners(projID, page int) (result []*gitlab.Runner, resp *gitlab.Response, err error) {
+	var opt = &gitlab.ListProjectRunnersOptions{}
+	opt.ListOptions.Page = page
+	opt.PerPage = _defaultPerPage
+	if result, resp, err = g.client.Runners.ListProjectRunners(projID, opt); err != nil {
+		err = errors.Wrapf(err, "ListProjectRunners projectId(%d), err(%+v)", projID, err)
+		return
+	}
+	return
+}

app/admin/ep/saga/service/gitlab/gitlab_test.go

@@ -0,0 +1,113 @@
+package gitlab
+
+import (
+	"flag"
+	"os"
+	"testing"
+	"time"
+
+	"go-common/app/admin/ep/saga/conf"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	g *Gitlab
+)
+
+// TestMain ...
+func TestMain(m *testing.M) {
+	flag.Set("conf", "../../cmd/saga-admin-test.toml")
+	var err error
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+	g = New(conf.Conf.Property.Gitlab.API, conf.Conf.Property.Gitlab.Token)
+	os.Exit(m.Run())
+}
+
+// TestListProjects ...
+func TestListProjects(t *testing.T) {
+	Convey("ListProjects", t, func() {
+
+		projects, err := g.ListProjects(1)
+		So(err, ShouldBeNil)
+		So(len(projects), ShouldBeGreaterThan, 1)
+	})
+}
+
+func TestListProjectPipelines(t *testing.T) {
+	Convey("listProjectPipelines", t, func() {
+		_, _, err := g.ListProjectPipelines(1, 682, "")
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestGetPipeline(t *testing.T) {
+	Convey("GetPipeline", t, func() {
+		_, _, err := g.GetPipeline(682, 166011)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestListProjectJobs(t *testing.T) {
+	Convey("ListJobs", t, func() {
+		jobs, resp, err := g.ListProjectJobs(5822, 1)
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(len(jobs), ShouldBeGreaterThan, 1)
+	})
+}
+
+func TestGitlab_ListProjectMergeRequests(t *testing.T) {
+	Convey("ListProjectMergeRequest", t, func() {
+		var (
+			project = 682
+			until   = time.Now()
+			since   = until.AddDate(0, -1, 0)
+		)
+		mrs, resp, err := g.ListProjectMergeRequests(project, &since, &until, -1)
+		So(len(mrs), ShouldBeGreaterThan, 1)
+		So(resp, ShouldNotBeNil)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestGitlab_ListProjectBranch(t *testing.T) {
+	Convey("ListProjectBranch", t, func() {
+		var (
+			project = 682
+			page    = 1
+		)
+		branches, resp, err := g.ListProjectBranch(project, page)
+		So(err, ShouldBeNil)
+		So(len(branches), ShouldBeGreaterThan, 0)
+		So(resp, ShouldNotBeNil)
+	})
+}
+
+func TestGitlab_ListProjectCommit(t *testing.T) {
+	Convey("List Project branch commit", t, func() {
+		var (
+			project = 682
+			page    = 1
+		)
+		commits, resp, err := g.ListProjectCommit(project, page, nil, nil)
+		So(err, ShouldBeNil)
+		So(commits, ShouldNotBeNil)
+		So(resp.StatusCode, ShouldEqual, 200)
+	})
+}
+
+func TestGitlab_ListProjectRunners(t *testing.T) {
+	Convey("test list project runners", t, func() {
+		var (
+			project = 4928
+			page    = 1
+		)
+		runners, resp, err := g.ListProjectRunners(project, page)
+		So(err, ShouldBeNil)
+		So(resp.StatusCode, ShouldEqual, 200)
+		So(runners, ShouldNotBeNil)
+	})
+}

app/admin/ep/saga/service/job.go

@@ -0,0 +1,485 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// QueryProjectJob ...
+func (s *Service) QueryProjectJob(c context.Context, req *model.ProjectJobRequest) (resp *model.ProjectJobResp, err error) {
+	var (
+		layout        = "2006-01-02"
+		queryCacheKey string
+		jobs          []*model.ProjectJob
+		since         time.Time
+		util          time.Time
+	)
+
+	resp = &model.ProjectJobResp{ProjectID: req.ProjectID, QueryDescription: "最近一月的Jobs日常", State: req.Scope, DataInfo: []*model.DateJobInfo{}}
+
+	year, month, day := time.Now().Date()
+	util = time.Date(year, month, day-1, 0, 0, 0, 0, time.Local)
+	since = util.AddDate(0, -1, 0)
+
+	//query from redis first
+	queryCacheKey = fmt.Sprintf("saga_admin_job_%d_%s_%s_%s_%d", req.ProjectID, req.Branch, req.Scope, req.Machine, req.StatisticsType)
+	if err = s.dao.ItemRedis(c, queryCacheKey, &resp); err != redis.ErrNil {
+		return
+	}
+
+	if resp.TotalItem, jobs, err = s.queryProjectJobByTime(c, req.ProjectID, since, util); err != nil {
+		return
+	}
+
+	//init map key
+	pendingTime := make(map[string][]float64)
+	runningTime := make(map[string][]float64)
+	for i := 1; ; i++ {
+		day := since.AddDate(0, 0, i)
+		if day.After(util) {
+			break
+		}
+		dayStr := day.Format(layout)
+		pendingTime[dayStr] = []float64{}
+		runningTime[dayStr] = []float64{}
+
+		resp.DataInfo = append(resp.DataInfo, &model.DateJobInfo{Date: dayStr, SlowestPendingJob: []*model.ProjectJob{}})
+	}
+
+	//根据查询条件进行过滤
+	newJobs := jobs[:0]
+	if req.Branch != "" {
+		for _, job := range jobs {
+			if job.Branch == req.Branch {
+				newJobs = append(newJobs, job)
+			}
+		}
+		jobs = newJobs
+		newJobs = jobs[:0]
+	}
+	if req.User != "" {
+		for _, job := range jobs {
+			if job.User == req.User {
+				newJobs = append(newJobs, job)
+			}
+		}
+		jobs = newJobs
+		newJobs = jobs[:0]
+	}
+	if req.Machine != "" {
+		for _, job := range jobs {
+			if job.Machine == req.Machine {
+				newJobs = append(newJobs, job)
+			}
+		}
+		jobs = newJobs
+	}
+
+	//统计pending running 时间
+	for _, job := range jobs {
+		var (
+			jobInfo *model.DateJobInfo
+		)
+		jobDate := job.CreatedAt.Format(layout)
+		for _, j := range resp.DataInfo {
+			if j.Date == jobDate {
+				jobInfo = j
+				break
+			}
+		}
+		jobInfo.JobTotal++
+		if job.Status == req.Scope {
+			jobInfo.StatusNum++
+			if job.StartedAt != nil && job.CreatedAt != nil {
+				pending := job.StartedAt.Sub(*job.CreatedAt).Seconds()
+				pendingTime[jobDate] = append(pendingTime[jobDate], pending)
+				if pending >= 300 {
+					jobInfo.SlowestPendingJob = append(jobInfo.SlowestPendingJob, job)
+				}
+			}
+			if job.Status == "success" {
+				running := job.FinishedAt.Sub(*job.StartedAt).Seconds()
+				runningTime[jobDate] = append(runningTime[jobDate], running)
+			}
+		}
+	}
+
+	for k, v := range runningTime {
+		var (
+			jobInfo *model.DateJobInfo
+		)
+		for _, j := range resp.DataInfo {
+			if j.Date == k {
+				jobInfo = j
+				break
+			}
+		}
+
+		jobInfo.PendingTime = utils.CalAverageTime(req.StatisticsType, v)
+		jobInfo.RunningTime = utils.CalAverageTime(req.StatisticsType, pendingTime[k])
+	}
+
+	// set data to redis
+	err = s.dao.SetItemRedis(c, queryCacheKey, resp, model.ExpiredOneDay)
+
+	return
+}
+
+// queryProjectJobByTime ...
+func (s *Service) queryProjectJobByTime(c context.Context, projectID int, since, util time.Time) (count int, result []*model.ProjectJob, err error) {
+	var (
+		resp     *gitlab.Response
+		jobs     []gitlab.Job
+		overTime bool
+	)
+
+	if _, resp, err = s.gitlab.ListProjectJobs(projectID, 1); err != nil {
+		return
+	}
+	if resp.TotalItems <= 0 {
+		return
+	}
+
+	for page := 1; ; page++ {
+		if jobs, resp, err = s.gitlab.ListProjectJobs(projectID, page); err != nil {
+			return
+		}
+		for _, job := range jobs {
+			if job.CreatedAt == nil {
+				continue
+			}
+			if job.CreatedAt.After(since) && job.CreatedAt.Before(util) {
+				count++
+
+				jobInfo := &model.ProjectJob{
+					Status:     job.Status,
+					Branch:     job.Ref,
+					Machine:    job.Runner.Description,
+					User:       job.User.Name,
+					CreatedAt:  job.CreatedAt,
+					StartedAt:  job.StartedAt,
+					FinishedAt: job.FinishedAt}
+
+				result = append(result, jobInfo)
+			}
+			if job.CreatedAt.Before(since) {
+				overTime = true
+			}
+		}
+		if overTime {
+			break
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+
+	return
+}
+
+// QueryProjectJobNew ...
+func (s *Service) QueryProjectJobNew(c context.Context, req *model.ProjectJobRequest) (resp *model.ProjectJobResp, err error) {
+	var (
+		layout    = "2006-01-02"
+		fmtLayout = `%d-%d-%d 00:00:00`
+		jobs      []*model.StatisticsJobs
+	)
+
+	resp = &model.ProjectJobResp{ProjectID: req.ProjectID, QueryDescription: "最近一月的Jobs日常", State: req.Scope, DataInfo: []*model.DateJobInfo{}}
+
+	year, month, day := time.Now().Date()
+	until := time.Date(year, month, day-1, 0, 0, 0, 0, time.Local)
+	since := until.AddDate(0, -1, 0)
+
+	sinceStr := fmt.Sprintf(fmtLayout, since.Year(), since.Month(), since.Day())
+	untilStr := fmt.Sprintf(fmtLayout, until.Year(), until.Month(), until.Day())
+	if resp.TotalItem, jobs, err = s.dao.QueryJobsByTime(req.ProjectID, req, sinceStr, untilStr); err != nil {
+		return
+	}
+
+	//init map key
+	pendingTime := make(map[string][]float64)
+	runningTime := make(map[string][]float64)
+	for i := 1; ; i++ {
+		day := since.AddDate(0, 0, i)
+		if day.After(until) {
+			break
+		}
+		dayStr := day.Format(layout)
+		pendingTime[dayStr] = []float64{}
+		runningTime[dayStr] = []float64{}
+
+		resp.DataInfo = append(resp.DataInfo, &model.DateJobInfo{Date: dayStr, SlowestPendingJob: []*model.ProjectJob{}})
+	}
+
+	//统计pending running 时间
+	for _, job := range jobs {
+		var (
+			jobInfo *model.DateJobInfo
+		)
+		jobDate := job.CreatedAt.Format(layout)
+		for _, j := range resp.DataInfo {
+			if j.Date == jobDate {
+				jobInfo = j
+				break
+			}
+		}
+		if jobInfo == nil {
+			continue
+		}
+		jobInfo.JobTotal++
+		if job.Status == req.Scope {
+			jobInfo.StatusNum++
+			if job.StartedAt != nil && job.CreatedAt != nil {
+				pending := job.StartedAt.Sub(*job.CreatedAt).Seconds()
+				pendingTime[jobDate] = append(pendingTime[jobDate], pending)
+				if pending >= 300 {
+					jo := &model.ProjectJob{
+						Status:     job.Status,
+						User:       job.UserName,
+						Branch:     job.Ref,
+						Machine:    job.RunnerDescription,
+						CreatedAt:  job.CreatedAt,
+						StartedAt:  job.StartedAt,
+						FinishedAt: job.FinishedAt,
+					}
+					jobInfo.SlowestPendingJob = append(jobInfo.SlowestPendingJob, jo)
+				}
+			}
+			if job.Status == "success" {
+				running := job.FinishedAt.Sub(*job.StartedAt).Seconds()
+				runningTime[jobDate] = append(runningTime[jobDate], running)
+			}
+		}
+	}
+
+	for k, v := range runningTime {
+		var (
+			jobInfo *model.DateJobInfo
+		)
+		for _, j := range resp.DataInfo {
+			if j.Date == k {
+				jobInfo = j
+				break
+			}
+		}
+
+		jobInfo.PendingTime = utils.CalAverageTime(req.StatisticsType, v)
+		jobInfo.RunningTime = utils.CalAverageTime(req.StatisticsType, pendingTime[k])
+	}
+
+	return
+}
+
+/*-------------------------------------- sync job ----------------------------------------*/
+
+// SyncProjectJobs ...
+func (s *Service) SyncProjectJobs(projectID int) (result *model.SyncResult, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		since       *time.Time
+		until       *time.Time
+		projectInfo *model.ProjectInfo
+	)
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+
+		since, until = utils.CalSyncTime()
+		log.Info("sync project(%d) job time since: %v, until: %v", projectID, since, until)
+		if result, err = s.SyncProjectJobsByTime(projectID, projectInfo.Name, *since, *until); err != nil {
+			return
+		}
+	} else {
+		if result, err = s.SyncProjectJobsNormal(projectID, projectInfo.Name); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// SyncProjectJobsNormal ...
+func (s *Service) SyncProjectJobsNormal(projectID int, projectName string) (result *model.SyncResult, err error) {
+	var (
+		jobs []gitlab.Job
+		resp *gitlab.Response
+	)
+	result = &model.SyncResult{}
+
+	for page := 1; ; page++ {
+		result.TotalPage++
+		if jobs, resp, err = s.gitlab.ListProjectJobs(projectID, page); err != nil {
+			return
+		}
+
+		for _, job := range jobs {
+			if err = s.structureDatabasejob(projectID, projectName, job); err != nil {
+				log.Error("job Save Database err: projectID(%d), JobID(%d)", projectID, job.ID)
+				err = nil
+
+				errData := &model.FailData{
+					ChildID: job.ID,
+				}
+				result.FailData = append(result.FailData, errData)
+
+				continue
+			}
+			result.TotalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// SyncProjectJobsByTime ...
+func (s *Service) SyncProjectJobsByTime(projectID int, projectName string, since, until time.Time) (result *model.SyncResult, err error) {
+	var (
+		jobs       []gitlab.Job
+		resp       *gitlab.Response
+		startQuery bool
+	)
+	result = &model.SyncResult{}
+
+	if _, resp, err = s.gitlab.ListProjectJobs(projectID, 1); err != nil {
+		return
+	}
+
+	page := 1
+	for page <= resp.TotalPages {
+		result.TotalPage++
+
+		if !startQuery {
+			if jobs, _, err = s.gitlab.ListProjectJobs(projectID, page); err != nil {
+				return
+			}
+			if page == 1 && len(jobs) <= 0 {
+				return
+			}
+
+			if jobs[0].CreatedAt.After(until) {
+				page++
+				continue
+			} else {
+				startQuery = true
+				page--
+				continue
+			}
+		}
+
+		if jobs, _, err = s.gitlab.ListProjectJobs(projectID, page); err != nil {
+			return
+		}
+
+		for _, job := range jobs {
+
+			createTime := job.CreatedAt
+			if createTime.After(since) && createTime.Before(until) {
+
+				if err = s.structureDatabasejob(projectID, projectName, job); err != nil {
+					log.Error("job Save Database err: projectID(%d), JobID(%d)", projectID, job.ID)
+					err = nil
+
+					errData := &model.FailData{
+						ChildID: job.ID,
+					}
+					result.FailData = append(result.FailData, errData)
+
+					continue
+				}
+				result.TotalNum++
+			}
+
+			if createTime.Before(since) {
+				return
+			}
+		}
+		page++
+	}
+
+	return
+}
+
+// structureDatabasejob ...
+func (s *Service) structureDatabasejob(projectID int, projectName string, job gitlab.Job) (err error) {
+	var (
+		jobArtifactsFile string
+		jobCommitID      string
+	)
+
+	jobArtifactsFileByte, _ := json.Marshal(job.ArtifactsFile)
+	jobArtifactsFile = string(jobArtifactsFileByte)
+	if job.Commit != nil {
+		jobCommitID = job.Commit.ID
+	}
+	jobDB := &model.StatisticsJobs{
+		ProjectID:         projectID,
+		ProjectName:       projectName,
+		CommitID:          jobCommitID,
+		CreatedAt:         job.CreatedAt,
+		Coverage:          job.Coverage,
+		ArtifactsFile:     jobArtifactsFile,
+		FinishedAt:        job.FinishedAt,
+		JobID:             job.ID,
+		Name:              job.Name,
+		Ref:               job.Ref,
+		RunnerID:          job.Runner.ID,
+		RunnerDescription: job.Runner.Description,
+		Stage:             job.Stage,
+		StartedAt:         job.StartedAt,
+		Status:            job.Status,
+		Tag:               job.Tag,
+		UserID:            job.User.ID,
+		UserName:          job.User.Name,
+		WebURL:            job.WebURL,
+	}
+
+	return s.SaveDatabasejob(jobDB)
+}
+
+// SaveDatabasejob ...
+func (s *Service) SaveDatabasejob(jobDB *model.StatisticsJobs) (err error) {
+	var total int
+
+	if total, err = s.dao.HasJob(jobDB.ProjectID, jobDB.JobID); err != nil {
+		log.Error("SaveDatabaseJob HasJob(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if err = s.dao.UpdateJob(jobDB.ProjectID, jobDB.JobID, jobDB); err != nil {
+			log.Error("SaveDatabaseJob UpdateJob(%+v)", err)
+			return
+		}
+
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabasejob job has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateJob(jobDB); err != nil {
+		log.Error("SaveDatabaseJob CreateJob(%+v)", err)
+		return
+	}
+
+	return
+}

app/admin/ep/saga/service/mail/BUILD.bazel

@@ -0,0 +1,50 @@
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "mail.go",
+        "tpl.go",
+    ],
+    importpath = "go-common/app/admin/ep/saga/service/mail",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["mail_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/ep/saga/service/mail/mail.go

@@ -0,0 +1,112 @@
+package mail
+
+import (
+	"bytes"
+	"fmt"
+	"text/template"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+
+	gomail "gopkg.in/gomail.v2"
+)
+
+// SendMail2 ...
+func SendMail2(addr *model.MailAddress, subject string, data string) (err error) {
+	var (
+		msg = gomail.NewMessage()
+	)
+	msg.SetAddressHeader("From", conf.Conf.Property.Mail.Address, conf.Conf.Property.Mail.Name)
+	msg.SetHeader("To", msg.FormatAddress(addr.Address, addr.Name))
+	msg.SetHeader("Subject", subject)
+	msg.SetBody("text/plain", data)
+	d := gomail.NewDialer(
+		conf.Conf.Property.Mail.Host,
+		conf.Conf.Property.Mail.Port,
+		conf.Conf.Property.Mail.Address,
+		conf.Conf.Property.Mail.Pwd,
+	)
+	if err = d.DialAndSend(msg); err != nil {
+		err = errors.WithMessage(err, fmt.Sprintf("Send mail (%+v,%s,%s) failed", addr, subject, data))
+		return
+	}
+	log.Info("Send mail  (%+v,%s,%s) success", addr, subject, data)
+	return
+}
+
+// SendMail  send mail
+func SendMail(m *model.Mail, data *model.MailData) (err error) {
+	var (
+		toUsers []string
+		msg     = gomail.NewMessage()
+		buf     = &bytes.Buffer{}
+	)
+	msg.SetAddressHeader("From", conf.Conf.Property.Mail.Address, conf.Conf.Property.Mail.Name) // 发件人
+	for _, ads := range m.ToAddress {
+		toUsers = append(toUsers, msg.FormatAddress(ads.Address, ads.Name))
+	}
+	t := template.New("MR Mail")
+	if t, err = t.Parse(mailTPL); err != nil {
+		log.Error("tpl.Parse(%s) error(%+v)", mailTPL, errors.WithStack(err))
+		return
+	}
+	err = t.Execute(buf, data)
+	if err != nil {
+		log.Error("t.Execute error(%+v)", errors.WithStack(err))
+		return
+	}
+	msg.SetHeader("To", toUsers...)
+	msg.SetHeader("Subject", m.Subject)    // 主题
+	msg.SetBody("text/html", buf.String()) // 正文
+	d := gomail.NewDialer(
+		conf.Conf.Property.Mail.Host,
+		conf.Conf.Property.Mail.Port,
+		conf.Conf.Property.Mail.Address,
+		conf.Conf.Property.Mail.Pwd,
+	)
+	if err = d.DialAndSend(msg); err != nil {
+		log.Error("Send mail Fail(%v) diff(%s)", msg, err)
+		return
+	}
+	return
+}
+
+// SendMail3 SendMail all parameter
+func SendMail3(from string, sender string, senderPwd string, m *model.Mail, data *model.MailData) (err error) {
+	var (
+		toUsers []string
+		msg     = gomail.NewMessage()
+		buf     = &bytes.Buffer{}
+	)
+	msg.SetAddressHeader("From", from, sender) // 发件人
+	for _, ads := range m.ToAddress {
+		toUsers = append(toUsers, msg.FormatAddress(ads.Address, ads.Name))
+	}
+	t := template.New("MR Mail")
+	if t, err = t.Parse(mailTPL3); err != nil {
+		log.Error("tpl.Parse(%s) error(%+v)", mailTPL3, errors.WithStack(err))
+		return
+	}
+	err = t.Execute(buf, data)
+	if err != nil {
+		log.Error("t.Execute error(%+v)", errors.WithStack(err))
+		return
+	}
+	msg.SetHeader("To", toUsers...)
+	msg.SetHeader("Subject", m.Subject)    // 主题
+	msg.SetBody("text/html", buf.String()) // 正文
+	d := gomail.NewDialer(
+		"smtp.exmail.qq.com",
+		465,
+		from,
+		senderPwd,
+	)
+	if err = d.DialAndSend(msg); err != nil {
+		log.Error("Send mail Fail(%v) diff(%s)", msg, err)
+		return
+	}
+	return
+}

app/admin/ep/saga/service/mail/mail_test.go

@@ -0,0 +1,46 @@
+package mail
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func init() {
+	var err error
+	flag.Set("conf", "../../cmd/saga-admin-test.toml")
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+}
+
+// go test  -test.v -test.run TestMail
+func TestMail(t *testing.T) {
+	Convey("Test mail", t, func() {
+		m := &model.Mail{
+			ToAddress: []*model.MailAddress{{Name: "baihai", Address: "yubaihai@bilibili.com"},
+				{Name: "muyan", Address: "muyang@bilibili.com"}},
+			Subject: fmt.Sprintf("【Sage 提醒】%s项目发生Merge Request事件", "kk"),
+		}
+		mergeOut := " Merge made by the 'recursive' strategy.\n" +
+			"tools/saga/CHANGELOG.md          |  4 ++++\n" +
+			"business/interface/app-show/service/rank/rank.go  | 28 +++++++++++------------\n" +
+			"business/interface/app-show/service/show/cache.go |  6 ++---\n" +
+			"3 files changed, 21 insertions(+), 17 deletions(-)"
+		err := SendMail(m, &model.MailData{
+			UserName:     "baihai",
+			SourceBranch: "featre_answer",
+			TargetBranch: "master",
+			Title:        "修改变量A",
+			Description:  "内容就是",
+			URL:          "http://www.baidu.com",
+			Info:         mergeOut,
+		})
+		So(err, ShouldBeNil)
+	})
+}

app/admin/ep/saga/service/mail/tpl.go

@@ -0,0 +1,130 @@
+package mail
+
+var (
+	mailTPL = `<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>saga page</title>
+</head>
+<body>
+<form action="" method="post" class="basic-grey" style="margin-left:auto;
+margin-right:auto;
+max-width: 500px;
+background: #F7F7F7;
+padding: 25px 15px 25px 10px;
+font: 12px Georgia, 'Times New Roman', Times, serif;
+color: #888;
+text-shadow: 1px 1px 1px #FFF;
+border:1px solid #E4E4E4;">
+<h1 style="font-size: 25px;
+padding: 0px 0px 10px 40px;
+display: block;
+border-bottom:1px solid #E4E4E4;
+margin: -10px -15px 30px -10px;;
+color: #888;">
+    Saga 
+<span style="display: block;font-size: 11px;">
+    Merge Request 事件通知</span>
+</h1>
+<label style="display: block;margin: 0px;">
+<span style="float: left;
+width:100%;
+text-align: left;
+padding-right: 10px;
+padding-left: 30px;
+margin-top: 10px;
+color: #888;">
+申请人 : {{.UserName}}
+<br />
+来源分支 : {{.SourceBranch}}
+<br />
+目标分支 : {{.TargetBranch}}
+<br />
+修改标题 : {{.Title}}
+<br />
+修改说明 : {{.Description}}
+<br />
+<a href="{{.URL}}">点击查看..</a>
+<br />
+<br />
+<br />
+<h3>额外信息: </h3>
+{{.Info}}
+<br />
+<br />
+<br />
+</span>
+</label>
+</form>
+</body>
+</html>`
+	mailTPL3 = `<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>saga page</title>
+</head>
+<body>
+<form action="" method="post" class="basic-grey" style="
+margin-right:auto;
+max-width: 500px;
+font: 12px Georgia, 'Times New Roman', Times, serif;
+color: #888;
+text-shadow: 1px 1px 1px #FFF;">
+<h1 style="font-size: 25px;
+padding: 10px 0px 10px 40px;
+display: block;
+border-bottom:1px solid #E4E4E4;
+margin: -10px -15px 5px -10px;;
+color: #03A9F4;">
+    Saga 
+<span style="display: block;font-size: 11px;">
+	事件通知</span>
+</h1>
+</form>
+<label style="display: block;margin: 0px;">
+<span style="float: left;
+width:100%;
+text-align: left;
+padding-right: 10px;
+padding-left: 30px;
+margin-top: 10px;
+color: #888;">
+执行状态 :
+<font class="{{.PipelineStatus}}" >
+ {{.PipeStatus}}
+ </font>
+<br />
+Pipeline信息:
+<font class="{{.PipelineStatus}}" >
+<a href="{{.URL}}">{{.URL}}</a>
+ </font>
+<br />
+来源分支 : {{.SourceBranch}}
+<br />
+修改说明 : {{.Description}}
+<br />
+额外信息:  {{.Info}}
+<br />
+<br />
+<br />
+</span>
+</label>
+</body>
+<style type="text/css">
+.failed {
+	color: #f21303;
+}
+.failed a{
+	color: #f21303;
+}
+.success {
+	color: #1aaa55;
+}
+.success a{
+	color: #1aaa55;
+}
+</style>
+</html>`
+)

app/admin/ep/saga/service/member.go

@@ -0,0 +1,110 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// QueryProjectMembers ...
+func (s *Service) QueryProjectMembers(c context.Context, req *model.ProjectDataReq) (resp []*gitlab.ProjectMember, err error) {
+
+	var (
+		members    []*gitlab.ProjectMember
+		response   *gitlab.Response
+		memberItem []*gitlab.ProjectMember
+	)
+	if members, response, err = s.gitlab.ListProjectMembers(req.ProjectID, 1); err != nil {
+		return
+	}
+
+	page := 2
+	for page <= response.TotalPages {
+		memberItem, _, err = s.gitlab.ListProjectMembers(req.ProjectID, page)
+		members = append(members, memberItem...)
+		page++
+	}
+	resp = members
+	return
+}
+
+/*-------------------------------------- sync member ----------------------------------------*/
+
+// SyncProjectMember ...
+func (s *Service) SyncProjectMember(c context.Context, projectID int) (totalPage, totalNum int, err error) {
+	var (
+		resp        *gitlab.Response
+		members     []*gitlab.ProjectMember
+		projectInfo *model.ProjectInfo
+	)
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	for page := 1; ; page++ {
+		totalPage++
+		if members, resp, err = s.gitlab.ListProjectMembers(projectID, page); err != nil {
+			return
+		}
+		for _, member := range members {
+			memberDB := &model.StatisticsMembers{
+				ProjectID:   projectID,
+				ProjectName: projectInfo.Name,
+				MemberID:    member.ID,
+				Username:    member.Username,
+				Email:       member.Email,
+				Name:        member.Name,
+				State:       member.State,
+				CreatedAt:   member.CreatedAt,
+				AccessLevel: int(member.AccessLevel),
+			}
+			if err = s.SaveDatabaseMember(c, memberDB); err != nil {
+				log.Error("member Save Database err: projectID(%d), MemberID(%d)", projectID, member.ID)
+				err = nil
+				continue
+			}
+			totalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+
+	return
+}
+
+// SaveDatabaseMember ...
+func (s *Service) SaveDatabaseMember(c context.Context, memberDB *model.StatisticsMembers) (err error) {
+	var total int
+
+	if total, err = s.dao.HasMember(c, memberDB.ProjectID, memberDB.MemberID); err != nil {
+		log.Error("SaveDatabaseMember HasMember(%+v)", err)
+		return
+	}
+
+	// update found row
+	if total == 1 {
+		if err = s.dao.UpdateMember(c, memberDB.ProjectID, memberDB.MemberID, memberDB); err != nil {
+			log.Error("SaveDatabaseMember UpdateMember(%+v)", err)
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseMember Note has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateMember(c, memberDB); err != nil {
+		log.Error("SaveDatabaseMember CreateMember(%+v)", err)
+		return
+
+	}
+
+	return
+}

app/admin/ep/saga/service/mr.go

@@ -0,0 +1,590 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+const _specialMrID = 10199
+
+// QueryProjectMr query project commit info according to project id.
+func (s *Service) QueryProjectMr(c context.Context, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) {
+
+	if resp, err = s.QueryProject(c, model.ObjectMR, req); err != nil {
+		return
+	}
+	return
+}
+
+// QueryTeamMr query team commit info according to department and business
+func (s *Service) QueryTeamMr(c context.Context, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) {
+
+	if resp, err = s.QueryTeam(c, model.ObjectMR, req); err != nil {
+		return
+	}
+	return
+}
+
+// QueryProjectMrReport query mr review
+func (s *Service) QueryProjectMrReport(c context.Context, req *model.ProjectMrReportReq) (resp *model.ProjectMrReportResp, err error) {
+	var (
+		info               []*model.MrInfo
+		changeAdd          int
+		changeDel          int
+		mrCount            int
+		stateCount         int
+		discussionCount    int
+		discussionResolved int
+		mrTime             int
+		spentTime          time.Duration
+		avarageTime        time.Duration
+		reviewers          []string
+		reviews            []string
+		reviewChangeAdd    int
+		reviewChangeDel    int
+		reviewTime         int
+		reviewTotalTime    time.Duration
+	)
+	if info, err = s.QueryAllMergeRequestInfo(c, req.ProjectID); err != nil {
+		return
+	}
+
+	for _, i := range info {
+		for _, r := range i.Reviewers {
+			if r.Name == req.Member {
+				reviews = append(reviews, i.Author)
+				reviewChangeAdd += i.ChangeAdd
+				reviewChangeDel += i.ChangeDel
+				reviewTime += i.SpentTime
+			}
+		}
+		if i.Author == req.Member {
+			mrCount++
+			if i.State == model.StatusMerged {
+				stateCount++
+				mrTime += i.SpentTime
+				changeAdd += i.ChangeAdd
+				changeDel += i.ChangeDel
+				discussionCount += i.TotalDiscussion
+				discussionResolved += i.SolvedDiscussion
+				for _, r := range i.Reviewers {
+					reviewers = append(reviewers, r.Name)
+				}
+			}
+		}
+	}
+	// 判断mr为零的情况
+	if mrCount == 0 {
+		resp = &model.ProjectMrReportResp{}
+		return
+	}
+
+	if spentTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime)); err != nil {
+		return
+	}
+	if avarageTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime/mrCount)); err != nil {
+		return
+	}
+	if reviewTotalTime, err = time.ParseDuration(fmt.Sprintf("%ds", reviewTime)); err != nil {
+		return
+	}
+
+	resp = &model.ProjectMrReportResp{
+		ChangeAdd:       changeAdd,
+		ChangeDel:       changeDel,
+		MrCount:         mrCount,
+		SpentTime:       spentTime.String(),
+		StateCount:      stateCount,
+		AverageMerge:    avarageTime.String(),
+		Reviewers:       reviewers,
+		Discussion:      discussionCount,
+		Resolve:         discussionResolved,
+		ReviewerOther:   reviews,
+		ReviewChangeAdd: reviewChangeAdd,
+		ReviewChangeDel: reviewChangeDel,
+		ReviewTotalTime: reviewTotalTime.String(),
+	}
+
+	return
+}
+
+// QueryAllMergeRequestInfo ...
+func (s *Service) QueryAllMergeRequestInfo(c context.Context, projID int) (info []*model.MrInfo, err error) {
+	var (
+		until = time.Now()
+		since = until.AddDate(0, -1, 0)
+		mrs   []*gitlab.MergeRequest
+		resp  *gitlab.Response
+	)
+
+	for page := 1; ; page++ {
+		if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projID, &since, &until, page); err != nil {
+			return
+		}
+		for _, m := range mrs {
+
+			mr := &model.MrInfo{ProjectID: projID, MrID: m.IID, Author: m.Author.Name, State: m.State}
+
+			if m.State == model.StatusMerged {
+				spent := m.UpdatedAt.Sub(*m.CreatedAt)
+				mr.SpentTime = int(spent.Seconds())
+			}
+
+			if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projID, m.IID); err != nil {
+				return
+			}
+
+			if mr.Reviewers, err = s.QueryMergeRequestReview(c, projID, m.IID); err != nil {
+				return
+			}
+
+			if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projID, m.IID); err != nil {
+				return
+			}
+
+			info = append(info, mr)
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+
+	return
+}
+
+// QueryMergeRequestReview 查询mr reviewer信息
+func (s *Service) QueryMergeRequestReview(c context.Context, projectID, mrIID int) (reviewers []*model.MrReviewer, err error) {
+	var (
+		notes  []*model.StatisticsNotes
+		emojis []*model.StatisticsMRAwardEmojis
+		owners []string
+		r      *regexp.Regexp
+	)
+
+	//query note
+	if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
+		return
+	}
+
+	// query emoji
+	if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
+		return
+	}
+
+	//评论中解析获取owner
+	if len(notes) == 0 {
+		return
+	}
+	if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil {
+		matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
+		if len(matchResult) > 0 {
+			owners = strings.Split(matchResult[1], " 或 @")
+		}
+	}
+
+	mrCreatedAt := notes[len(notes)-1].CreatedAt
+
+	// 从评论和表情中解析 reviewers
+	for _, note := range notes {
+		if note.Body == "+1" {
+			reviewers = append(reviewers, &model.MrReviewer{Name: note.AuthorName, FinishedAt: note.CreatedAt, SpentTime: int(note.CreatedAt.Sub(*mrCreatedAt))})
+		}
+	}
+	for _, emoji := range emojis {
+		if emoji.Name == "thumbsup" {
+			reviewers = append(reviewers, &model.MrReviewer{Name: emoji.UserName, FinishedAt: emoji.CreatedAt, SpentTime: int(emoji.CreatedAt.Sub(*mrCreatedAt))})
+		}
+	}
+
+	// 判断reviewer类型
+	for _, r := range reviewers {
+		for _, owner := range owners {
+			if owner == r.Name {
+				r.UserType = "owner"
+				break
+			}
+		}
+		if r.UserType == "" {
+			r.UserType = "other"
+		}
+	}
+
+	return
+}
+
+// QueryMergeRequestDiscussion 查询获取mr的discussion
+func (s *Service) QueryMergeRequestDiscussion(c context.Context, projectID, mrIID int) (total, solved int, err error) {
+	var discussions []*model.StatisticsDiscussions
+	if discussions, err = s.dao.DiscussionsByMRIID(c, projectID, mrIID); err != nil {
+		return
+	}
+	for _, d := range discussions {
+		var notesArray []int
+		if err = json.Unmarshal([]byte(d.Notes), &notesArray); err != nil {
+			return
+		}
+		for _, n := range notesArray {
+			var note *model.StatisticsNotes
+			if note, err = s.dao.NoteByID(c, projectID, mrIID, n); err != nil {
+				return
+			}
+			if note.Resolvable {
+				total++
+				if note.Resolved {
+					solved++
+				}
+			}
+		}
+	}
+	return
+}
+
+// QueryMergeRequestDiff 查询获取到mr修改文件的总行数, 参数 p project_ID m MR_ID
+func (s *Service) QueryMergeRequestDiff(c context.Context, p, m int) (ChangeAdd, ChangeDel int, err error) {
+	var mr *gitlab.MergeRequest
+
+	// 此MR数据量太大，曾导致过gitlab服务器崩溃，访问会返回502服务器错误，因此特殊处理。
+	if m == _specialMrID {
+		return 0, 0, nil
+	}
+
+	if mr, _, err = s.gitlab.GetMergeRequestDiff(p, m); err != nil {
+		return
+	}
+	for _, change := range mr.Changes {
+		rows := strings.Split(change.Diff, "\n")
+		if len(rows) < 3 {
+			// 处理diff为空
+			continue
+		}
+
+		for _, row := range rows[3:] {
+			if strings.HasPrefix(row, "+") {
+				ChangeAdd++
+			} else if strings.HasPrefix(row, "-") {
+				ChangeDel++
+			}
+		}
+	}
+
+	return
+}
+
+/*-------------------------------------- sync MR ----------------------------------------*/
+
+// SyncProjectMR ...
+func (s *Service) SyncProjectMR(c context.Context, projectID int) (result *model.SyncResult, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		mrs         []*gitlab.MergeRequest
+		resp        *gitlab.Response
+		since       *time.Time
+		until       *time.Time
+		projectInfo *model.ProjectInfo
+	)
+	result = &model.SyncResult{}
+
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+	log.Info("sync project(%d) MR time since: %v, until: %v", projectID, since, until)
+	for page := 1; ; page++ {
+		result.TotalPage++
+		if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projectID, since, until, page); err != nil {
+			return
+		}
+
+		for _, mr := range mrs {
+			if err = s.structureDatabaseMR(c, projectID, projectInfo.Name, mr); err != nil {
+				log.Error("mr Save Database err: projectID(%d), MRIID(%d)", projectID, mr.IID)
+				err = nil
+
+				errData := &model.FailData{
+					ChildID: mr.IID,
+				}
+				result.FailData = append(result.FailData, errData)
+				continue
+			}
+			result.TotalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// structureDatabaseMR ...
+func (s *Service) structureDatabaseMR(c context.Context, projectID int, projectName string, mr *gitlab.MergeRequest) (err error) {
+	var (
+		milestoneID   int
+		mrLables      string
+		mrChanges     string
+		mrLablesByte  []byte
+		mrChangesByte []byte
+	)
+
+	if mr.Milestone != nil {
+		milestoneID = mr.Milestone.ID
+	}
+	if mrLablesByte, err = json.Marshal(mr.Labels); err != nil {
+		mrLables = model.JsonMarshalErrorText
+	} else {
+		mrLables = string(mrLablesByte)
+	}
+
+	if mrChangesByte, err = json.Marshal(mr.Changes); err != nil {
+		mrChanges = model.JsonMarshalErrorText
+	} else {
+		mrChanges = string(mrChangesByte)
+	}
+
+	mrDB := &model.StatisticsMrs{
+		MRID:                         mr.ID,
+		MRIID:                        mr.IID,
+		TargetBranch:                 mr.TargetBranch,
+		SourceBranch:                 mr.SourceBranch,
+		ProjectID:                    mr.ProjectID,
+		ProjectName:                  projectName,
+		Title:                        mr.Title,
+		State:                        mr.State,
+		CreatedAt:                    mr.CreatedAt,
+		UpdatedAt:                    mr.UpdatedAt,
+		Upvotes:                      mr.Upvotes,
+		Downvotes:                    mr.Downvotes,
+		AuthorID:                     mr.Author.ID,
+		AuthorName:                   mr.Author.Name,
+		AssigneeID:                   mr.Assignee.ID,
+		AssigneeName:                 mr.Assignee.Name,
+		SourceProjectID:              mr.SourceProjectID,
+		TargetProjectID:              mr.TargetProjectID,
+		Labels:                       mrLables,
+		Description:                  mr.Description,
+		WorkInProgress:               mr.WorkInProgress,
+		MilestoneID:                  milestoneID,
+		MergeWhenPipelineSucceeds:    mr.MergeWhenPipelineSucceeds,
+		MergeStatus:                  mr.MergeStatus,
+		MergedByID:                   mr.MergedBy.ID,
+		MergedByName:                 mr.MergedBy.Name,
+		MergedAt:                     mr.MergedAt,
+		ClosedByID:                   mr.ClosedBy.ID,
+		ClosedAt:                     mr.ClosedAt,
+		Subscribed:                   mr.Subscribed,
+		SHA:                          mr.SHA,
+		MergeCommitSHA:               mr.MergeCommitSHA,
+		UserNotesCount:               mr.UserNotesCount,
+		ChangesCount:                 mr.ChangesCount,
+		ShouldRemoveSourceBranch:     mr.ShouldRemoveSourceBranch,
+		ForceRemoveSourceBranch:      mr.ForceRemoveSourceBranch,
+		WebURL:                       mr.WebURL,
+		DiscussionLocked:             mr.DiscussionLocked,
+		Changes:                      mrChanges,
+		TimeStatsHumanTimeEstimate:   mr.TimeStats.HumanTimeEstimate,
+		TimeStatsHumanTotalTimeSpent: mr.TimeStats.HumanTotalTimeSpent,
+		TimeStatsTimeEstimate:        mr.TimeStats.TimeEstimate,
+		TimeStatsTotalTimeSpent:      mr.TimeStats.TotalTimeSpent,
+		Squash:                       mr.Squash,
+		PipelineID:                   mr.Pipeline.ID,
+	}
+	if len(mrDB.Labels) > model.MessageMaxLen {
+		mrDB.Labels = mrDB.Labels[0 : model.MessageMaxLen-1]
+	}
+	if len(mrDB.Description) > model.MessageMaxLen {
+		mrDB.Description = mrDB.Description[0 : model.MessageMaxLen-1]
+	}
+	return s.SaveDatabaseMR(c, mrDB)
+}
+
+// SaveDatabaseMR ...
+func (s *Service) SaveDatabaseMR(c context.Context, mrDB *model.StatisticsMrs) (err error) {
+	var total int
+
+	if total, err = s.dao.HasMR(c, mrDB.ProjectID, mrDB.MRIID); err != nil {
+		log.Error("SaveDatabaseMR HasMR(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		return s.dao.UpdateMR(c, mrDB.ProjectID, mrDB.MRIID, mrDB)
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseMR mr has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	return s.dao.CreateMR(c, mrDB)
+}
+
+/*-------------------------------------- agg MR ----------------------------------------*/
+
+// AggregateProjectMR ...
+func (s *Service) AggregateProjectMR(c context.Context, projectID int) (err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		mrs         []*model.StatisticsMrs
+		projectInfo *model.ProjectInfo
+		since       *time.Time
+		until       *time.Time
+	)
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	if !syncAllTime {
+		since, until = utils.CalSyncTime()
+	}
+
+	if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
+		return
+	}
+
+	for _, mr := range mrs {
+		if err = s.MRAddedInfoDB(c, projectID, mr); err != nil {
+			log.Error("MRAddedInfoDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
+			err = nil
+		}
+		if err = s.MRReviewerDB(c, projectID, mr.MRIID, projectInfo.Name, mr); err != nil {
+			log.Error("MRReviewerDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
+			err = nil
+		}
+	}
+	return
+}
+
+// MRAddedInfoDB ...
+func (s *Service) MRAddedInfoDB(c context.Context, projectID int, mr *model.StatisticsMrs) (err error) {
+	if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projectID, mr.MRIID); err != nil {
+		return
+	}
+	if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projectID, mr.MRIID); err != nil {
+		return
+	}
+	return s.dao.UpdateMR(c, projectID, mr.MRIID, mr)
+}
+
+// MRReviewerDB 查询mr reviewer信息
+func (s *Service) MRReviewerDB(c context.Context, projectID, mrIID int, projectName string, mr *model.StatisticsMrs) (err error) {
+	var (
+		notes     []*model.StatisticsNotes
+		emojis    []*model.StatisticsMRAwardEmojis
+		owners    []string
+		r         *regexp.Regexp
+		reviewers []*model.AggregateMrReviewer
+	)
+
+	//query note
+	if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
+		return
+	}
+
+	// query emoji
+	if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
+		return
+	}
+
+	//评论中解析获取owner
+	if len(notes) == 0 {
+		return
+	}
+	if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil {
+		matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
+		if len(matchResult) > 0 {
+			owners = strings.Split(matchResult[1], " 或 @")
+		}
+	}
+
+	mrCreatedAt := notes[len(notes)-1].CreatedAt
+
+	// 从评论和表情中解析 reviewers
+	for _, note := range notes {
+		if note.Body == "+1" {
+			reviewer := &model.AggregateMrReviewer{
+				ReviewerID:    note.AuthorID,
+				ReviewerName:  note.AuthorName,
+				ReviewType:    "note",
+				ReviewID:      note.NoteID,
+				ReviewCommand: "+1",
+				CreatedAt:     note.CreatedAt,
+				ApproveTime:   int(note.CreatedAt.Sub(*mrCreatedAt).Seconds()),
+				MergeTime:     int(mr.UpdatedAt.Sub(*note.CreatedAt).Seconds()),
+			}
+			reviewers = append(reviewers, reviewer)
+		}
+	}
+	for _, emoji := range emojis {
+		if emoji.Name == "thumbsup" {
+			reviewer := &model.AggregateMrReviewer{
+				ReviewerID:    emoji.UserID,
+				ReviewerName:  emoji.UserName,
+				ReviewType:    "emoji",
+				ReviewID:      emoji.AwardEmojiID,
+				ReviewCommand: "thumbsup",
+				CreatedAt:     emoji.CreatedAt,
+				ApproveTime:   int(emoji.CreatedAt.Sub(*mrCreatedAt).Seconds()),
+				MergeTime:     int(mr.UpdatedAt.Sub(*emoji.CreatedAt).Seconds()),
+			}
+			reviewers = append(reviewers, reviewer)
+		}
+	}
+
+	// 判断reviewer类型
+	for _, r := range reviewers {
+		r.ProjectID = projectID
+		r.ProjectName = projectName
+		r.MrIID = mrIID
+		r.Title = mr.Title
+		r.WebUrl = mr.WebURL
+		r.AuthorName = mr.AuthorName
+		if utils.InSlice(r.ReviewerName, owners) {
+			r.UserType = "owner"
+		} else {
+			r.UserType = "other"
+		}
+		if err = s.SaveDatabaseAggMR(c, r); err != nil {
+			log.Error("mrReviewer 存数据库报错(%+V)", err)
+			continue
+		}
+	}
+	return
+}
+
+// SaveDatabaseAggMR ...
+func (s *Service) SaveDatabaseAggMR(c context.Context, mrDB *model.AggregateMrReviewer) (err error) {
+	var total int
+
+	if total, err = s.dao.HasAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID); err != nil {
+		log.Error("SaveDatabaseAggMR HasAggMR(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		return s.dao.UpdateAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID, mrDB)
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseAggMR aggMR has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	return s.dao.CreateAggregateReviewer(c, mrDB)
+}

app/admin/ep/saga/service/pipeline.go

@@ -0,0 +1,838 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/utils"
+	"go-common/app/admin/ep/saga/service/wechat"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+const (
+	_gitHTTP            = "http://git.bilibili.co/"
+	_gitSSH             = "git@git-test.bilibili.co:"
+	_gitSSHTail         = ".git"
+	_manualJob          = "manual"
+	_androidScheduleJob = "daily branch check"
+	_iosScheduleJob     = "daily:on-schedule"
+)
+
+// QueryTeamPipeline query pipeline info according to team.
+func (s *Service) QueryTeamPipeline(c context.Context, req *model.TeamDataRequest) (resp *model.PipelineDataResp, err error) {
+	var (
+		projectInfo []*model.ProjectInfo
+		reqProject  = &model.ProjectInfoRequest{}
+
+		data        []*model.PipelineDataTime
+		queryDes    string
+		total       int
+		succNum     int
+		key         string
+		keyNotExist bool
+	)
+
+	if len(req.Department) <= 0 && len(req.Business) <= 0 {
+		log.Warn("query department and business are empty!")
+		return
+	}
+
+	//get pipeline info from mc
+	key = "saga_admin_" + req.Department + "_" + req.Business + "_" + model.KeyTypeConst[3]
+	if resp, err = s.dao.GetPipeline(c, key); err != nil {
+		if err == memcache.ErrNotFound {
+			keyNotExist = true
+		} else {
+			return
+		}
+	} else {
+		return
+	}
+
+	log.Info("sync team pipeline start => type= %d, Department= %s, Business= %s", req.QueryType, req.Department, req.Business)
+
+	//query team projects
+	reqProject.Department = req.Department
+	reqProject.Business = req.Business
+	if _, projectInfo, err = s.dao.QueryProjectInfo(false, reqProject); err != nil {
+		return
+	}
+	if len(projectInfo) <= 0 {
+		log.Warn("Found no project!")
+		return
+	}
+
+	if data, total, succNum, err = s.QueryTeamPipelineByTime(projectInfo, model.LastWeekPerDay); err != nil {
+		return
+	}
+
+	successScale := succNum * 100 / total
+	queryDes = req.Department + " " + req.Business + " " + "pipeline上一周每天数量"
+	resp = &model.PipelineDataResp{
+		Department:   req.Department,
+		Business:     req.Business,
+		QueryDes:     queryDes,
+		Total:        total,
+		SuccessNum:   succNum,
+		SuccessScale: successScale,
+		Data:         data,
+	}
+
+	//set pipeline info to mc
+	if keyNotExist {
+		if err = s.dao.SetPipeline(c, key, resp); err != nil {
+			return
+		}
+	}
+
+	log.Info("sync team pipeline end")
+	return
+}
+
+// QueryTeamPipelineByTime ...
+func (s *Service) QueryTeamPipelineByTime(projectInfo []*model.ProjectInfo, queryType int) (resp []*model.PipelineDataTime, allNum, succNum int, err error) {
+	var (
+		layout = "2006-01-02"
+		since  time.Time
+		until  time.Time
+
+		total   int
+		success int
+		count   int
+	)
+
+	if queryType == model.LastWeekPerDay {
+		count = model.DayNumPerWeek
+	} else {
+		log.Warn("Query Type is not in range!")
+		return
+	}
+
+	year, month, day := time.Now().Date()
+	weekDay := (int)(time.Now().Weekday())
+	today := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
+
+	for i := 0; i < count; i++ {
+		since = today.AddDate(0, 0, -weekDay-i)
+		until = today.AddDate(0, 0, -weekDay-i+1)
+
+		totalAll := 0
+		successAll := 0
+		//log.Info("== start query from: %v, to: %v", since, until)
+		for _, project := range projectInfo {
+			if total, success, err = s.QueryProjectPipeline(project.ProjectID, "success", since, until); err != nil {
+				return
+			}
+			totalAll = totalAll + total
+			successAll = successAll + success
+		}
+
+		perData := &model.PipelineDataTime{
+			TotalItem:   totalAll,
+			SuccessItem: successAll,
+			StartTime:   since.Format(layout),
+			EndTime:     until.Format(layout),
+		}
+		resp = append(resp, perData)
+		allNum = allNum + totalAll
+		succNum = succNum + successAll
+	}
+
+	return
+}
+
+// QueryProjectPipeline query pipeline info according to project id.
+func (s *Service) QueryProjectPipeline(projectID int, state string, since, until time.Time) (totalNum, stateNum int, err error) {
+	var (
+		pipelineList gitlab.PipelineList
+		pipeline     *gitlab.Pipeline
+		resp         *gitlab.Response
+		startQuery   bool
+	)
+
+	if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
+		return
+	}
+
+	page := 1
+	for page <= resp.TotalPages {
+
+		if !startQuery {
+			if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+				return
+			}
+			if page == 1 && len(pipelineList) <= 0 {
+				return
+			}
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelineList[0].ID); err != nil {
+				return
+			}
+
+			if pipeline.CreatedAt.After(until) {
+				page++
+				continue
+			} else {
+				startQuery = true
+				page--
+				continue
+			}
+		}
+
+		if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+			return
+		}
+
+		for _, v := range pipelineList {
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
+				return
+			}
+
+			createTime := pipeline.CreatedAt
+			//year, month, day := createTime.Date()
+			//log.Info("index: %d createTime: %d, month: %d, day: %d", k, year, month, day)
+
+			if createTime.After(since) && createTime.Before(until) {
+				totalNum = totalNum + 1
+				if pipeline.Status == state {
+					stateNum = stateNum + 1
+				}
+			}
+
+			if createTime.Before(since) {
+				return
+			}
+		}
+		page++
+	}
+
+	return
+}
+
+// QueryProjectPipelineNew ...
+func (s *Service) QueryProjectPipelineNew(c context.Context, req *model.PipelineDataReq) (resp *model.PipelineDataAvgResp, err error) {
+	var (
+		data            []*model.PipelineDataAvg
+		queryDes        string
+		total           int
+		totalStatus     int
+		avgDurationTime float64
+		avgPendingTime  float64
+		avgRunningTime  float64
+	)
+
+	log.Info("QuerySingleProjectData Type: %d", req.Type)
+	switch req.Type {
+	case model.LastYearPerMonth:
+		queryDes = model.LastYearPerMonthNote
+	case model.LastMonthPerDay:
+		queryDes = model.LastMonthPerDayNote
+	case model.LastYearPerDay:
+		queryDes = model.LastYearPerDayNote
+	default:
+		log.Warn("QueryProjectCommit Type is not in range")
+		return
+	}
+	queryDes = req.ProjectName + " pipeline " + req.State + " " + queryDes
+
+	if data, total, totalStatus, avgDurationTime, avgPendingTime, avgRunningTime, err = s.QueryProjectByTimeNew(c, req, req.Type); err != nil {
+		return
+	}
+
+	resp = &model.PipelineDataAvgResp{
+		ProjectName:     req.ProjectName,
+		QueryDes:        queryDes,
+		Status:          req.State,
+		Total:           total,
+		TotalStatus:     totalStatus,
+		AvgDurationTime: avgDurationTime,
+		AvgPendingTime:  avgPendingTime,
+		AvgRunningTime:  avgRunningTime,
+		Data:            data,
+	}
+
+	return
+}
+
+// QueryProjectByTimeNew ...
+func (s *Service) QueryProjectByTimeNew(c context.Context, req *model.PipelineDataReq, queryType int) (resp []*model.PipelineDataAvg, allNum, allStatusNum int, avgDurationTime, avgPendingTime, avgRunningTime float64, err error) {
+	var (
+		layout              = "2006-01-02"
+		since               time.Time
+		until               time.Time
+		count               int
+		pendingTimeListAll  []float64
+		runningTimeListAll  []float64
+		durationTimeListAll []float64
+		pipelineTime        *model.PipelineTime
+		perData             *model.PipelineDataAvg
+		avgTotalTime        float64
+		totalNum            int
+		statusNum           int
+	)
+
+	year, month, day := time.Now().Date()
+	thisMonth := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
+
+	if queryType == model.LastYearPerMonth {
+		count = model.MonthNumPerYear
+	} else if queryType == model.LastMonthPerDay {
+		//_, _, count = thisMonth.AddDate(0, 0, -1).Date()
+		count = model.DayNumPerMonth
+	} else if queryType == model.LastYearPerDay {
+		count = model.DayNumPerYear
+	}
+
+	for i := count; i >= 1; i-- {
+
+		if queryType == model.LastYearPerMonth {
+			since = thisMonth.AddDate(0, -i, 0)
+			until = thisMonth.AddDate(0, -i+1, 0)
+		} else if queryType == model.LastMonthPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		} else if queryType == model.LastYearPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		}
+
+		/*if totalNum, statusNum, pipelineTime, err = s.QueryProjectPipelines(c, req, since, until); err != nil {
+			log.Error("QueryProjectPipelines err：%+v", err)
+			return
+		}*/
+		if totalNum, statusNum, pipelineTime, err = s.QueryPipelinesFromDB(c, req, since, until); err != nil {
+			log.Error("QueryPipelinesFromDB err：%+v", err)
+			return
+		}
+
+		avgTotalTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.DurationList)
+		avgPendingTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.PendingList)
+		avgRunningTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.RunningList)
+
+		perData = &model.PipelineDataAvg{
+			TotalItem:       totalNum,
+			TotalStatusItem: statusNum,
+			AvgDurationTime: avgTotalTime,
+			AvgPendingTime:  avgPendingTime,
+			AvgRunningTime:  avgRunningTime,
+			MaxDurationTime: pipelineTime.DurationMax,
+			MinDurationTime: pipelineTime.DurationMin,
+			MaxPendingTime:  pipelineTime.PendingMax,
+			MinPendingTime:  pipelineTime.PendingMin,
+			MaxRunningTime:  pipelineTime.RunningMax,
+			MinRunningTime:  pipelineTime.RunningMin,
+			StartTime:       since.Format(layout),
+			EndTime:         until.Format(layout),
+		}
+		resp = append(resp, perData)
+		allNum = allNum + totalNum
+		allStatusNum = allStatusNum + statusNum
+
+		pendingTimeListAll = utils.CombineSlice(pendingTimeListAll, pipelineTime.PendingList)
+		runningTimeListAll = utils.CombineSlice(runningTimeListAll, pipelineTime.RunningList)
+		durationTimeListAll = utils.CombineSlice(durationTimeListAll, pipelineTime.DurationList)
+	}
+
+	avgDurationTime = utils.CalAverageTime(req.StatisticsType, durationTimeListAll)
+	avgPendingTime = utils.CalAverageTime(req.StatisticsType, pendingTimeListAll)
+	avgRunningTime = utils.CalAverageTime(req.StatisticsType, runningTimeListAll)
+	log.Info("avgDurationTime: %v, avgPendingTime: %v, avgRunningTime: %v", avgDurationTime, avgPendingTime, avgRunningTime)
+
+	return
+}
+
+// QueryProjectPipelines ...
+func (s *Service) QueryProjectPipelines(c context.Context, req *model.PipelineDataReq, since, until time.Time) (totalNum, statusNum int, pipelineTime *model.PipelineTime, err error) {
+	var (
+		pipelineList gitlab.PipelineList
+		pipeline     *gitlab.Pipeline
+		resp         *gitlab.Response
+		startQuery   bool
+		meetTime     bool
+		projectID    = req.ProjectID
+		pendingTime  float64
+		runningTime  float64
+		totalTime    float64
+	)
+
+	pipelineTime = &model.PipelineTime{}
+
+	opt := &gitlab.ListProjectPipelinesOptions{}
+	if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
+		log.Error("ListProjectPipelines err: %+v", err)
+		return
+	}
+
+	page := 1
+	for page <= resp.TotalPages {
+		opt.ListOptions.Page = page
+		if !startQuery && (!since.IsZero() || !until.IsZero()) {
+			if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+				log.Error("ListProjectPipelines err: %+v", err)
+				return
+			}
+			if page == 1 && len(pipelineList) <= 0 {
+				return
+			}
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelineList[0].ID); err != nil {
+				return
+			}
+
+			if pipeline.CreatedAt.After(until) {
+				page++
+				continue
+			} else {
+				startQuery = true
+				page--
+				continue
+			}
+		}
+
+		// start query
+		if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+			return
+		}
+
+		meetTime = true
+		for _, v := range pipelineList {
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
+				return
+			}
+
+			createTime := pipeline.CreatedAt
+			if !since.IsZero() || !until.IsZero() {
+				meetTime = createTime.After(since) && createTime.Before(until)
+			}
+
+			//the pipeline is we need
+			if meetTime {
+				totalNum = totalNum + 1
+
+				if req.Branch != "" && req.Branch != pipeline.Ref {
+					continue
+				} else if req.User != "" && req.User != pipeline.User.Name {
+					continue
+				} else if req.State != "" && req.State != pipeline.Status {
+					continue
+				}
+
+				statusNum = statusNum + 1
+
+				if pipeline.Status != "cancel" {
+					if pipeline.StartedAt == nil {
+						pendingTime = 0
+						runningTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
+					} else {
+						pendingTime = pipeline.StartedAt.Sub(*pipeline.CreatedAt).Seconds()
+						runningTime = pipeline.FinishedAt.Sub(*pipeline.StartedAt).Seconds()
+					}
+					totalTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
+
+					pipelineTime.PendingMax, pipelineTime.PendingMin = utils.CalSizeTime(pendingTime, pipelineTime.PendingMax, pipelineTime.PendingMin)
+					pipelineTime.RunningMax, pipelineTime.RunningMin = utils.CalSizeTime(runningTime, pipelineTime.RunningMax, pipelineTime.RunningMin)
+					pipelineTime.DurationMax, pipelineTime.DurationMin = utils.CalSizeTime(totalTime, pipelineTime.DurationMax, pipelineTime.DurationMin)
+
+					pipelineTime.PendingList = append(pipelineTime.PendingList, pendingTime)
+					pipelineTime.RunningList = append(pipelineTime.RunningList, runningTime)
+					pipelineTime.DurationList = append(pipelineTime.DurationList, totalTime)
+				}
+			}
+
+			// time is over, so return
+			if (!since.IsZero() || !until.IsZero()) && createTime.Before(since) {
+				return
+			}
+		}
+
+		page++
+	}
+	return
+}
+
+// QueryPipelinesFromDB ...
+func (s *Service) QueryPipelinesFromDB(c context.Context, req *model.PipelineDataReq, since, until time.Time) (totalNum, statusNum int, pipelineTime *model.PipelineTime, err error) {
+	var (
+		fmtLayout   = `%d-%d-%d 00:00:00`
+		pipelines   []*model.StatisticsPipeline
+		projectID   = req.ProjectID
+		pendingTime float64
+		runningTime float64
+		totalTime   float64
+	)
+	pipelineTime = &model.PipelineTime{}
+
+	sinceStr := fmt.Sprintf(fmtLayout, since.Year(), since.Month(), since.Day())
+	untilStr := fmt.Sprintf(fmtLayout, until.Year(), until.Month(), until.Day())
+	if totalNum, statusNum, pipelines, err = s.dao.QueryPipelinesByTime(projectID, req, sinceStr, untilStr); err != nil {
+		return
+	}
+
+	for _, pipeline := range pipelines {
+		if pipeline.Status == model.StatusCancel {
+			continue
+		}
+		if pipeline.FinishedAt == nil {
+			continue
+		}
+
+		if pipeline.StartedAt == nil {
+			pendingTime = 0
+			runningTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
+		} else {
+			pendingTime = pipeline.StartedAt.Sub(*pipeline.CreatedAt).Seconds()
+			runningTime = pipeline.FinishedAt.Sub(*pipeline.StartedAt).Seconds()
+		}
+		totalTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
+
+		pipelineTime.PendingMax, pipelineTime.PendingMin = utils.CalSizeTime(pendingTime, pipelineTime.PendingMax, pipelineTime.PendingMin)
+		pipelineTime.RunningMax, pipelineTime.RunningMin = utils.CalSizeTime(runningTime, pipelineTime.RunningMax, pipelineTime.RunningMin)
+		pipelineTime.DurationMax, pipelineTime.DurationMin = utils.CalSizeTime(totalTime, pipelineTime.DurationMax, pipelineTime.DurationMin)
+
+		pipelineTime.PendingList = append(pipelineTime.PendingList, pendingTime)
+		pipelineTime.RunningList = append(pipelineTime.RunningList, runningTime)
+		pipelineTime.DurationList = append(pipelineTime.DurationList, totalTime)
+	}
+	return
+}
+
+//alertProjectPipelineProc cron func
+func (s *Service) alertProjectPipelineProc() {
+	for _, alert := range conf.Conf.Property.Git.AlertPipeline {
+		projectId := alert.ProjectID
+		runningTimeout := alert.RunningTimeout
+		runningRate := alert.RunningRate
+		runningThreshold := alert.RunningThreshold
+		pendingTimeout := alert.PendingTimeout
+		pendingThreshold := alert.PendingThreshold
+		go func() {
+			var err error
+			if err = s.PipelineAlert(context.TODO(), projectId, runningTimeout, runningThreshold, runningRate, gitlab.Running); err != nil {
+				log.Error("PipelineAlert Running (%+v)", err)
+			}
+			if err = s.PipelineAlert(context.TODO(), projectId, pendingTimeout, pendingThreshold, 0, gitlab.Pending); err != nil {
+				log.Error("PipelineAlert Pending (%+v)", err)
+			}
+		}()
+	}
+}
+
+//PipelineAlert ...
+func (s *Service) PipelineAlert(c context.Context, projectID, timeout, threshold, rate int, status gitlab.BuildStateValue) (err error) {
+	var (
+		layout          = "2006-01-02 15:04:05"
+		pipeline        *gitlab.Pipeline
+		timeoutNum      int
+		message         string
+		pipelineurl     string
+		durationTime    float64
+		pipelineSUM     int
+		timeoutPipeline string
+		pipelineList    gitlab.PipelineList
+		resp            *gitlab.Response
+		projectInfo     *model.ProjectInfo
+		userlist        = conf.Conf.Property.Git.UserList
+		w               = wechat.New(s.dao)
+		sendMessage     = false
+	)
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+	repo := projectInfo.Repo
+	if len(repo) > len(_gitSSH) {
+		repo = repo[len(_gitSSH) : len(repo)-len(_gitSSHTail)]
+		repo = _gitHTTP + repo
+	}
+	timeNow := time.Now().Format(layout)
+	message = fmt.Sprintf("[SAGA]Pipeline 告警   %v\n项目：%s\n", timeNow, repo)
+	for page := 1; ; page++ {
+		if pipelineList, resp, err = s.git.ListProjectPipelines(page, projectID, status); err != nil {
+			return
+		}
+		for _, item := range pipelineList {
+			pipelineSUM += 1
+			if pipeline, _, err = s.git.GetPipeline(projectID, item.ID); err != nil {
+				return
+			}
+			if status == gitlab.Pending {
+				durationTime = pipeline.UpdatedAt.Sub(*pipeline.CreatedAt).Minutes()
+			} else if status == gitlab.Running {
+				//此处时间计算换成job
+				if durationTime, err = s.PipelineRunningTime(projectID, item.ID); err != nil {
+					return
+				}
+			}
+			if int(durationTime) >= timeout {
+				timeoutNum += 1
+				pipelineurl = fmt.Sprintf("%d. %s/pipelines/%d (%vmin)\n", timeoutNum, repo, pipeline.ID, int(durationTime))
+				timeoutPipeline += pipelineurl
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+
+	if timeoutPipeline != "" {
+		message += fmt.Sprintf(`列表(url|%s时间):%s%s`, status, "\n", timeoutPipeline)
+	}
+	if pipelineSUM > 0 {
+		message += fmt.Sprintf(`状态：%s 总数为%d个`, status, pipelineSUM)
+	}
+	if status == gitlab.Pending {
+		var alertMessage string
+		message += fmt.Sprintf(`%s告警：`, "\n")
+		if pipelineSUM >= threshold {
+			alertMessage = fmt.Sprintf(`[ 数量（%d）>=警戒值（%d） ]`, pipelineSUM, threshold)
+			sendMessage = true
+		}
+		message += alertMessage
+		if timeoutNum >= 1 {
+			if alertMessage != "" {
+				message = message[:strings.LastIndex(message, " ]")] + fmt.Sprintf(`，%s时间>=警戒值（%d） ]`, status, timeout)
+			} else {
+				message += fmt.Sprintf(`[ %s时间>=警戒值（%d） ]`, status, timeout)
+			}
+			sendMessage = true
+		}
+	}
+	if status == gitlab.Running && timeoutNum >= threshold {
+		sendMessage = true
+		message += fmt.Sprintf(`，时间>%dmin为%d个%s告警：[ 数量（%d）>=警戒值（%d) ]`, timeout, timeoutNum, "\n", timeoutNum, threshold)
+		if timeoutNum*100/pipelineSUM >= rate {
+			message = message[:strings.LastIndex(message, " ]")] + fmt.Sprintf(`，比例（%v%s）>=警戒值%d%s ]`, timeoutNum*100/pipelineSUM, "%", rate, "%")
+		}
+	}
+	if sendMessage {
+		return w.PushMsg(c, userlist, message)
+	}
+	return
+}
+
+//PipelineRunningTime ...
+func (s *Service) PipelineRunningTime(projectID, pipelineID int) (durationTime float64, err error) {
+	var jobList []*gitlab.Job
+	if jobList, _, err = s.git.ListPipelineJobs(nil, projectID, pipelineID); err != nil {
+		return
+	}
+	for _, job := range jobList {
+		if job.Status != _manualJob && job.Name != _androidScheduleJob && job.Name != _iosScheduleJob {
+			if job.FinishedAt != nil && job.StartedAt != nil {
+				durationTime += job.FinishedAt.Sub(*job.StartedAt).Minutes()
+			} else if job.StartedAt != nil {
+				durationTime += time.Since(*job.StartedAt).Minutes()
+			}
+		}
+	}
+	return
+}
+
+/*-------------------------------------- sync pipeline ----------------------------------------*/
+
+// SyncProjectPipelines ...
+func (s *Service) SyncProjectPipelines(projectID int) (result *model.SyncResult, err error) {
+	var (
+		//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
+		syncAllTime = false
+		since       *time.Time
+		until       *time.Time
+		projectInfo *model.ProjectInfo
+	)
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+	if !syncAllTime {
+
+		since, until = utils.CalSyncTime()
+		log.Info("sync project id(%d), name(%s) pipeline, time since: %v, until: %v", projectID, projectInfo.Name, since, until)
+		if result, err = s.SyncProjectPipelinesByTime(projectID, projectInfo.Name, *since, *until); err != nil {
+			return
+		}
+	} else {
+		log.Info("sync project id(%d), name(%s) pipeline", projectID, projectInfo.Name)
+		if result, err = s.SyncProjectAllPipelines(projectID, projectInfo.Name); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+// SyncProjectPipelinesByTime ...
+func (s *Service) SyncProjectPipelinesByTime(projectID int, projectName string, since, until time.Time) (result *model.SyncResult, err error) {
+	var (
+		pipelines  gitlab.PipelineList
+		pipeline   *gitlab.Pipeline
+		resp       *gitlab.Response
+		startQuery bool
+	)
+	result = &model.SyncResult{}
+
+	if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
+		return
+	}
+
+	page := 1
+	for page <= resp.TotalPages {
+		result.TotalPage++
+
+		if !startQuery {
+			if pipelines, resp, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+				return
+			}
+			if page == 1 && len(pipelines) <= 0 {
+				return
+			}
+
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelines[0].ID); err != nil {
+				return
+			}
+
+			if pipeline.CreatedAt.After(until) {
+				page++
+				continue
+			} else {
+				startQuery = true
+				page--
+				continue
+			}
+		}
+
+		// start query
+		if pipelines, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+			return
+		}
+
+		for _, v := range pipelines {
+
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
+				return
+			}
+
+			createTime := pipeline.CreatedAt
+			if createTime.After(since) && createTime.Before(until) {
+
+				if err = s.structureDBPipeline(projectID, projectName, pipeline); err != nil {
+					log.Error("pipeline Save Database err: projectID(%d), PipelineID(%d)", projectID, pipeline.ID)
+					err = nil
+
+					errData := &model.FailData{
+						ChildID: pipeline.ID,
+					}
+					result.FailData = append(result.FailData, errData)
+
+					continue
+				}
+				result.TotalNum++
+			}
+
+			if createTime.Before(since) {
+				return
+			}
+		}
+		page++
+	}
+
+	return
+}
+
+// SyncProjectAllPipelines ...
+func (s *Service) SyncProjectAllPipelines(projectID int, projectName string) (result *model.SyncResult, err error) {
+	var (
+		pipelines gitlab.PipelineList
+		pipeline  *gitlab.Pipeline
+		resp      *gitlab.Response
+	)
+	result = &model.SyncResult{}
+
+	for page := 1; ; page++ {
+		result.TotalPage++
+		if pipelines, resp, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
+			return
+		}
+
+		for _, v := range pipelines {
+			if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
+				return
+			}
+
+			if err = s.structureDBPipeline(projectID, projectName, pipeline); err != nil {
+				log.Error("pipeline Save Database err: projectID(%d), PipelineID(%d)", projectID, pipeline.ID)
+				err = nil
+
+				errData := &model.FailData{
+					ChildID: pipeline.ID,
+				}
+				result.FailData = append(result.FailData, errData)
+
+				continue
+			}
+			result.TotalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// structureDBPipeline ...
+func (s *Service) structureDBPipeline(projectID int, projectName string, pipeline *gitlab.Pipeline) (err error) {
+
+	statisticPipeline := &model.StatisticsPipeline{
+		PipelineID:   pipeline.ID,
+		ProjectID:    projectID,
+		ProjectName:  projectName,
+		Status:       pipeline.Status,
+		Ref:          pipeline.Ref,
+		Tag:          pipeline.Tag,
+		User:         pipeline.User.Name,
+		CreatedAt:    pipeline.CreatedAt,
+		UpdatedAt:    pipeline.UpdatedAt,
+		StartedAt:    pipeline.StartedAt,
+		FinishedAt:   pipeline.FinishedAt,
+		CommittedAt:  pipeline.CommittedAt,
+		Coverage:     pipeline.Coverage,
+		Duration:     pipeline.Duration,
+		DurationTime: 0,
+	}
+
+	return s.SaveDatabasePipeline(statisticPipeline)
+}
+
+// SaveDatabasePipeline ...
+func (s *Service) SaveDatabasePipeline(pipelineDB *model.StatisticsPipeline) (err error) {
+	var total int
+
+	if total, err = s.dao.HasPipeline(pipelineDB.ProjectID, pipelineDB.PipelineID); err != nil {
+		log.Error("SaveDatabasePipeline HasPipeline(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if err = s.dao.UpdatePipeline(pipelineDB.ProjectID, pipelineDB.PipelineID, pipelineDB); err != nil {
+			log.Error("SaveDatabasePipeline UpdatePipeline err(%+v)", err)
+			return
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabasePipeline pipeline has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreatePipeline(pipelineDB); err != nil {
+		log.Error("SaveDatabasePipeline CreatePipeline err(%+v)", err)
+		return
+	}
+
+	return
+}

app/admin/ep/saga/service/pipeline_test.go

@@ -0,0 +1,43 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/admin/ep/saga/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestService_QueryProjectRunners(t *testing.T) {
+	convey.Convey("get saga-admin runners", t, func() {
+		var (
+			ctx = context.Background()
+			req = &model.ProjectDataReq{ProjectID: 682}
+		)
+
+		resp, err := srv.QueryProjectRunners(ctx, req)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(len(resp), convey.ShouldBeGreaterThan, 1)
+	})
+	convey.Convey("get saga-admin runners", t, func() {
+		var (
+			ctx = context.Background()
+			req = &model.ProjectDataReq{ProjectID: 4928}
+		)
+
+		resp, err := srv.QueryProjectRunners(ctx, req)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(len(resp), convey.ShouldBeGreaterThan, 1)
+	})
+	convey.Convey("get saga-admin runners", t, func() {
+		var (
+			ctx = context.Background()
+			req = &model.ProjectDataReq{ProjectID: 5822}
+		)
+
+		resp, err := srv.QueryProjectRunners(ctx, req)
+		convey.So(err, convey.ShouldBeNil)
+		convey.So(len(resp), convey.ShouldBeGreaterThan, 1)
+	})
+}

app/admin/ep/saga/service/project.go

@@ -0,0 +1,159 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+)
+
+// FavoriteProjects list user's favorite projects
+func (s *Service) FavoriteProjects(c context.Context, req *model.Pagination, userName string) (resp *model.FavoriteProjectsResp, err error) {
+	var (
+		favorite    *model.ProjectFavorite
+		favorites   []*model.ProjectFavorite
+		projectInfo *model.ProjectInfo
+	)
+	resp = &model.FavoriteProjectsResp{}
+
+	if favorites, err = s.dao.FavoriteProjects(userName); err != nil {
+		return
+	}
+
+	for _, favorite = range favorites {
+		if projectInfo, err = s.dao.ProjectInfoByID(favorite.ProjID); err != nil {
+			return
+		}
+
+		myProject := &model.MyProjectInfo{
+			ProjectInfo: projectInfo,
+		}
+		myProject.Star = true
+		resp.Projects = append(resp.Projects, myProject)
+	}
+	resp.PageSize = req.PageSize
+	resp.PageNum = req.PageNum
+	resp.Total = len(favorites)
+	return
+}
+
+// EditFavorite edit user's favorites, star/unstar
+func (s *Service) EditFavorite(c context.Context, req *model.EditFavoriteReq, userName string) (resp *model.EmptyResp, err error) {
+	var (
+		projID    = req.ProjID
+		favorites []*model.ProjectFavorite
+		exist     bool
+	)
+	resp = &model.EmptyResp{}
+
+	if favorites, err = s.dao.FavoriteProjects(userName); err != nil {
+		return
+	}
+	if req.Star {
+		if !inFavorites(favorites, projID) {
+			if exist, err = s.dao.ProjectExist(projID); err != nil {
+				return
+			}
+			if exist {
+				if err = s.dao.AddFavorite(userName, projID); err != nil {
+					return
+				}
+			}
+		}
+	} else {
+		if inFavorites(favorites, projID) {
+			if err = s.dao.DelFavorite(userName, projID); err != nil {
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// inFavorites ...
+func inFavorites(favorites []*model.ProjectFavorite, projID int) bool {
+	var (
+		f *model.ProjectFavorite
+	)
+	for _, f = range favorites {
+		if projID == f.ProjID {
+			return true
+		}
+	}
+
+	return false
+}
+
+// QueryCommonProjects ...
+func (s *Service) QueryCommonProjects(c context.Context) (result []string, err error) {
+	var (
+		projectInfo *model.ProjectInfo
+	)
+	ids := conf.Conf.Property.DefaultProject.ProjectIDs
+	for _, id := range ids {
+		if projectInfo, err = s.dao.ProjectInfoByID(id); err != nil {
+			return
+		}
+		result = append(result, projectInfo.Name)
+	}
+	return
+}
+
+// QueryProjectInfo query project info.
+func (s *Service) QueryProjectInfo(c context.Context, req *model.ProjectInfoRequest) (resp *model.ProjectInfoResp, err error) {
+	var (
+		projectInfo     []*model.ProjectInfo
+		project         *model.ProjectInfo
+		total           int
+		saga            int
+		runner          int
+		sagaScale       int
+		runnerScale     int
+		mproject        *model.MyProjectInfo
+		favorites       []*model.ProjectFavorite
+		projectInfoResp []*model.MyProjectInfo
+	)
+	userName := req.Username
+	if total, projectInfo, err = s.dao.QueryProjectInfo(true, req); err != nil {
+		return
+	}
+	if favorites, err = s.dao.FavoriteProjects(userName); err != nil {
+		return
+	}
+	for _, project = range projectInfo {
+		mproject = &model.MyProjectInfo{
+			ProjectInfo: project,
+		}
+		if !inFavorites(favorites, project.ProjectID) {
+			mproject.Star = false
+		} else {
+			mproject.Star = true
+		}
+		projectInfoResp = append(projectInfoResp, mproject)
+	}
+
+	if saga, err = s.dao.QueryConfigInfo(req.Name, req.Department, req.Business, "saga"); err != nil {
+		return
+	}
+	if runner, err = s.dao.QueryConfigInfo(req.Name, req.Department, req.Business, "runner"); err != nil {
+		return
+	}
+
+	if total != 0 {
+		sagaScale = saga * 100 / total
+		runnerScale = runner * 100 / total
+	}
+
+	resp = &model.ProjectInfoResp{
+		PageNum:     req.PageNum,
+		PageSize:    req.PageSize,
+		Total:       total,
+		Saga:        saga,
+		Runner:      runner,
+		SagaScale:   sagaScale,
+		RunnerScale: runnerScale,
+		ProjectInfo: projectInfoResp,
+	}
+	return
+}

app/admin/ep/saga/service/runner.go

@@ -0,0 +1,112 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+//QueryProjectRunners query project runners info according to project id
+func (s *Service) QueryProjectRunners(c context.Context, req *model.ProjectDataReq) (resp []*gitlab.Runner, err error) {
+	var (
+		runners  []*gitlab.Runner
+		response *gitlab.Response
+	)
+
+	for page := 1; ; page++ {
+		if runners, response, err = s.gitlab.ListProjectRunners(req.ProjectID, page); err != nil {
+			return
+		}
+		resp = append(resp, runners...)
+		if response.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+/*-------------------------------------- sync runner ----------------------------------------*/
+
+// SyncAllRunners ...
+func (s *Service) SyncAllRunners(projectID int) (totalPage, totalNum int, err error) {
+	var (
+		runners     []*gitlab.Runner
+		resp        *gitlab.Response
+		projectInfo *model.ProjectInfo
+	)
+	if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
+		return
+	}
+
+	for page := 1; ; page++ {
+		totalPage++
+		if runners, resp, err = s.gitlab.ListProjectRunners(projectID, page); err != nil {
+			return
+		}
+		for _, runner := range runners {
+			var (
+				ipAddress string
+			)
+			//ipAddress = runner.IPAddress.String()
+			runnerDB := &model.StatisticsRunners{
+				ProjectID:   projectID,
+				ProjectName: projectInfo.Name,
+				RunnerID:    runner.ID,
+				Description: runner.Description,
+				Active:      runner.Active,
+				IsShared:    runner.IsShared,
+				IPAddress:   ipAddress,
+				Name:        runner.Name,
+				Online:      runner.Online,
+				Status:      runner.Status,
+				Token:       runner.Token,
+			}
+
+			if err = s.SaveDatabaseRunner(runnerDB); err != nil {
+				log.Error("runner Save Database err: projectID(%d), RunnerID(%d)", projectID, runner.ID)
+				err = nil
+				continue
+			}
+			totalNum++
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+	}
+	return
+}
+
+// SaveDatabaseRunner ...
+func (s *Service) SaveDatabaseRunner(runnerDB *model.StatisticsRunners) (err error) {
+	var total int
+
+	if total, err = s.dao.HasRunner(runnerDB.ProjectID, runnerDB.RunnerID); err != nil {
+		log.Error("SaveDatabaseRunner HasRunner(%+v)", err)
+		return
+	}
+
+	// found only one, so update
+	if total == 1 {
+		if err = s.dao.UpdateRunner(runnerDB.ProjectID, runnerDB.RunnerID, runnerDB); err != nil {
+			log.Error("SaveDatabaseRunner UpdateRunner(%+v)", err)
+			return
+		}
+		return
+	} else if total > 1 {
+		// found repeated row, this situation will not exist under normal
+		log.Warn("SaveDatabaseRunner commit has more rows(%d)", total)
+		return
+	}
+
+	// insert row now
+	if err = s.dao.CreateRunner(runnerDB); err != nil {
+		log.Error("SaveDatabaseRunner CreateRunner(%+v)", err)
+		return
+	}
+
+	return
+}

app/admin/ep/saga/service/service.go

@@ -0,0 +1,72 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/dao"
+	"go-common/app/admin/ep/saga/service/gitlab"
+	"go-common/app/admin/ep/saga/service/wechat"
+
+	"github.com/robfig/cron"
+)
+
+// Service struct
+type Service struct {
+	dao    *dao.Dao
+	gitlab *gitlab.Gitlab
+	git    *gitlab.Gitlab
+	cron   *cron.Cron
+	wechat *wechat.Wechat
+}
+
+// New a DirService and return.
+func New() (s *Service) {
+	var (
+		err error
+	)
+	s = &Service{
+		dao:  dao.New(),
+		cron: cron.New(),
+	}
+	if err = s.cron.AddFunc(conf.Conf.Property.SyncProject.CheckCron, s.collectprojectproc); err != nil {
+		panic(err)
+	}
+	if err = s.cron.AddFunc(conf.Conf.Property.Git.CheckCron, s.alertProjectPipelineProc); err != nil {
+		panic(err)
+	}
+
+	if err = s.cron.AddFunc(conf.Conf.Property.SyncData.CheckCron, s.syncdataproc); err != nil {
+		panic(err)
+	}
+	if err = s.cron.AddFunc(conf.Conf.Property.SyncData.CheckCronAll, s.syncalldataproc); err != nil {
+		panic(err)
+	}
+	if err = s.cron.AddFunc(conf.Conf.Property.SyncData.CheckCronWeek, s.syncweekdataproc); err != nil {
+		panic(err)
+	}
+	s.cron.Start()
+
+	// init gitlab client
+	s.gitlab = gitlab.New(conf.Conf.Property.Gitlab.API, conf.Conf.Property.Gitlab.Token)
+	// init online gitlab client
+	s.git = gitlab.New(conf.Conf.Property.Git.API, conf.Conf.Property.Git.Token)
+	// init wechat client
+	s.wechat = wechat.New(s.dao)
+
+	return
+}
+
+// Ping check dao health.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Wait wait all closed.
+func (s *Service) Wait() {
+}
+
+// Close close all dao.
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/admin/ep/saga/service/service_test.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+	"github.com/xanzy/go-gitlab"
+)
+
+var (
+	srv *Service
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	flag.Set("conf", "../cmd/saga-admin-test.toml")
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+
+	srv = New()
+	os.Exit(m.Run())
+}
+
+// TestInsertDB ...
+func TestInsertDB(t *testing.T) {
+
+	Convey("insertDB", t, func() {
+		p := &gitlab.Project{
+			ID:            11,
+			Name:          "test",
+			Description:   "[主站 android java] test",
+			WebURL:        "http://gitlab.bilibili.co/platform/go-common",
+			SSHURLToRepo:  "git@gitlab.bilibili.co:platform/go-common.git",
+			DefaultBranch: "master",
+			Namespace: &gitlab.ProjectNamespace{
+				Name: "mytest",
+				Kind: "group",
+			},
+		}
+
+		err := srv.insertDB(p)
+		So(err, ShouldBeNil)
+
+		projectInfo, err := srv.dao.ProjectInfoByID(p.ID)
+		So(err, ShouldBeNil)
+		So(projectInfo.ProjectID, ShouldEqual, 11)
+		So(projectInfo.Name, ShouldEqual, "test")
+		So(projectInfo.WebURL, ShouldEqual, "http://gitlab.bilibili.co/platform/go-common")
+		So(projectInfo.Repo, ShouldEqual, "git@gitlab.bilibili.co:platform/go-common.git")
+		So(projectInfo.DefaultBranch, ShouldEqual, "master")
+		So(projectInfo.Department, ShouldEqual, "主站")
+		So(projectInfo.Business, ShouldEqual, "android")
+		So(projectInfo.Language, ShouldEqual, "java")
+		So(projectInfo.SpaceName, ShouldEqual, "mytest")
+		So(projectInfo.SpaceKind, ShouldEqual, "group")
+	})
+
+}
+
+// TestResolveDes ...
+func TestResolveDes(t *testing.T) {
+
+	Convey("resolveDes", t, func() {
+
+		s := "[主站 android java] test"
+		department, business, language, parseFail := parseDes(s)
+		So(department, ShouldEqual, "主站")
+		So(business, ShouldEqual, "android")
+		So(language, ShouldEqual, "java")
+		So(parseFail, ShouldEqual, false)
+	})
+
+}
+
+// TestCollectProject ...
+func TestCollectProject(t *testing.T) {
+
+	Convey("CollectProject", t, func() {
+
+		err := srv.CollectProject(context.Background())
+		So(err, ShouldBeNil)
+	})
+
+}
+
+// TestPushMsg ...
+func TestPushMsg(t *testing.T) {
+
+	Convey("TestPushMsg", t, func() {
+
+		var err error
+
+		result := &model.SyncResult{}
+
+		for i := 0; i < 10; i++ {
+			errData := &model.FailData{
+				ChildID: i,
+			}
+			result.FailData = append(result.FailData, errData)
+		}
+
+		err = srv.WechatFailData(model.DataTypeJob, 888, result, nil)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestServiceSaveAggregateBranchDatabase(t *testing.T) {
+	Convey("test service aggregate branch include special char", t, func(ctx C) {
+		var (
+			branch = &model.AggregateBranches{
+				ID:               4,
+				ProjectID:        666,
+				ProjectName:      "六六大顺",
+				BranchName:       "666",
+				BranchUserName:   "吴维",
+				BranchMaster:     "wuwei",
+				Behind:           1111,
+				Ahead:            2222,
+				LatestSyncTime:   nil,
+				LatestUpdateTime: nil,
+				IsDeleted:        true,
+			}
+			//total int
+			err error
+		)
+		Convey("SaveAggregateBranchDatabase", func(ctx C) {
+			err = srv.SaveAggregateBranchDatabase(context.TODO(), branch)
+			Convey("Then err should be nil.", func(ctx C) {
+				So(err, ShouldBeNil)
+			})
+		})
+	})
+}

app/admin/ep/saga/service/statistics.go

@@ -0,0 +1,308 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/cache/memcache"
+	"go-common/library/log"
+
+	"github.com/xanzy/go-gitlab"
+)
+
+// QueryProject query commit info according to project id.
+func (s *Service) QueryProject(c context.Context, object string, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) {
+	var (
+		data     []*model.DataWithTime
+		queryDes string
+		total    int
+	)
+
+	log.Info("QuerySingleProjectData Type: %d", req.QueryType)
+	switch req.QueryType {
+	case model.LastYearPerMonth:
+		queryDes = model.LastYearPerMonthNote
+	case model.LastMonthPerDay:
+		queryDes = model.LastMonthPerDayNote
+	case model.LastYearPerDay:
+		queryDes = model.LastYearPerDayNote
+	default:
+		log.Warn("QueryProjectCommit Type is not in range")
+		return
+	}
+	queryDes = req.ProjectName + " " + object + queryDes
+
+	if data, total, err = s.QueryProjectByTime(req.ProjectID, object, req.QueryType); err != nil {
+		return
+	}
+
+	resp = &model.ProjectDataResp{
+		ProjectName: req.ProjectName,
+		QueryDes:    queryDes,
+		Total:       total,
+		Data:        data,
+	}
+	return
+}
+
+// QueryProjectByTime ...
+func (s *Service) QueryProjectByTime(projectID int, object string, queryType int) (resp []*model.DataWithTime, allNum int, err error) {
+	var (
+		layout    = "2006-01-02"
+		fmtLayout = `%d-%d-%d 00:00:00`
+		//response *gitlab.Response
+		since      time.Time
+		until      time.Time
+		count      int
+		totalItems int
+	)
+
+	year, month, _ := time.Now().Date()
+	thisMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.Local)
+
+	if queryType == model.LastYearPerMonth {
+		count = model.MonthNumPerYear
+	} else if queryType == model.LastMonthPerDay {
+		_, _, count = thisMonth.AddDate(0, 0, -1).Date()
+	} else if queryType == model.LastYearPerDay {
+		count = model.DayNumPerYear
+	}
+
+	for i := 1; i <= count; i++ {
+		if queryType == model.LastYearPerMonth {
+			since = thisMonth.AddDate(0, -i, 0)
+			until = thisMonth.AddDate(0, -i+1, 0)
+		} else if queryType == model.LastMonthPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		} else if queryType == model.LastYearPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		}
+
+		sinceStr := fmt.Sprintf(fmtLayout, since.Year(), since.Month(), since.Day())
+		untilStr := fmt.Sprintf(fmtLayout, until.Year(), until.Month(), until.Day())
+		if object == model.ObjectCommit {
+			/*if _, response, err = s.gitlab.ListProjectCommit(projectID, 1, &since, &until); err != nil {
+				return
+			}*/
+			if totalItems, err = s.dao.CountCommitByTime(projectID, sinceStr, untilStr); err != nil {
+				return
+			}
+		} else if object == model.ObjectMR {
+			/*if _, response, err = s.gitlab.ListProjectMergeRequests(projectID, &since, &until, -1); err != nil {
+				return
+			}*/
+			if totalItems, err = s.dao.CountMRByTime(projectID, sinceStr, untilStr); err != nil {
+				return
+			}
+		} else {
+			log.Warn("QueryProjectByTime object(%s) is not support!", object)
+			return
+		}
+
+		perData := &model.DataWithTime{
+			//TotalItem: response.TotalItems,
+			TotalItem: totalItems,
+			StartTime: since.Format(layout),
+			EndTime:   until.Format(layout),
+		}
+		resp = append(resp, perData)
+		//allNum = allNum + response.TotalItems
+		allNum = allNum + totalItems
+	}
+	return
+}
+
+// QueryTeam ...
+func (s *Service) QueryTeam(c context.Context, object string, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) {
+	var (
+		projectInfo []*model.ProjectInfo
+		reqProject  = &model.ProjectInfoRequest{}
+
+		dataMap     = make(map[string]*model.TeamDataResp)
+		data        []*model.DataWithTime
+		queryDes    string
+		total       int
+		key         string
+		keyNotExist bool
+	)
+
+	if len(req.Department) <= 0 && len(req.Business) <= 0 {
+		log.Warn("query department and business are empty!")
+		return
+	}
+
+	//log.Info("QueryTeamCommit Query Type: %d", req.QueryType)
+	switch req.QueryType {
+	case model.LastYearPerMonth:
+		queryDes = model.LastYearPerMonthNote
+	case model.LastMonthPerDay:
+		queryDes = model.LastMonthPerDayNote
+	case model.LastYearPerDay:
+		queryDes = model.LastYearPerDayNote
+	default:
+		log.Warn("QueryTeamCommit Type is not in range")
+		return
+	}
+	queryDes = req.Department + " " + req.Business + " " + object + queryDes
+
+	//get value from mc
+	key = "saga_admin_" + req.Department + "_" + req.Business + "_" + model.KeyTypeConst[req.QueryType]
+	if err = s.dao.GetData(c, key, &dataMap); err != nil {
+		if err == memcache.ErrNotFound {
+			log.Warn("no such key (%s) in cache, err (%s)", key, err.Error())
+			keyNotExist = true
+		} else {
+			return
+		}
+	}
+	if _, ok := dataMap[object]; !ok {
+		keyNotExist = true
+	} else {
+		resp = dataMap[object]
+		return
+	}
+
+	log.Info("sync team %s start => type= %d, Department= %s, Business= %s", object, req.QueryType, req.Department, req.Business)
+
+	reqProject.Department = req.Department
+	reqProject.Business = req.Business
+	if _, projectInfo, err = s.dao.QueryProjectInfo(false, reqProject); err != nil {
+		return
+	}
+
+	if len(projectInfo) <= 0 {
+		log.Warn("Found no project!")
+		return
+	}
+
+	if data, total, err = s.QueryTeamByTime(object, req, req.QueryType, projectInfo); err != nil {
+		return
+	}
+
+	resp = &model.TeamDataResp{
+		Department: req.Department,
+		Business:   req.Business,
+		QueryDes:   queryDes,
+		Total:      total,
+		Data:       data,
+	}
+
+	//set value to mc
+	if keyNotExist {
+		dataMap[object] = resp
+		if err = s.dao.SetData(c, key, dataMap); err != nil {
+			return
+		}
+	}
+
+	log.Info("sync team %s end", object)
+	return
+}
+
+// QueryTeamByTime query commit info per month of last year and per day of last month according to team.
+func (s *Service) QueryTeamByTime(object string, req *model.TeamDataRequest, queryType int, projectInfo []*model.ProjectInfo) (resp []*model.DataWithTime, allNum int, err error) {
+	var (
+		layout   = "2006-01-02"
+		response *gitlab.Response
+		since    time.Time
+		until    time.Time
+		count    int
+		num      int
+	)
+
+	year, month, _ := time.Now().Date()
+	thisMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.Local)
+
+	if queryType == model.LastYearPerMonth {
+		count = model.MonthNumPerYear
+	} else if queryType == model.LastMonthPerDay {
+		_, _, count = thisMonth.AddDate(0, 0, -1).Date()
+	} else if queryType == model.LastYearPerDay {
+		count = model.DayNumPerYear
+	}
+
+	for i := 1; i <= count; i++ {
+		if queryType == model.LastYearPerMonth {
+			since = thisMonth.AddDate(0, -i, 0)
+			until = thisMonth.AddDate(0, -i+1, 0)
+		} else if queryType == model.LastMonthPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		} else if queryType == model.LastYearPerDay {
+			since = thisMonth.AddDate(0, 0, -i)
+			until = thisMonth.AddDate(0, 0, -i+1)
+		}
+
+		num = 0
+		for _, project := range projectInfo {
+			if object == model.ObjectCommit {
+				if _, response, err = s.gitlab.ListProjectCommit(project.ProjectID, 1, &since, &until); err != nil {
+					return
+				}
+			} else if object == model.ObjectMR {
+				if _, response, err = s.gitlab.ListProjectMergeRequests(project.ProjectID, &since, &until, -1); err != nil {
+					return
+				}
+			} else {
+				log.Warn("QueryTeamByTime object(%s) is not support!", object)
+				return
+			}
+			num = num + response.TotalItems
+		}
+
+		perData := &model.DataWithTime{
+			TotalItem: num,
+			StartTime: since.Format(layout),
+			EndTime:   until.Format(layout),
+		}
+		resp = append(resp, perData)
+		allNum = allNum + num
+	}
+	return
+}
+
+// SyncData ...
+func (s *Service) SyncData(c context.Context) (err error) {
+	log.Info("sync all data info start!")
+	for _, de := range conf.Conf.Property.DeInfo {
+		for _, bu := range conf.Conf.Property.BuInfo {
+			for k, keyType := range model.KeyTypeConst {
+
+				key := "saga_admin_" + de.Value + "_" + bu.Value + "_" + keyType
+				if err = s.dao.DeleteData(c, key); err != nil {
+					return
+				}
+
+				req := &model.TeamDataRequest{
+					TeamParam: model.TeamParam{
+						Department: de.Value,
+						Business:   bu.Value,
+					},
+					QueryType: k,
+				}
+
+				if k == 3 {
+					if _, err = s.QueryTeamPipeline(c, req); err != nil {
+						return
+					}
+					continue
+				}
+
+				if _, err = s.QueryTeamCommit(c, req); err != nil {
+					return
+				}
+
+				if _, err = s.QueryTeamMr(c, req); err != nil {
+					return
+				}
+			}
+		}
+	}
+	log.Info("sync all data info end!")
+	return
+}

app/admin/ep/saga/service/sync.go

@@ -0,0 +1,405 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+)
+
+const _baseBranch = "master"
+
+// syncalldataproc ...
+func (s *Service) syncweekdataproc() {
+	//the blow object will be sync or update all time data
+	go s.SyncMember()
+	go s.SyncRunners()
+	go s.SyncContacts(context.TODO())
+}
+
+// syncalldataproc ...
+func (s *Service) syncalldataproc() {
+	if err := s.SyncBranch(); err != nil {
+		return
+	}
+	// AggregateBranch base on branch, if SyncBranch err, so return
+	s.AggregateBranch()
+}
+
+// syncdataproc ...
+func (s *Service) syncdataproc() {
+	var (
+		wg   sync.WaitGroup
+		done = func(f func() error) {
+			defer wg.Done()
+			f()
+		}
+	)
+
+	wg.Add(2)
+	go done(s.SyncIssues)
+	go done(s.SyncPipelines)
+	wg.Wait()
+
+	wg.Add(1)
+	go done(s.SyncCommit)
+	wg.Wait()
+
+	wg.Add(2)
+	go done(s.SyncJobs)
+	go done(s.SyncMR)
+	wg.Wait()
+
+	wg.Add(3)
+	go done(s.SyncMRNote)
+	go done(s.SyncMRAwardEmoji)
+	go done(s.SyncMRDiscussion)
+	wg.Wait()
+
+	s.AggregateMR()
+}
+
+// SyncCommit ...
+func (s *Service) SyncCommit() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		allPage    int
+		allNum     int
+		result     *model.SyncResult
+	)
+	log.Info("===================== SyncCommit start ========================")
+	for _, projectID := range projectIDs {
+		if result, err = s.SyncProjectCommit(projectID); err != nil {
+			log.Error("SyncCommit projectID(%d), err(%+v)", projectID, err)
+			go s.WechatFailData(model.DataTypeCommit, projectID, result, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncCommit projectID(%d) complete Page(%d), Num(%d)", projectID, result.TotalPage, result.TotalNum)
+
+		if result != nil && len(result.FailData) > 0 {
+			go s.WechatFailData(model.DataTypeCommit, projectID, result, nil)
+		}
+
+		allPage = allPage + result.TotalPage
+		allNum = allNum + result.TotalNum
+	}
+	log.Info("===================== SyncCommit finished totalPage(%d), totalNum(%d)========================", allPage, allNum)
+	return
+}
+
+// SyncMR ...
+func (s *Service) SyncMR() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		allPage    int
+		allNum     int
+		result     *model.SyncResult
+	)
+	log.Info("===================== SyncMR start ========================")
+	for _, projectID := range projectIDs {
+		if result, err = s.SyncProjectMR(context.TODO(), projectID); err != nil {
+			log.Error("SyncMR projectID(%d), err(%+v)", projectID, err)
+			go s.WechatFailData(model.DataTypeMR, projectID, result, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncMR projectID(%d) complete Page(%d), Num(%d)", projectID, result.TotalPage, result.TotalNum)
+		if result != nil && len(result.FailData) > 0 {
+			go s.WechatFailData(model.DataTypeMR, projectID, result, nil)
+		}
+		allPage = allPage + result.TotalPage
+		allNum = allNum + result.TotalNum
+	}
+	log.Info("===================== SyncMR finished totalPage(%d), totalNum(%d)========================", allPage, allNum)
+	return
+}
+
+// AggregateMR ...
+func (s *Service) AggregateMR() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+	)
+	log.Info("===================== AggMR start ========================")
+	for _, projectID := range projectIDs {
+		if err = s.AggregateProjectMR(context.TODO(), projectID); err != nil {
+			log.Error("AggMR projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> AggMR projectID(%d) complete", projectID)
+	}
+
+	log.Info("===================== AggMR finished ========================")
+	return
+}
+
+// SyncJobs ...
+func (s *Service) SyncJobs() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		allPage    int
+		allNum     int
+		result     *model.SyncResult
+	)
+	log.Info("===================== SyncJobs start ========================")
+	for _, projectID := range projectIDs {
+
+		if result, err = s.SyncProjectJobs(projectID); err != nil {
+			log.Error("SyncJobs projectID(%d), err(%+v)", projectID, err)
+			go s.WechatFailData(model.DataTypeJob, projectID, result, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncJobs projectID(%d) complete Page(%d), Num(%d)", projectID, result.TotalPage, result.TotalNum)
+
+		if result != nil && len(result.FailData) > 0 {
+			go s.WechatFailData(model.DataTypeJob, projectID, result, nil)
+		}
+
+		allPage = allPage + result.TotalPage
+		allNum = allNum + result.TotalNum
+	}
+	log.Info("===================== SyncJobs finished totalPage(%d), totalNum(%d)========================", allPage, allNum)
+	return
+}
+
+// SyncPipelines ...
+func (s *Service) SyncPipelines() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		allPage    int
+		allNum     int
+		result     *model.SyncResult
+	)
+	log.Info("===================== SyncPipelines start ========================")
+	for _, projectID := range projectIDs {
+
+		if result, err = s.SyncProjectPipelines(projectID); err != nil {
+			log.Error("SyncPipelines projectID(%d), err(%+v)", projectID, err)
+			go s.WechatFailData(model.DataTypePipeline, projectID, result, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncPipelines projectID(%d) complete Page(%d), Num(%d)", projectID, result.TotalPage, result.TotalNum)
+
+		if result != nil && len(result.FailData) > 0 {
+			go s.WechatFailData(model.DataTypePipeline, projectID, result, nil)
+		}
+
+		allPage = allPage + result.TotalPage
+		allNum = allNum + result.TotalNum
+	}
+	log.Info("===================== SyncPipelines finished totalPage(%d), totalNum(%d)========================", allPage, allNum)
+	return
+}
+
+// SyncMRNote ...
+func (s *Service) SyncMRNote() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncNote start ========================")
+	for _, projectID := range projectIDs {
+		if page, num, err = s.SyncProjectNotes(context.TODO(), projectID); err != nil {
+			log.Error("SyncNotes projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncNote projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncNote finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncMember ...
+func (s *Service) SyncMember() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncMember start ========================")
+	for _, projectID := range projectIDs {
+		if page, num, err = s.SyncProjectMember(context.TODO(), projectID); err != nil {
+			log.Error("SyncMember projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncMember projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncMember finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncMRAwardEmoji ...
+func (s *Service) SyncMRAwardEmoji() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncMRAwardEmoji start ========================")
+	for _, projectID := range projectIDs {
+		if page, num, err = s.SyncProjectAwardEmoji(context.TODO(), projectID); err != nil {
+			log.Error("SyncMRAwardEmoji projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncMRAwardEmoji projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncMRAwardEmoji finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncMRDiscussion ...
+func (s *Service) SyncMRDiscussion() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncMRDiscussion start ========================")
+	for _, projectID := range projectIDs {
+		if page, num, err = s.SyncProjectDiscussion(context.TODO(), projectID); err != nil {
+			log.Error("SyncMRDiscussion projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncMRDiscussion projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncMRDiscussion finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncRunners ...
+func (s *Service) SyncRunners() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncRunners start ========================")
+	for _, projectID := range projectIDs {
+		if page, num, err = s.SyncAllRunners(projectID); err != nil {
+			log.Error("SyncRunners projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncRunners projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncRunners finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncIssues ...
+func (s *Service) SyncIssues() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		totalPage  int
+		totalNum   int
+		page       int
+		num        int
+	)
+	log.Info("===================== SyncIssues start ========================")
+	for _, projectID := range projectIDs {
+
+		if page, num, err = s.SyncAllIssues(projectID); err != nil {
+			log.Error("SyncIssues projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncIssues projectID(%d) complete Page(%d), Num(%d)", projectID, page, num)
+		totalPage = totalPage + page
+		totalNum = totalNum + num
+	}
+	log.Info("===================== SyncIssues finished totalPage(%d), totalNum(%d)========================", totalPage, totalNum)
+	return
+}
+
+// SyncBranch ...
+func (s *Service) SyncBranch() (err error) {
+	var (
+		projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+		allPage    int
+		allNum     int
+		result     *model.SyncResult
+	)
+	log.Info("===================== SyncBranch start ========================")
+	for _, projectID := range projectIDs {
+		if result, err = s.SyncProjectBranch(context.TODO(), projectID); err != nil {
+			log.Error("SyncBranch projectID(%d), err(%+v)", projectID, err)
+			go s.WechatFailData(model.DataTypeBranch, projectID, result, err)
+			return
+		}
+		log.Info(">>>>>>>>> SyncBranch projectID(%d) complete Page(%d), Num(%d)", projectID, result.TotalPage, result.TotalNum)
+		if result != nil && len(result.FailData) > 0 {
+			go s.WechatFailData(model.DataTypeBranch, projectID, result, nil)
+		}
+		allPage = allPage + result.TotalPage
+		allNum = allNum + result.TotalNum
+	}
+	log.Info("===================== SyncBranch finished totalPage(%d), totalNum(%d)========================", allPage, allNum)
+	return
+}
+
+// AggregateBranch ...
+func (s *Service) AggregateBranch() (err error) {
+	var projectIDs = conf.Conf.Property.DefaultProject.ProjectIDs
+
+	log.Info("===================== SyncAggregateBranch start ========================")
+	for _, projectID := range projectIDs {
+		if err = s.AggregateProjectBranch(context.TODO(), projectID, _baseBranch); err != nil {
+			log.Error("SyncAggregateBranch projectID(%d), err(%+v)", projectID, err)
+			return
+		}
+	}
+	log.Info("===================== SyncAggregateBranch finished ========================")
+	return
+}
+
+// WechatFailData ...
+func (s *Service) WechatFailData(dataType string, projectID int, result *model.SyncResult, error error) (err error) {
+	var (
+		ctx     = context.Background()
+		users   = conf.Conf.Property.SyncData.WechatUser
+		content string
+	)
+
+	title := "[SAGA-ADMIN] SYNC ERROR !!!"
+	subject := fmt.Sprintf("%s\n\nDataType ( %s )\nProjectID ( %d )", title, dataType, projectID)
+
+	if error != nil {
+		return s.wechat.PushMsg(ctx, users, fmt.Sprintf("%s\n\n%s", subject, error.Error()))
+	}
+
+	if result == nil || len(result.FailData) <= 0 {
+		return
+	}
+
+	content = "LIST : \n"
+	for _, data := range result.FailData {
+
+		if dataType == model.DataTypeJob || dataType == model.DataTypePipeline || dataType == model.DataTypeMR {
+			content += fmt.Sprintf("%d,  ", data.ChildID)
+		} else if dataType == model.DataTypeCommit || dataType == model.DataTypeBranch {
+			content += fmt.Sprintf("%s\n", data.ChildIDStr)
+		}
+	}
+
+	return s.wechat.PushMsg(ctx, users, fmt.Sprintf("%s\n\n%s", subject, content))
+}

app/admin/ep/saga/service/task.go

@@ -0,0 +1,28 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/admin/ep/saga/model"
+)
+
+// MergeTasks query all tasks for the project.
+func (s *Service) MergeTasks(c context.Context, req *model.TasksReq) (resp *model.TasksResp, err error) {
+	var (
+		tasks  []*model.Task
+		status int
+	)
+	resp = new(model.TasksResp)
+	if _, err = s.dao.ProjectInfoByID(req.ProjID); err != nil {
+		return
+	}
+
+	for _, status = range req.Statuses {
+		if tasks, err = s.dao.Tasks(req.ProjID, status); err != nil {
+			return
+		}
+		resp.Tasks = append(resp.Tasks, tasks...)
+	}
+
+	return
+}

app/admin/ep/saga/service/user.go

@@ -0,0 +1,13 @@
+package service
+
+import "go-common/app/admin/ep/saga/model"
+
+// UserInfo get username and email.
+func (s *Service) UserInfo(userName string) (userInfo *model.User) {
+
+	userInfo = &model.User{
+		Name:  userName,
+		EMail: userName + "@bilibili.com",
+	}
+	return
+}

app/admin/ep/saga/service/utils/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["utils.go"],
+    importpath = "go-common/app/admin/ep/saga/service/utils",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/admin/ep/saga/conf:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/ep/saga/service/utils/utils.go

@@ -0,0 +1,119 @@
+package utils
+
+import (
+	"bytes"
+	"sort"
+	"strconv"
+	"time"
+
+	"go-common/app/admin/ep/saga/conf"
+)
+
+// CalAverageTime 计算时间的平均/*分位 时间
+func CalAverageTime(timeType int, timeArray []float64) (result float64) {
+	sort.Float64s(timeArray)
+	if timeType == 0 {
+		// 判断数量为零
+		if len(timeArray) == 0 {
+			result = 0
+		} else {
+			var sum float64
+			for _, t := range timeArray {
+				sum += t
+			}
+			result = sum / float64(len(timeArray))
+		}
+	} else if timeType > 0 && timeType < 11 {
+		if len(timeArray) == 0 {
+			result = 0
+		} else {
+			index := len(timeArray) * timeType / 10
+			result = timeArray[index]
+		}
+	}
+	return
+}
+
+// CalSizeTime ...
+func CalSizeTime(time, max, min float64) (float64, float64) {
+	if max == 0 {
+		max = time
+		return max, min
+	}
+	if min == 0 {
+		min = time
+		return max, min
+	}
+
+	if time > max {
+		max = time
+		return max, min
+	}
+
+	if time != 0 && time < min {
+		min = time
+		return max, min
+	}
+
+	return max, min
+}
+
+// CalSyncTime ...
+func CalSyncTime() (since, until *time.Time) {
+	syncDays := conf.Conf.Property.SyncData.DefaultSyncDays
+
+	year, month, day := time.Now().Date()
+
+	untilTime := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
+	sinceTime := time.Date(year, month, day-syncDays, 0, 0, 0, 0, time.Local)
+
+	since = &sinceTime
+	until = &untilTime
+
+	return
+}
+
+// CombineSlice ...
+func CombineSlice(s1, s2 []float64) []float64 {
+	slice := make([]float64, len(s1)+len(s2))
+	copy(slice, s1)
+	copy(slice[len(s1):], s2)
+	return slice
+}
+
+// InSlice ...
+func InSlice(key interface{}, list []string) bool {
+	for _, item := range list {
+		if key == item {
+			return true
+		}
+	}
+	return false
+}
+
+// Unicode2Chinese ...
+func Unicode2Chinese(str string) string {
+	buf := bytes.NewBuffer(nil)
+	i, j := 0, len(str)
+	for i < j {
+		x := i + 6
+		if x > j {
+			buf.WriteString(str[i:])
+			break
+		}
+		if str[i] == '\\' && str[i+1] == 'u' {
+			hex := str[i+2 : x]
+			r, err := strconv.ParseUint(hex, 16, 64)
+			if err == nil {
+				buf.WriteRune(rune(r))
+			} else {
+				buf.WriteString(str[i:x])
+			}
+			i = x
+		} else {
+			buf.WriteByte(str[i])
+			i++
+		}
+	}
+	return buf.String()
+}

app/admin/ep/saga/service/wechat.go

@@ -0,0 +1,327 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/app/admin/ep/saga/service/wechat"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+const qyWechatURL = "https://qyapi.weixin.qq.com"
+
+// CollectWachatUsers send required wechat visible users stored in memcache by email
+func (s *Service) CollectWachatUsers(c context.Context) (err error) {
+	var (
+		contactInfo *model.ContactInfo
+		userMap     = make(map[string]model.RequireVisibleUser)
+		user        string
+	)
+
+	if err = s.dao.RequireVisibleUsersRedis(c, &userMap); err != nil {
+		log.Error("get require visible user error(%v)", err)
+		return
+	}
+
+	for k, v := range userMap {
+		if contactInfo, err = s.dao.QueryUserByID(k); err == nil {
+			if contactInfo.VisibleSaga {
+				continue
+			}
+		}
+		user += v.UserName + " ,  " + v.NickName + "\n"
+	}
+
+	/*content := fmt.Sprintf("\n\n邮箱前缀     昵称\n\n%s", user)
+	for _, addr := range conf.Conf.Property.ReportRequiredVisible.AlertAddrs {
+		if err = mail.SendMail2(addr, "需添加的企业微信名单", content); err != nil {
+			return
+		}
+	}*/
+	if err = s.dao.DeleteRequireVisibleUsersRedis(c); err != nil {
+		log.Error("Delete require visible user error(%v)", err)
+		return
+	}
+
+	return
+}
+
+// SyncContacts sync the wechat contacts 更新企业微信列表（用户信息和saga信息）
+func (s *Service) SyncContacts(c context.Context) (err error) {
+	var (
+		w = wechat.New(s.dao)
+	)
+
+	if err = w.SyncContacts(c); err != nil {
+		return
+	}
+	return
+}
+
+// QueryContacts query machine logs.
+func (s *Service) QueryContacts(c context.Context, queryRequest *model.Pagination) (p *model.PaginateContact, err error) {
+	var (
+		total    int64
+		contacts []*model.ContactInfo
+	)
+	fmt.Print(queryRequest.PageNum)
+	if total, contacts, err = s.dao.FindContacts(queryRequest.PageNum, queryRequest.PageSize); err != nil {
+		return
+	}
+	fmt.Print(queryRequest.PageNum)
+	p = &model.PaginateContact{
+		PageNum:  queryRequest.PageNum,
+		PageSize: queryRequest.PageSize,
+		Total:    total,
+		Contacts: contacts,
+	}
+	return
+}
+
+// QueryContactLogs query contact logs.
+func (s *Service) QueryContactLogs(c context.Context, queryRequest *model.QueryContactLogRequest) (p *model.PaginateContactLog, err error) {
+	var (
+		total       int64
+		machineLogs []*model.AboundContactLog
+	)
+	if total, machineLogs, err = s.dao.FindMachineLogs(queryRequest); err != nil {
+		return
+	}
+	p = &model.PaginateContactLog{
+		PageNum:     queryRequest.PageNum,
+		PageSize:    queryRequest.PageSize,
+		Total:       total,
+		MachineLogs: machineLogs,
+	}
+	return
+}
+
+// Wechat ...
+func (s *Service) Wechat() *wechat.Wechat {
+	return wechat.New(s.dao)
+}
+
+// CreateWechat ...
+func (s *Service) CreateWechat(c context.Context, req *model.CreateChatReq, username string) (resp *model.CreateChatResp, err error) {
+	var (
+		token     string
+		userIDs   []string
+		ownerInfo *model.ContactInfo
+		w         = wechat.New(s.dao)
+	)
+	u := qyWechatURL + "/cgi-bin/appchat/create"
+	params := url.Values{}
+	wechatInfo := &model.WechatCreateLog{
+		Name:   req.Name,
+		Owner:  req.Owner,
+		ChatID: req.ChatID,
+		Cuser:  username,
+		Status: 1,
+	}
+
+	//获取企业token
+	if token, err = w.AccessToken(c, conf.Conf.Property.Wechat); err != nil {
+		return
+	}
+	params.Set("access_token", token)
+
+	//get owner and users id
+	if ownerInfo, err = s.dao.QueryUserByUserName(req.Owner); err != nil {
+		return
+	}
+	if userIDs, err = s.QueryUserIds(req.UserList); err != nil {
+		return
+	}
+
+	req.Owner = ownerInfo.UserID
+	req.UserList = userIDs
+	if err = s.dao.PostJSON(c, u, "", params, &resp, req); err != nil {
+		return
+	}
+
+	//add create wechat info to database
+	if err = s.dao.AddWechatCreateLog(wechatInfo); err != nil {
+		return
+	}
+
+	resp = &model.CreateChatResp{
+		ChatID: wechatInfo.ChatID,
+	}
+	return
+}
+
+// QueryUserIds ...
+func (s *Service) QueryUserIds(userNames []string) (userIds []string, err error) {
+	var (
+		userName    string
+		contactInfo *model.ContactInfo
+	)
+	if len(userNames) == 0 {
+		err = errors.Errorf("UserIds: userNames is empty!")
+		return
+	}
+
+	for _, userName = range userNames {
+		if contactInfo, err = s.dao.QueryUserByUserName(userName); err != nil {
+			err = errors.Wrapf(err, "UserIds: no such user (%s) in db, err (%s)", userName, err.Error())
+			return
+		}
+
+		log.Info("UserIds: username (%s), userid (%s)", userName, contactInfo.UserID)
+		if contactInfo.UserID != "" {
+			userIds = append(userIds, contactInfo.UserID)
+		}
+	}
+	return
+}
+
+// QueryWechatCreateLog ...
+func (s *Service) QueryWechatCreateLog(c context.Context, req *model.Pagination, username string) (resp *model.CreateChatLogResp, err error) {
+	var (
+		logs             []*model.WechatCreateLog
+		logsResp         []*model.CreateChatLog
+		total            int
+		wechatCreateInfo *model.WechatCreateLog
+	)
+
+	if logs, total, err = s.dao.QueryWechatCreateLog(true, req, wechatCreateInfo); err != nil {
+		return
+	}
+	for _, log := range logs {
+		createChatlog := &model.CreateChatLog{}
+		if log.Cuser == username {
+			createChatlog.Buttons = append(createChatlog.Buttons, "WECHAT_TEST")
+		}
+		createChatlog.WechatCreateLog = log
+		logsResp = append(logsResp, createChatlog)
+	}
+	resp = &model.CreateChatLogResp{
+		Total:      total,
+		Pagination: req,
+		Logs:       logsResp,
+	}
+	return
+}
+
+// WechatParams ...
+func (s *Service) WechatParams(c context.Context, chatid string) (resp *model.GetChatResp, err error) {
+	var (
+		w     = wechat.New(s.dao)
+		token string
+	)
+
+	if token, err = w.AccessToken(c, conf.Conf.Property.Wechat); err != nil {
+		return
+	}
+
+	u := qyWechatURL + "/cgi-bin/appchat/get"
+	params := url.Values{}
+	params.Set("access_token", token)
+	params.Set("chatid", chatid)
+	err = s.dao.WechatParams(c, u, params, &resp)
+	return
+}
+
+// SendGroupWechat ...
+func (s *Service) SendGroupWechat(c context.Context, req *model.SendChatReq) (resp *model.ChatResp, err error) {
+	var (
+		token       string
+		w           = wechat.New(s.dao)
+		total       int
+		getChatResp *model.GetChatResp
+		owner       string
+		contentDB   = req.Text.Content
+	)
+	u := qyWechatURL + "/cgi-bin/appchat/send"
+	params := url.Values{}
+
+	if token, err = w.AccessToken(c, conf.Conf.Property.Wechat); err != nil {
+		return
+	}
+	params.Set("access_token", token)
+
+	if err = s.dao.PostJSON(c, u, "", params, &resp, req); err != nil {
+		return
+	}
+	if len(contentDB) > model.MaxWechatLen {
+		contentDB = contentDB[:model.MaxWechatLen]
+	}
+	chatLog := &model.WechatChatLog{
+		ChatID:  req.ChatID,
+		MsgType: req.MsgType,
+		Content: contentDB,
+		Safe:    req.Safe,
+		Status:  1,
+	}
+	if err = s.dao.CreateChatLog(chatLog); err != nil {
+		return
+	}
+
+	info := &model.WechatCreateLog{
+		ChatID: req.ChatID,
+	}
+	if _, total, err = s.dao.QueryWechatCreateLog(false, nil, info); err != nil {
+		return
+	}
+
+	if total == 0 {
+		getChatResp, _ = s.WechatParams(c, req.ChatID)
+		owner = getChatResp.ChatInfo.Owner
+		contactInfo, _ := s.dao.QueryUserByID(owner)
+		wechatInfo := &model.WechatCreateLog{
+			Name:   getChatResp.ChatInfo.Name,
+			Owner:  contactInfo.UserName,
+			ChatID: req.ChatID,
+			Status: 2,
+		}
+		if err = s.dao.AddWechatCreateLog(wechatInfo); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// SendWechat ...
+func (s *Service) SendWechat(c context.Context, req *model.SendMessageReq) (resp *model.ChatResp, err error) {
+	var (
+		w = wechat.New(s.dao)
+	)
+	err = w.PushMsg(c, req.Touser, req.Content)
+	return
+}
+
+// UpdateWechat ...
+func (s *Service) UpdateWechat(c context.Context, req *model.UpdateChatReq) (resp *model.ChatResp, err error) {
+	var (
+		token string
+		w     = wechat.New(s.dao)
+	)
+	u := qyWechatURL + "/cgi-bin/appchat/update"
+	params := url.Values{}
+
+	if token, err = w.AccessToken(c, conf.Conf.Property.Wechat); err != nil {
+		return
+	}
+	params.Set("access_token", token)
+	if err = s.dao.PostJSON(c, u, "", params, &resp, req); err != nil {
+		return
+	}
+	return
+}
+
+// SyncWechatContacts ...
+func (s *Service) SyncWechatContacts(c context.Context) (message string, err error) {
+	var (
+		w = wechat.New(s.dao)
+	)
+	if err = w.AnalysisContacts(c); err != nil {
+		return
+	}
+	message = "同步完成"
+	return
+}

app/admin/ep/saga/service/wechat/BUILD

@@ -0,0 +1,53 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["wechat_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/dao:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "contact.go",
+        "wechat.go",
+    ],
+    importpath = "go-common/app/admin/ep/saga/service/wechat",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/admin/ep/saga/conf:go_default_library",
+        "//app/admin/ep/saga/dao:go_default_library",
+        "//app/admin/ep/saga/model:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/admin/ep/saga/service/wechat/contact.go

@@ -0,0 +1,189 @@
+package wechat
+
+import (
+	"context"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+)
+
+// Changes changes structure
+type Changes struct {
+	Adds []*model.ContactInfo
+	Upts []*model.ContactInfo
+	Dels []*model.ContactInfo
+}
+
+// SyncContacts sync the contacts from wechat work 更新用户信息列表
+func (w *Wechat) SyncContacts(c context.Context) (err error) {
+	//更新数据库用户列表 contact_infos
+	if err = w.AnalysisContacts(c); err != nil {
+		return
+	}
+	//更新用户saga信息
+	/*if err = w.UpdateVisible(c); err != nil {
+		return
+	}*/
+	return
+}
+
+// AnalysisContacts analysis the contact difference and save them 从企业微信更新用户列表最新
+func (w *Wechat) AnalysisContacts(c context.Context) (err error) {
+	var (
+		contactsInDB   []*model.ContactInfo
+		wechatContacts []*model.ContactInfo
+		changes        = &Changes{}
+	)
+	//数据库里查询用户信息列表
+	if contactsInDB, err = w.dao.ContactInfos(); err != nil {
+		return
+	}
+	//企业微信接口查询用户列表
+	if wechatContacts, err = w.QueryWechatContacts(c); err != nil {
+		return
+	}
+
+	if changes, err = w.diffChanges(wechatContacts, contactsInDB); err != nil {
+		return
+	}
+
+	//saveChanges 更新用户信息
+	if err = w.saveChanges(changes); err != nil {
+		return
+	}
+	return
+}
+
+// QueryWechatContacts query wechat contacts with access token 获取用户信息列表
+func (w *Wechat) QueryWechatContacts(c context.Context) (contacts []*model.ContactInfo, err error) {
+	var (
+		token string
+	)
+	if token, err = w.AccessToken(c, w.contact); err != nil {
+		return
+	}
+
+	if contacts, err = w.dao.WechatContacts(c, token); err != nil {
+		return
+	}
+
+	return
+}
+
+//saveChanges 保存企业微信与数据库用户信息的更新内容
+func (w *Wechat) saveChanges(changes *Changes) (err error) {
+	var (
+		contact *model.ContactInfo
+	)
+
+	log.Info("saveChanges add(%d), upt(%d), del(%d)", len(changes.Adds), len(changes.Upts), len(changes.Dels))
+	for _, contact = range changes.Adds {
+		if err = w.dao.CreateContact(contact); err != nil {
+			return
+		}
+		log.Info("saveChanges add: %v", contact)
+	}
+
+	for _, contact = range changes.Upts {
+		if err = w.dao.UptContact(contact); err != nil {
+			return
+		}
+		log.Info("saveChanges upt: %v", contact)
+	}
+
+	for _, contact = range changes.Dels {
+		if err = w.dao.DelContact(contact); err != nil {
+			return
+		}
+		log.Info("saveChanges del: %v", contact)
+	}
+
+	return
+}
+
+//diffChanges 对比企业微信用户列表和数据库用户列表
+func (w *Wechat) diffChanges(wechatContacts, contactsInDB []*model.ContactInfo) (changes *Changes, err error) {
+	var (
+		contact           *model.ContactInfo
+		wechatContactsMap = make(map[string]*model.ContactInfo)
+		contactsInDBMap   = make(map[string]*model.ContactInfo)
+		wechatContactIDs  []string
+		dbContactsIDs     []string
+		userID            string
+	)
+	changes = new(Changes)
+	for _, contact = range wechatContacts {
+		wechatContactsMap[contact.UserID] = contact
+		wechatContactIDs = append(wechatContactIDs, contact.UserID)
+	}
+	for _, contact = range contactsInDB {
+		contactsInDBMap[contact.UserID] = contact
+		dbContactsIDs = append(dbContactsIDs, contact.UserID)
+	}
+
+	// 分析变化
+	for _, userID = range wechatContactIDs {
+		contact = wechatContactsMap[userID]
+		if w.inSlice(dbContactsIDs, userID) { // 企业微信联系人ID，在数据库中能找到
+			if !contact.AlmostEqual(contactsInDBMap[userID]) { // 但是域不同
+				contact.ID = contactsInDBMap[userID].ID
+				changes.Upts = append(changes.Upts, contact)
+			}
+		} else {
+			changes.Adds = append(changes.Adds, contact) // 这个联系人是新增的
+		}
+	}
+
+	for _, userID = range dbContactsIDs {
+		if !w.inSlice(wechatContactIDs, userID) {
+			changes.Dels = append(changes.Dels, contactsInDBMap[userID])
+		}
+	}
+
+	return
+}
+
+func (w *Wechat) inSlice(slice []string, target string) bool {
+	for _, v := range slice {
+		if v == target {
+			return true
+		}
+	}
+	return false
+}
+
+// UpdateVisible update the visible property 更新用户saga信息
+func (w *Wechat) UpdateVisible(c context.Context) (err error) {
+	var (
+		user    *model.UserInfo
+		users   []*model.UserInfo
+		contact *model.ContactInfo
+	)
+	//获取用户ID列表
+	if users, err = w.querySagaVisible(c); err != nil {
+		return
+	}
+
+	for _, user = range users {
+		//定义用户信息--saga是否可使用（是）
+		contact = &model.ContactInfo{UserID: user.UserID, VisibleSaga: true}
+		if err = w.dao.UptContact(contact); err != nil {
+			return
+		}
+	}
+	return
+}
+
+//querySagaVisible 获取用户ID列表
+func (w *Wechat) querySagaVisible(c context.Context) (users []*model.UserInfo, err error) {
+	var (
+		token string
+	)
+	if token, err = w.AccessToken(c, w.saga); err != nil {
+		return
+	}
+
+	if users, err = w.dao.WechatSagaVisible(c, token, w.saga.AppID); err != nil {
+		return
+	}
+	return
+}

app/admin/ep/saga/service/wechat/wechat.go

@@ -0,0 +1,182 @@
+package wechat
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/dao"
+	"go-common/app/admin/ep/saga/model"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+// Wechat 企业微信应用
+type Wechat struct {
+	dao     *dao.Dao
+	saga    *model.AppConfig
+	contact *model.AppConfig
+}
+
+// New create an new wechat work
+func New(d *dao.Dao) (w *Wechat) {
+	w = &Wechat{
+		dao:     d,
+		saga:    conf.Conf.Property.Wechat,
+		contact: conf.Conf.Property.Contact,
+	}
+
+	return w
+}
+
+// NewTxtNotify create wechat format text notification 从配置初始化企业微信TxtNotification
+func (w *Wechat) NewTxtNotify(content string) (txtMsg *model.TxtNotification) {
+	return &model.TxtNotification{
+		Notification: model.Notification{
+			MsgType: "text",
+			AgentID: w.saga.AppID,
+		},
+		Body: model.Text{
+			Content: content,
+		},
+		Safe: 0,
+	}
+}
+
+// AccessToken get access_token from cache first, if not found, get it via wechat api.
+func (w *Wechat) AccessToken(c context.Context, app *model.AppConfig) (token string, err error) {
+	var (
+		key    string
+		expire int32
+	)
+
+	key = fmt.Sprintf("appid_%d", app.AppID)
+	if token, err = w.dao.AccessTokenRedis(c, key); err != nil {
+		log.Warn("AccessToken: failed to get access_token from cache, appId (%s), error (%s)", app.AppID, err.Error())
+		//企业微信api获取公司token
+		if token, expire, err = w.dao.WechatAccessToken(c, app.AppSecret); err != nil {
+			err = errors.Wrapf(err, "AccessToken: both mc and api can't provide access_token, appId(%s)", app.AppID)
+			return
+		}
+
+		// 通过API获取到了，缓存一波
+		err = w.dao.SetAccessTokenRedis(c, key, token, expire)
+		return
+	}
+
+	if token == "" {
+		if token, expire, err = w.dao.WechatAccessToken(c, app.AppSecret); err != nil {
+			return
+		}
+
+		// 通过API获取到了，缓存一波
+		err = w.dao.SetAccessTokenRedis(c, key, token, expire)
+	}
+
+	return
+}
+
+// PushMsg push text message via wechat notification api with access_token.推送企业微信
+func (w *Wechat) PushMsg(c context.Context, userNames []string, content string) (err error) {
+	var (
+		token         string
+		userIds       string
+		invalidUser   string
+		userNamesByte []byte
+		txtMsg        = w.NewTxtNotify(content)
+		contentDB     = content
+	)
+	//获取企业token
+	if token, err = w.AccessToken(c, w.saga); err != nil {
+		return
+	}
+
+	if token == "" {
+		err = errors.Errorf("PushMsg: get access token failed, it's empty. appid (%s), secret (%s)", w.saga.AppID, w.saga.AppSecret)
+		return
+	}
+	//员工编号以竖线分隔
+	if userIds, err = w.UserIds(userNames); err != nil {
+		return
+	}
+	txtMsg.ToUser = userIds
+
+	if invalidUser, err = w.dao.WechatPushMsg(c, token, txtMsg); err != nil {
+		if err = w.addRequireVisible(c, invalidUser); err != nil {
+			log.Error("PushMsg add userID (%s) in cache, error(%s)", invalidUser, err.Error())
+		}
+		return
+	}
+	if userNamesByte, err = json.Marshal(userNames); err != nil {
+		return
+	}
+	if len(contentDB) > model.MaxWechatLen {
+		contentDB = contentDB[:model.MaxWechatLen]
+	}
+	messageLog := &model.WechatMessageLog{
+		Touser:  string(userNamesByte),
+		Content: contentDB,
+		Status:  1,
+	}
+	return w.dao.CreateMessageLog(messageLog)
+}
+
+// UserIds query user ids for user name list 查询员工编号
+func (w *Wechat) UserIds(userNames []string) (ids string, err error) {
+	if ids, err = w.dao.UserIds(userNames); err != nil {
+		return
+	}
+	return
+}
+
+// addRequireVisible update wechat require visible users in memcache
+func (w *Wechat) addRequireVisible(c context.Context, userIDs string) (err error) {
+	var (
+		contactInfo *model.ContactInfo
+		userID      string
+		alreadyIn   bool
+	)
+
+	users := strings.Split(userIDs, "|")
+	for _, userID = range users {
+		//查看是否缓存，缓存则继续
+		if alreadyIn, err = w.alreadyInCache(c, userID); err != nil || alreadyIn {
+			continue
+		}
+		//未缓存从数据库查询
+		if contactInfo, err = w.dao.QueryUserByID(userID); err != nil {
+			log.Error("no such userID (%s) in db, error(%s)", userID, err.Error())
+			return
+		}
+		//数据库查询结果缓存
+		if err = w.dao.SetRequireVisibleUsersRedis(c, contactInfo); err != nil {
+			log.Error("failed set to cache userID (%s) username (%s), err (%s)", userID, contactInfo.UserName, err.Error())
+			return
+		}
+	}
+	return
+}
+
+// alreadyInCache check user is or not in the memcache
+func (w *Wechat) alreadyInCache(c context.Context, userID string) (alreadyIn bool, err error) {
+	var (
+		userMap = make(map[string]model.RequireVisibleUser)
+	)
+	//查询所有的值
+	if err = w.dao.RequireVisibleUsersRedis(c, &userMap); err != nil {
+		log.Error("get userID (%s) from cache error(%s)", userID, err.Error())
+		return
+	}
+	//匹配需要查询的用户id
+	for k, v := range userMap {
+		if userID == k {
+			log.Info("(%s) is already exist in cache, value(%v)", k, v)
+			alreadyIn = true
+			return
+		}
+	}
+	return
+}

app/admin/ep/saga/service/wechat/wechat_test.go

@@ -0,0 +1,143 @@
+package wechat
+
+import (
+	"context"
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/admin/ep/saga/conf"
+	"go-common/app/admin/ep/saga/dao"
+	"go-common/app/admin/ep/saga/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	mydao  *dao.Dao
+	wechat *Wechat
+	ctx    = context.Background()
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	flag.Set("conf", "../../cmd/saga-admin-test.toml")
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+
+	mydao = dao.New()
+	defer mydao.Close()
+	wechat = New(mydao)
+
+	os.Exit(m.Run())
+}
+
+func TestAddRequireVisible(t *testing.T) {
+	var (
+		err     error
+		userMap = make(map[string]model.RequireVisibleUser)
+	)
+
+	Convey("TEST addRequireVisible", t, func() {
+		err = wechat.addRequireVisible(ctx, "000000")
+		So(err, ShouldNotBeNil)
+		err = wechat.addRequireVisible(ctx, "001134")
+		So(err, ShouldBeNil)
+		err = mydao.RequireVisibleUsersRedis(ctx, &userMap)
+		So(err, ShouldBeNil)
+		So(userMap, ShouldContainKey, "001134")
+	})
+}
+
+func TestAlreadyInCache(t *testing.T) {
+	var (
+		err         error
+		result      bool
+		contactInfo model.ContactInfo
+	)
+
+	contactInfo = model.ContactInfo{
+		ID:          "111",
+		UserName:    "zhangsan",
+		UserID:      "222",
+		NickName:    "xiaolizi",
+		VisibleSaga: true,
+	}
+
+	Convey("TEST alreadyInCache", t, func() {
+		result, err = wechat.alreadyInCache(ctx, "000")
+		So(err, ShouldBeNil)
+		So(result, ShouldEqual, false)
+		So(mydao.SetRequireVisibleUsersRedis(ctx, &contactInfo), ShouldBeNil)
+		result, err = wechat.alreadyInCache(ctx, "222")
+		So(err, ShouldBeNil)
+		So(result, ShouldEqual, true)
+	})
+}
+
+func TestSyncContacts(t *testing.T) {
+	var (
+		err         error
+		contactInfo = &model.ContactInfo{
+			UserID:   "E10021",
+			UserName: "eyotang",
+			NickName: "ben大神点C",
+		}
+		modify = &model.ContactInfo{
+			UserID:      "000328",
+			UserName:    "eyotang",
+			NickName:    "ben大神点C",
+			VisibleSaga: false,
+		}
+		target      *model.ContactInfo
+		almostEqual bool
+	)
+
+	Convey("TEST sync after add incorrect", t, func() {
+		err = wechat.dao.CreateContact(contactInfo)
+		So(err, ShouldBeNil)
+		target, err = wechat.dao.QueryUserByID(contactInfo.UserID)
+		So(err, ShouldBeNil)
+		almostEqual = contactInfo.AlmostEqual(target)
+		So(almostEqual, ShouldBeTrue)
+
+		err = wechat.SyncContacts(ctx)
+		So(err, ShouldBeNil)
+		target, err = wechat.dao.QueryUserByID(contactInfo.UserID)
+		So(err, ShouldNotBeNil)
+	})
+
+	Convey("TEST aync after change", t, func() {
+		contactInfo, err = wechat.dao.QueryUserByID(modify.UserID)
+		So(err, ShouldBeNil)
+		modify.ID = contactInfo.ID
+		err = wechat.dao.UptContact(contactInfo)
+		So(err, ShouldBeNil)
+
+		err = wechat.SyncContacts(ctx)
+		So(err, ShouldBeNil)
+		target, err = wechat.dao.QueryUserByID(modify.UserID)
+		So(err, ShouldBeNil)
+		So(target.VisibleSaga, ShouldBeTrue)
+		So(target.UserName, ShouldNotEqual, "eyotang")
+	})
+}
+
+func TestPushMsg(t *testing.T) {
+	var err error
+	userName := []string{"wuwei"}
+	content := "测试发送企业微信"
+	Convey("TEST push message", t, func() {
+		err = wechat.PushMsg(ctx, userName, content)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestAnalysisContacts(t *testing.T) {
+	var err error
+	Convey("TEST analysis contacts", t, func() {
+		err = wechat.AnalysisContacts(ctx)
+		So(err, ShouldBeNil)
+	})
+}