go-common/app/admin/ep/saga/service/mr.go

591 lines
16 KiB
Go
Raw Normal View History

2019-04-22 10:49:16 +00:00
package service
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
"go-common/app/admin/ep/saga/model"
"go-common/app/admin/ep/saga/service/utils"
"go-common/library/log"
"github.com/xanzy/go-gitlab"
)
const _specialMrID = 10199
// QueryProjectMr query project commit info according to project id.
func (s *Service) QueryProjectMr(c context.Context, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) {
if resp, err = s.QueryProject(c, model.ObjectMR, req); err != nil {
return
}
return
}
// QueryTeamMr query team commit info according to department and business
func (s *Service) QueryTeamMr(c context.Context, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) {
if resp, err = s.QueryTeam(c, model.ObjectMR, req); err != nil {
return
}
return
}
// QueryProjectMrReport query mr review
func (s *Service) QueryProjectMrReport(c context.Context, req *model.ProjectMrReportReq) (resp *model.ProjectMrReportResp, err error) {
var (
info []*model.MrInfo
changeAdd int
changeDel int
mrCount int
stateCount int
discussionCount int
discussionResolved int
mrTime int
spentTime time.Duration
avarageTime time.Duration
reviewers []string
reviews []string
reviewChangeAdd int
reviewChangeDel int
reviewTime int
reviewTotalTime time.Duration
)
if info, err = s.QueryAllMergeRequestInfo(c, req.ProjectID); err != nil {
return
}
for _, i := range info {
for _, r := range i.Reviewers {
if r.Name == req.Member {
reviews = append(reviews, i.Author)
reviewChangeAdd += i.ChangeAdd
reviewChangeDel += i.ChangeDel
reviewTime += i.SpentTime
}
}
if i.Author == req.Member {
mrCount++
if i.State == model.StatusMerged {
stateCount++
mrTime += i.SpentTime
changeAdd += i.ChangeAdd
changeDel += i.ChangeDel
discussionCount += i.TotalDiscussion
discussionResolved += i.SolvedDiscussion
for _, r := range i.Reviewers {
reviewers = append(reviewers, r.Name)
}
}
}
}
// 判断mr为零的情况
if mrCount == 0 {
resp = &model.ProjectMrReportResp{}
return
}
if spentTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime)); err != nil {
return
}
if avarageTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime/mrCount)); err != nil {
return
}
if reviewTotalTime, err = time.ParseDuration(fmt.Sprintf("%ds", reviewTime)); err != nil {
return
}
resp = &model.ProjectMrReportResp{
ChangeAdd: changeAdd,
ChangeDel: changeDel,
MrCount: mrCount,
SpentTime: spentTime.String(),
StateCount: stateCount,
AverageMerge: avarageTime.String(),
Reviewers: reviewers,
Discussion: discussionCount,
Resolve: discussionResolved,
ReviewerOther: reviews,
ReviewChangeAdd: reviewChangeAdd,
ReviewChangeDel: reviewChangeDel,
ReviewTotalTime: reviewTotalTime.String(),
}
return
}
// QueryAllMergeRequestInfo ...
func (s *Service) QueryAllMergeRequestInfo(c context.Context, projID int) (info []*model.MrInfo, err error) {
var (
until = time.Now()
since = until.AddDate(0, -1, 0)
mrs []*gitlab.MergeRequest
resp *gitlab.Response
)
for page := 1; ; page++ {
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projID, &since, &until, page); err != nil {
return
}
for _, m := range mrs {
mr := &model.MrInfo{ProjectID: projID, MrID: m.IID, Author: m.Author.Name, State: m.State}
if m.State == model.StatusMerged {
spent := m.UpdatedAt.Sub(*m.CreatedAt)
mr.SpentTime = int(spent.Seconds())
}
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projID, m.IID); err != nil {
return
}
if mr.Reviewers, err = s.QueryMergeRequestReview(c, projID, m.IID); err != nil {
return
}
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projID, m.IID); err != nil {
return
}
info = append(info, mr)
}
if resp.NextPage == 0 {
break
}
}
return
}
// QueryMergeRequestReview 查询mr reviewer信息
func (s *Service) QueryMergeRequestReview(c context.Context, projectID, mrIID int) (reviewers []*model.MrReviewer, err error) {
var (
notes []*model.StatisticsNotes
emojis []*model.StatisticsMRAwardEmojis
owners []string
r *regexp.Regexp
)
//query note
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
return
}
// query emoji
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
return
}
//评论中解析获取owner
if len(notes) == 0 {
return
}
if r, err = regexp.Compile("OWNER:.*?@(.*)(|)"); err == nil {
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
if len(matchResult) > 0 {
owners = strings.Split(matchResult[1], " 或 @")
}
}
mrCreatedAt := notes[len(notes)-1].CreatedAt
// 从评论和表情中解析 reviewers
for _, note := range notes {
if note.Body == "+1" {
reviewers = append(reviewers, &model.MrReviewer{Name: note.AuthorName, FinishedAt: note.CreatedAt, SpentTime: int(note.CreatedAt.Sub(*mrCreatedAt))})
}
}
for _, emoji := range emojis {
if emoji.Name == "thumbsup" {
reviewers = append(reviewers, &model.MrReviewer{Name: emoji.UserName, FinishedAt: emoji.CreatedAt, SpentTime: int(emoji.CreatedAt.Sub(*mrCreatedAt))})
}
}
// 判断reviewer类型
for _, r := range reviewers {
for _, owner := range owners {
if owner == r.Name {
r.UserType = "owner"
break
}
}
if r.UserType == "" {
r.UserType = "other"
}
}
return
}
// QueryMergeRequestDiscussion 查询获取mr的discussion
func (s *Service) QueryMergeRequestDiscussion(c context.Context, projectID, mrIID int) (total, solved int, err error) {
var discussions []*model.StatisticsDiscussions
if discussions, err = s.dao.DiscussionsByMRIID(c, projectID, mrIID); err != nil {
return
}
for _, d := range discussions {
var notesArray []int
if err = json.Unmarshal([]byte(d.Notes), &notesArray); err != nil {
return
}
for _, n := range notesArray {
var note *model.StatisticsNotes
if note, err = s.dao.NoteByID(c, projectID, mrIID, n); err != nil {
return
}
if note.Resolvable {
total++
if note.Resolved {
solved++
}
}
}
}
return
}
// QueryMergeRequestDiff 查询获取到mr修改文件的总行数, 参数 p project_ID m MR_ID
func (s *Service) QueryMergeRequestDiff(c context.Context, p, m int) (ChangeAdd, ChangeDel int, err error) {
var mr *gitlab.MergeRequest
// 此MR数据量太大曾导致过gitlab服务器崩溃访问会返回502服务器错误因此特殊处理。
if m == _specialMrID {
return 0, 0, nil
}
if mr, _, err = s.gitlab.GetMergeRequestDiff(p, m); err != nil {
return
}
for _, change := range mr.Changes {
rows := strings.Split(change.Diff, "\n")
if len(rows) < 3 {
// 处理diff为空
continue
}
for _, row := range rows[3:] {
if strings.HasPrefix(row, "+") {
ChangeAdd++
} else if strings.HasPrefix(row, "-") {
ChangeDel++
}
}
}
return
}
/*-------------------------------------- sync MR ----------------------------------------*/
// SyncProjectMR ...
func (s *Service) SyncProjectMR(c context.Context, projectID int) (result *model.SyncResult, err error) {
var (
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
syncAllTime = false
mrs []*gitlab.MergeRequest
resp *gitlab.Response
since *time.Time
until *time.Time
projectInfo *model.ProjectInfo
)
result = &model.SyncResult{}
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
return
}
if !syncAllTime {
since, until = utils.CalSyncTime()
}
log.Info("sync project(%d) MR time since: %v, until: %v", projectID, since, until)
for page := 1; ; page++ {
result.TotalPage++
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projectID, since, until, page); err != nil {
return
}
for _, mr := range mrs {
if err = s.structureDatabaseMR(c, projectID, projectInfo.Name, mr); err != nil {
log.Error("mr Save Database err: projectID(%d), MRIID(%d)", projectID, mr.IID)
err = nil
errData := &model.FailData{
ChildID: mr.IID,
}
result.FailData = append(result.FailData, errData)
continue
}
result.TotalNum++
}
if resp.NextPage == 0 {
break
}
}
return
}
// structureDatabaseMR ...
func (s *Service) structureDatabaseMR(c context.Context, projectID int, projectName string, mr *gitlab.MergeRequest) (err error) {
var (
milestoneID int
mrLables string
mrChanges string
mrLablesByte []byte
mrChangesByte []byte
)
if mr.Milestone != nil {
milestoneID = mr.Milestone.ID
}
if mrLablesByte, err = json.Marshal(mr.Labels); err != nil {
mrLables = model.JsonMarshalErrorText
} else {
mrLables = string(mrLablesByte)
}
if mrChangesByte, err = json.Marshal(mr.Changes); err != nil {
mrChanges = model.JsonMarshalErrorText
} else {
mrChanges = string(mrChangesByte)
}
mrDB := &model.StatisticsMrs{
MRID: mr.ID,
MRIID: mr.IID,
TargetBranch: mr.TargetBranch,
SourceBranch: mr.SourceBranch,
ProjectID: mr.ProjectID,
ProjectName: projectName,
Title: mr.Title,
State: mr.State,
CreatedAt: mr.CreatedAt,
UpdatedAt: mr.UpdatedAt,
Upvotes: mr.Upvotes,
Downvotes: mr.Downvotes,
AuthorID: mr.Author.ID,
AuthorName: mr.Author.Name,
AssigneeID: mr.Assignee.ID,
AssigneeName: mr.Assignee.Name,
SourceProjectID: mr.SourceProjectID,
TargetProjectID: mr.TargetProjectID,
Labels: mrLables,
Description: mr.Description,
WorkInProgress: mr.WorkInProgress,
MilestoneID: milestoneID,
MergeWhenPipelineSucceeds: mr.MergeWhenPipelineSucceeds,
MergeStatus: mr.MergeStatus,
MergedByID: mr.MergedBy.ID,
MergedByName: mr.MergedBy.Name,
MergedAt: mr.MergedAt,
ClosedByID: mr.ClosedBy.ID,
ClosedAt: mr.ClosedAt,
Subscribed: mr.Subscribed,
SHA: mr.SHA,
MergeCommitSHA: mr.MergeCommitSHA,
UserNotesCount: mr.UserNotesCount,
ChangesCount: mr.ChangesCount,
ShouldRemoveSourceBranch: mr.ShouldRemoveSourceBranch,
ForceRemoveSourceBranch: mr.ForceRemoveSourceBranch,
WebURL: mr.WebURL,
DiscussionLocked: mr.DiscussionLocked,
Changes: mrChanges,
TimeStatsHumanTimeEstimate: mr.TimeStats.HumanTimeEstimate,
TimeStatsHumanTotalTimeSpent: mr.TimeStats.HumanTotalTimeSpent,
TimeStatsTimeEstimate: mr.TimeStats.TimeEstimate,
TimeStatsTotalTimeSpent: mr.TimeStats.TotalTimeSpent,
Squash: mr.Squash,
PipelineID: mr.Pipeline.ID,
}
if len(mrDB.Labels) > model.MessageMaxLen {
mrDB.Labels = mrDB.Labels[0 : model.MessageMaxLen-1]
}
if len(mrDB.Description) > model.MessageMaxLen {
mrDB.Description = mrDB.Description[0 : model.MessageMaxLen-1]
}
return s.SaveDatabaseMR(c, mrDB)
}
// SaveDatabaseMR ...
func (s *Service) SaveDatabaseMR(c context.Context, mrDB *model.StatisticsMrs) (err error) {
var total int
if total, err = s.dao.HasMR(c, mrDB.ProjectID, mrDB.MRIID); err != nil {
log.Error("SaveDatabaseMR HasMR(%+v)", err)
return
}
// found only one, so update
if total == 1 {
return s.dao.UpdateMR(c, mrDB.ProjectID, mrDB.MRIID, mrDB)
} else if total > 1 {
// found repeated row, this situation will not exist under normal
log.Warn("SaveDatabaseMR mr has more rows(%d)", total)
return
}
// insert row now
return s.dao.CreateMR(c, mrDB)
}
/*-------------------------------------- agg MR ----------------------------------------*/
// AggregateProjectMR ...
func (s *Service) AggregateProjectMR(c context.Context, projectID int) (err error) {
var (
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
syncAllTime = false
mrs []*model.StatisticsMrs
projectInfo *model.ProjectInfo
since *time.Time
until *time.Time
)
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
return
}
if !syncAllTime {
since, until = utils.CalSyncTime()
}
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
return
}
for _, mr := range mrs {
if err = s.MRAddedInfoDB(c, projectID, mr); err != nil {
log.Error("MRAddedInfoDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
err = nil
}
if err = s.MRReviewerDB(c, projectID, mr.MRIID, projectInfo.Name, mr); err != nil {
log.Error("MRReviewerDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
err = nil
}
}
return
}
// MRAddedInfoDB ...
func (s *Service) MRAddedInfoDB(c context.Context, projectID int, mr *model.StatisticsMrs) (err error) {
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projectID, mr.MRIID); err != nil {
return
}
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projectID, mr.MRIID); err != nil {
return
}
return s.dao.UpdateMR(c, projectID, mr.MRIID, mr)
}
// MRReviewerDB 查询mr reviewer信息
func (s *Service) MRReviewerDB(c context.Context, projectID, mrIID int, projectName string, mr *model.StatisticsMrs) (err error) {
var (
notes []*model.StatisticsNotes
emojis []*model.StatisticsMRAwardEmojis
owners []string
r *regexp.Regexp
reviewers []*model.AggregateMrReviewer
)
//query note
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
return
}
// query emoji
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
return
}
//评论中解析获取owner
if len(notes) == 0 {
return
}
if r, err = regexp.Compile("OWNER:.*?@(.*)(|)"); err == nil {
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
if len(matchResult) > 0 {
owners = strings.Split(matchResult[1], " 或 @")
}
}
mrCreatedAt := notes[len(notes)-1].CreatedAt
// 从评论和表情中解析 reviewers
for _, note := range notes {
if note.Body == "+1" {
reviewer := &model.AggregateMrReviewer{
ReviewerID: note.AuthorID,
ReviewerName: note.AuthorName,
ReviewType: "note",
ReviewID: note.NoteID,
ReviewCommand: "+1",
CreatedAt: note.CreatedAt,
ApproveTime: int(note.CreatedAt.Sub(*mrCreatedAt).Seconds()),
MergeTime: int(mr.UpdatedAt.Sub(*note.CreatedAt).Seconds()),
}
reviewers = append(reviewers, reviewer)
}
}
for _, emoji := range emojis {
if emoji.Name == "thumbsup" {
reviewer := &model.AggregateMrReviewer{
ReviewerID: emoji.UserID,
ReviewerName: emoji.UserName,
ReviewType: "emoji",
ReviewID: emoji.AwardEmojiID,
ReviewCommand: "thumbsup",
CreatedAt: emoji.CreatedAt,
ApproveTime: int(emoji.CreatedAt.Sub(*mrCreatedAt).Seconds()),
MergeTime: int(mr.UpdatedAt.Sub(*emoji.CreatedAt).Seconds()),
}
reviewers = append(reviewers, reviewer)
}
}
// 判断reviewer类型
for _, r := range reviewers {
r.ProjectID = projectID
r.ProjectName = projectName
r.MrIID = mrIID
r.Title = mr.Title
r.WebUrl = mr.WebURL
r.AuthorName = mr.AuthorName
if utils.InSlice(r.ReviewerName, owners) {
r.UserType = "owner"
} else {
r.UserType = "other"
}
if err = s.SaveDatabaseAggMR(c, r); err != nil {
log.Error("mrReviewer 存数据库报错(%+V)", err)
continue
}
}
return
}
// SaveDatabaseAggMR ...
func (s *Service) SaveDatabaseAggMR(c context.Context, mrDB *model.AggregateMrReviewer) (err error) {
var total int
if total, err = s.dao.HasAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID); err != nil {
log.Error("SaveDatabaseAggMR HasAggMR(%+v)", err)
return
}
// found only one, so update
if total == 1 {
return s.dao.UpdateAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID, mrDB)
} else if total > 1 {
// found repeated row, this situation will not exist under normal
log.Warn("SaveDatabaseAggMR aggMR has more rows(%d)", total)
return
}
// insert row now
return s.dao.CreateAggregateReviewer(c, mrDB)
}