591 lines
16 KiB
Go
591 lines
16 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"regexp"
|
||
"strings"
|
||
"time"
|
||
|
||
"go-common/app/admin/ep/saga/model"
|
||
"go-common/app/admin/ep/saga/service/utils"
|
||
"go-common/library/log"
|
||
|
||
"github.com/xanzy/go-gitlab"
|
||
)
|
||
|
||
const _specialMrID = 10199
|
||
|
||
// QueryProjectMr query project commit info according to project id.
|
||
func (s *Service) QueryProjectMr(c context.Context, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) {
|
||
|
||
if resp, err = s.QueryProject(c, model.ObjectMR, req); err != nil {
|
||
return
|
||
}
|
||
return
|
||
}
|
||
|
||
// QueryTeamMr query team commit info according to department and business
|
||
func (s *Service) QueryTeamMr(c context.Context, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) {
|
||
|
||
if resp, err = s.QueryTeam(c, model.ObjectMR, req); err != nil {
|
||
return
|
||
}
|
||
return
|
||
}
|
||
|
||
// QueryProjectMrReport query mr review
|
||
func (s *Service) QueryProjectMrReport(c context.Context, req *model.ProjectMrReportReq) (resp *model.ProjectMrReportResp, err error) {
|
||
var (
|
||
info []*model.MrInfo
|
||
changeAdd int
|
||
changeDel int
|
||
mrCount int
|
||
stateCount int
|
||
discussionCount int
|
||
discussionResolved int
|
||
mrTime int
|
||
spentTime time.Duration
|
||
avarageTime time.Duration
|
||
reviewers []string
|
||
reviews []string
|
||
reviewChangeAdd int
|
||
reviewChangeDel int
|
||
reviewTime int
|
||
reviewTotalTime time.Duration
|
||
)
|
||
if info, err = s.QueryAllMergeRequestInfo(c, req.ProjectID); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, i := range info {
|
||
for _, r := range i.Reviewers {
|
||
if r.Name == req.Member {
|
||
reviews = append(reviews, i.Author)
|
||
reviewChangeAdd += i.ChangeAdd
|
||
reviewChangeDel += i.ChangeDel
|
||
reviewTime += i.SpentTime
|
||
}
|
||
}
|
||
if i.Author == req.Member {
|
||
mrCount++
|
||
if i.State == model.StatusMerged {
|
||
stateCount++
|
||
mrTime += i.SpentTime
|
||
changeAdd += i.ChangeAdd
|
||
changeDel += i.ChangeDel
|
||
discussionCount += i.TotalDiscussion
|
||
discussionResolved += i.SolvedDiscussion
|
||
for _, r := range i.Reviewers {
|
||
reviewers = append(reviewers, r.Name)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// 判断mr为零的情况
|
||
if mrCount == 0 {
|
||
resp = &model.ProjectMrReportResp{}
|
||
return
|
||
}
|
||
|
||
if spentTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime)); err != nil {
|
||
return
|
||
}
|
||
if avarageTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime/mrCount)); err != nil {
|
||
return
|
||
}
|
||
if reviewTotalTime, err = time.ParseDuration(fmt.Sprintf("%ds", reviewTime)); err != nil {
|
||
return
|
||
}
|
||
|
||
resp = &model.ProjectMrReportResp{
|
||
ChangeAdd: changeAdd,
|
||
ChangeDel: changeDel,
|
||
MrCount: mrCount,
|
||
SpentTime: spentTime.String(),
|
||
StateCount: stateCount,
|
||
AverageMerge: avarageTime.String(),
|
||
Reviewers: reviewers,
|
||
Discussion: discussionCount,
|
||
Resolve: discussionResolved,
|
||
ReviewerOther: reviews,
|
||
ReviewChangeAdd: reviewChangeAdd,
|
||
ReviewChangeDel: reviewChangeDel,
|
||
ReviewTotalTime: reviewTotalTime.String(),
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryAllMergeRequestInfo ...
|
||
func (s *Service) QueryAllMergeRequestInfo(c context.Context, projID int) (info []*model.MrInfo, err error) {
|
||
var (
|
||
until = time.Now()
|
||
since = until.AddDate(0, -1, 0)
|
||
mrs []*gitlab.MergeRequest
|
||
resp *gitlab.Response
|
||
)
|
||
|
||
for page := 1; ; page++ {
|
||
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projID, &since, &until, page); err != nil {
|
||
return
|
||
}
|
||
for _, m := range mrs {
|
||
|
||
mr := &model.MrInfo{ProjectID: projID, MrID: m.IID, Author: m.Author.Name, State: m.State}
|
||
|
||
if m.State == model.StatusMerged {
|
||
spent := m.UpdatedAt.Sub(*m.CreatedAt)
|
||
mr.SpentTime = int(spent.Seconds())
|
||
}
|
||
|
||
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projID, m.IID); err != nil {
|
||
return
|
||
}
|
||
|
||
if mr.Reviewers, err = s.QueryMergeRequestReview(c, projID, m.IID); err != nil {
|
||
return
|
||
}
|
||
|
||
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projID, m.IID); err != nil {
|
||
return
|
||
}
|
||
|
||
info = append(info, mr)
|
||
}
|
||
if resp.NextPage == 0 {
|
||
break
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryMergeRequestReview 查询mr reviewer信息
|
||
func (s *Service) QueryMergeRequestReview(c context.Context, projectID, mrIID int) (reviewers []*model.MrReviewer, err error) {
|
||
var (
|
||
notes []*model.StatisticsNotes
|
||
emojis []*model.StatisticsMRAwardEmojis
|
||
owners []string
|
||
r *regexp.Regexp
|
||
)
|
||
|
||
//query note
|
||
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
|
||
return
|
||
}
|
||
|
||
// query emoji
|
||
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
|
||
return
|
||
}
|
||
|
||
//评论中解析获取owner
|
||
if len(notes) == 0 {
|
||
return
|
||
}
|
||
if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil {
|
||
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
|
||
if len(matchResult) > 0 {
|
||
owners = strings.Split(matchResult[1], " 或 @")
|
||
}
|
||
}
|
||
|
||
mrCreatedAt := notes[len(notes)-1].CreatedAt
|
||
|
||
// 从评论和表情中解析 reviewers
|
||
for _, note := range notes {
|
||
if note.Body == "+1" {
|
||
reviewers = append(reviewers, &model.MrReviewer{Name: note.AuthorName, FinishedAt: note.CreatedAt, SpentTime: int(note.CreatedAt.Sub(*mrCreatedAt))})
|
||
}
|
||
}
|
||
for _, emoji := range emojis {
|
||
if emoji.Name == "thumbsup" {
|
||
reviewers = append(reviewers, &model.MrReviewer{Name: emoji.UserName, FinishedAt: emoji.CreatedAt, SpentTime: int(emoji.CreatedAt.Sub(*mrCreatedAt))})
|
||
}
|
||
}
|
||
|
||
// 判断reviewer类型
|
||
for _, r := range reviewers {
|
||
for _, owner := range owners {
|
||
if owner == r.Name {
|
||
r.UserType = "owner"
|
||
break
|
||
}
|
||
}
|
||
if r.UserType == "" {
|
||
r.UserType = "other"
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryMergeRequestDiscussion 查询获取mr的discussion
|
||
func (s *Service) QueryMergeRequestDiscussion(c context.Context, projectID, mrIID int) (total, solved int, err error) {
|
||
var discussions []*model.StatisticsDiscussions
|
||
if discussions, err = s.dao.DiscussionsByMRIID(c, projectID, mrIID); err != nil {
|
||
return
|
||
}
|
||
for _, d := range discussions {
|
||
var notesArray []int
|
||
if err = json.Unmarshal([]byte(d.Notes), ¬esArray); err != nil {
|
||
return
|
||
}
|
||
for _, n := range notesArray {
|
||
var note *model.StatisticsNotes
|
||
if note, err = s.dao.NoteByID(c, projectID, mrIID, n); err != nil {
|
||
return
|
||
}
|
||
if note.Resolvable {
|
||
total++
|
||
if note.Resolved {
|
||
solved++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// QueryMergeRequestDiff 查询获取到mr修改文件的总行数, 参数 p project_ID m MR_ID
|
||
func (s *Service) QueryMergeRequestDiff(c context.Context, p, m int) (ChangeAdd, ChangeDel int, err error) {
|
||
var mr *gitlab.MergeRequest
|
||
|
||
// 此MR数据量太大，曾导致过gitlab服务器崩溃，访问会返回502服务器错误，因此特殊处理。
|
||
if m == _specialMrID {
|
||
return 0, 0, nil
|
||
}
|
||
|
||
if mr, _, err = s.gitlab.GetMergeRequestDiff(p, m); err != nil {
|
||
return
|
||
}
|
||
for _, change := range mr.Changes {
|
||
rows := strings.Split(change.Diff, "\n")
|
||
if len(rows) < 3 {
|
||
// 处理diff为空
|
||
continue
|
||
}
|
||
|
||
for _, row := range rows[3:] {
|
||
if strings.HasPrefix(row, "+") {
|
||
ChangeAdd++
|
||
} else if strings.HasPrefix(row, "-") {
|
||
ChangeDel++
|
||
}
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
/*-------------------------------------- sync MR ----------------------------------------*/
|
||
|
||
// SyncProjectMR ...
|
||
func (s *Service) SyncProjectMR(c context.Context, projectID int) (result *model.SyncResult, err error) {
|
||
var (
|
||
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
syncAllTime = false
|
||
mrs []*gitlab.MergeRequest
|
||
resp *gitlab.Response
|
||
since *time.Time
|
||
until *time.Time
|
||
projectInfo *model.ProjectInfo
|
||
)
|
||
result = &model.SyncResult{}
|
||
|
||
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
return
|
||
}
|
||
|
||
if !syncAllTime {
|
||
since, until = utils.CalSyncTime()
|
||
}
|
||
log.Info("sync project(%d) MR time since: %v, until: %v", projectID, since, until)
|
||
for page := 1; ; page++ {
|
||
result.TotalPage++
|
||
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projectID, since, until, page); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, mr := range mrs {
|
||
if err = s.structureDatabaseMR(c, projectID, projectInfo.Name, mr); err != nil {
|
||
log.Error("mr Save Database err: projectID(%d), MRIID(%d)", projectID, mr.IID)
|
||
err = nil
|
||
|
||
errData := &model.FailData{
|
||
ChildID: mr.IID,
|
||
}
|
||
result.FailData = append(result.FailData, errData)
|
||
continue
|
||
}
|
||
result.TotalNum++
|
||
}
|
||
|
||
if resp.NextPage == 0 {
|
||
break
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// structureDatabaseMR ...
|
||
func (s *Service) structureDatabaseMR(c context.Context, projectID int, projectName string, mr *gitlab.MergeRequest) (err error) {
|
||
var (
|
||
milestoneID int
|
||
mrLables string
|
||
mrChanges string
|
||
mrLablesByte []byte
|
||
mrChangesByte []byte
|
||
)
|
||
|
||
if mr.Milestone != nil {
|
||
milestoneID = mr.Milestone.ID
|
||
}
|
||
if mrLablesByte, err = json.Marshal(mr.Labels); err != nil {
|
||
mrLables = model.JsonMarshalErrorText
|
||
} else {
|
||
mrLables = string(mrLablesByte)
|
||
}
|
||
|
||
if mrChangesByte, err = json.Marshal(mr.Changes); err != nil {
|
||
mrChanges = model.JsonMarshalErrorText
|
||
} else {
|
||
mrChanges = string(mrChangesByte)
|
||
}
|
||
|
||
mrDB := &model.StatisticsMrs{
|
||
MRID: mr.ID,
|
||
MRIID: mr.IID,
|
||
TargetBranch: mr.TargetBranch,
|
||
SourceBranch: mr.SourceBranch,
|
||
ProjectID: mr.ProjectID,
|
||
ProjectName: projectName,
|
||
Title: mr.Title,
|
||
State: mr.State,
|
||
CreatedAt: mr.CreatedAt,
|
||
UpdatedAt: mr.UpdatedAt,
|
||
Upvotes: mr.Upvotes,
|
||
Downvotes: mr.Downvotes,
|
||
AuthorID: mr.Author.ID,
|
||
AuthorName: mr.Author.Name,
|
||
AssigneeID: mr.Assignee.ID,
|
||
AssigneeName: mr.Assignee.Name,
|
||
SourceProjectID: mr.SourceProjectID,
|
||
TargetProjectID: mr.TargetProjectID,
|
||
Labels: mrLables,
|
||
Description: mr.Description,
|
||
WorkInProgress: mr.WorkInProgress,
|
||
MilestoneID: milestoneID,
|
||
MergeWhenPipelineSucceeds: mr.MergeWhenPipelineSucceeds,
|
||
MergeStatus: mr.MergeStatus,
|
||
MergedByID: mr.MergedBy.ID,
|
||
MergedByName: mr.MergedBy.Name,
|
||
MergedAt: mr.MergedAt,
|
||
ClosedByID: mr.ClosedBy.ID,
|
||
ClosedAt: mr.ClosedAt,
|
||
Subscribed: mr.Subscribed,
|
||
SHA: mr.SHA,
|
||
MergeCommitSHA: mr.MergeCommitSHA,
|
||
UserNotesCount: mr.UserNotesCount,
|
||
ChangesCount: mr.ChangesCount,
|
||
ShouldRemoveSourceBranch: mr.ShouldRemoveSourceBranch,
|
||
ForceRemoveSourceBranch: mr.ForceRemoveSourceBranch,
|
||
WebURL: mr.WebURL,
|
||
DiscussionLocked: mr.DiscussionLocked,
|
||
Changes: mrChanges,
|
||
TimeStatsHumanTimeEstimate: mr.TimeStats.HumanTimeEstimate,
|
||
TimeStatsHumanTotalTimeSpent: mr.TimeStats.HumanTotalTimeSpent,
|
||
TimeStatsTimeEstimate: mr.TimeStats.TimeEstimate,
|
||
TimeStatsTotalTimeSpent: mr.TimeStats.TotalTimeSpent,
|
||
Squash: mr.Squash,
|
||
PipelineID: mr.Pipeline.ID,
|
||
}
|
||
if len(mrDB.Labels) > model.MessageMaxLen {
|
||
mrDB.Labels = mrDB.Labels[0 : model.MessageMaxLen-1]
|
||
}
|
||
if len(mrDB.Description) > model.MessageMaxLen {
|
||
mrDB.Description = mrDB.Description[0 : model.MessageMaxLen-1]
|
||
}
|
||
return s.SaveDatabaseMR(c, mrDB)
|
||
}
|
||
|
||
// SaveDatabaseMR ...
|
||
func (s *Service) SaveDatabaseMR(c context.Context, mrDB *model.StatisticsMrs) (err error) {
|
||
var total int
|
||
|
||
if total, err = s.dao.HasMR(c, mrDB.ProjectID, mrDB.MRIID); err != nil {
|
||
log.Error("SaveDatabaseMR HasMR(%+v)", err)
|
||
return
|
||
}
|
||
|
||
// found only one, so update
|
||
if total == 1 {
|
||
return s.dao.UpdateMR(c, mrDB.ProjectID, mrDB.MRIID, mrDB)
|
||
} else if total > 1 {
|
||
// found repeated row, this situation will not exist under normal
|
||
log.Warn("SaveDatabaseMR mr has more rows(%d)", total)
|
||
return
|
||
}
|
||
|
||
// insert row now
|
||
return s.dao.CreateMR(c, mrDB)
|
||
}
|
||
|
||
/*-------------------------------------- agg MR ----------------------------------------*/
|
||
|
||
// AggregateProjectMR ...
|
||
func (s *Service) AggregateProjectMR(c context.Context, projectID int) (err error) {
|
||
var (
|
||
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
syncAllTime = false
|
||
mrs []*model.StatisticsMrs
|
||
projectInfo *model.ProjectInfo
|
||
since *time.Time
|
||
until *time.Time
|
||
)
|
||
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
return
|
||
}
|
||
|
||
if !syncAllTime {
|
||
since, until = utils.CalSyncTime()
|
||
}
|
||
|
||
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, mr := range mrs {
|
||
if err = s.MRAddedInfoDB(c, projectID, mr); err != nil {
|
||
log.Error("MRAddedInfoDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
|
||
err = nil
|
||
}
|
||
if err = s.MRReviewerDB(c, projectID, mr.MRIID, projectInfo.Name, mr); err != nil {
|
||
log.Error("MRReviewerDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID)
|
||
err = nil
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// MRAddedInfoDB ...
|
||
func (s *Service) MRAddedInfoDB(c context.Context, projectID int, mr *model.StatisticsMrs) (err error) {
|
||
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projectID, mr.MRIID); err != nil {
|
||
return
|
||
}
|
||
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projectID, mr.MRIID); err != nil {
|
||
return
|
||
}
|
||
return s.dao.UpdateMR(c, projectID, mr.MRIID, mr)
|
||
}
|
||
|
||
// MRReviewerDB 查询mr reviewer信息
|
||
func (s *Service) MRReviewerDB(c context.Context, projectID, mrIID int, projectName string, mr *model.StatisticsMrs) (err error) {
|
||
var (
|
||
notes []*model.StatisticsNotes
|
||
emojis []*model.StatisticsMRAwardEmojis
|
||
owners []string
|
||
r *regexp.Regexp
|
||
reviewers []*model.AggregateMrReviewer
|
||
)
|
||
|
||
//query note
|
||
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil {
|
||
return
|
||
}
|
||
|
||
// query emoji
|
||
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil {
|
||
return
|
||
}
|
||
|
||
//评论中解析获取owner
|
||
if len(notes) == 0 {
|
||
return
|
||
}
|
||
if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil {
|
||
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body)
|
||
if len(matchResult) > 0 {
|
||
owners = strings.Split(matchResult[1], " 或 @")
|
||
}
|
||
}
|
||
|
||
mrCreatedAt := notes[len(notes)-1].CreatedAt
|
||
|
||
// 从评论和表情中解析 reviewers
|
||
for _, note := range notes {
|
||
if note.Body == "+1" {
|
||
reviewer := &model.AggregateMrReviewer{
|
||
ReviewerID: note.AuthorID,
|
||
ReviewerName: note.AuthorName,
|
||
ReviewType: "note",
|
||
ReviewID: note.NoteID,
|
||
ReviewCommand: "+1",
|
||
CreatedAt: note.CreatedAt,
|
||
ApproveTime: int(note.CreatedAt.Sub(*mrCreatedAt).Seconds()),
|
||
MergeTime: int(mr.UpdatedAt.Sub(*note.CreatedAt).Seconds()),
|
||
}
|
||
reviewers = append(reviewers, reviewer)
|
||
}
|
||
}
|
||
for _, emoji := range emojis {
|
||
if emoji.Name == "thumbsup" {
|
||
reviewer := &model.AggregateMrReviewer{
|
||
ReviewerID: emoji.UserID,
|
||
ReviewerName: emoji.UserName,
|
||
ReviewType: "emoji",
|
||
ReviewID: emoji.AwardEmojiID,
|
||
ReviewCommand: "thumbsup",
|
||
CreatedAt: emoji.CreatedAt,
|
||
ApproveTime: int(emoji.CreatedAt.Sub(*mrCreatedAt).Seconds()),
|
||
MergeTime: int(mr.UpdatedAt.Sub(*emoji.CreatedAt).Seconds()),
|
||
}
|
||
reviewers = append(reviewers, reviewer)
|
||
}
|
||
}
|
||
|
||
// 判断reviewer类型
|
||
for _, r := range reviewers {
|
||
r.ProjectID = projectID
|
||
r.ProjectName = projectName
|
||
r.MrIID = mrIID
|
||
r.Title = mr.Title
|
||
r.WebUrl = mr.WebURL
|
||
r.AuthorName = mr.AuthorName
|
||
if utils.InSlice(r.ReviewerName, owners) {
|
||
r.UserType = "owner"
|
||
} else {
|
||
r.UserType = "other"
|
||
}
|
||
if err = s.SaveDatabaseAggMR(c, r); err != nil {
|
||
log.Error("mrReviewer 存数据库报错(%+V)", err)
|
||
continue
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// SaveDatabaseAggMR ...
|
||
func (s *Service) SaveDatabaseAggMR(c context.Context, mrDB *model.AggregateMrReviewer) (err error) {
|
||
var total int
|
||
|
||
if total, err = s.dao.HasAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID); err != nil {
|
||
log.Error("SaveDatabaseAggMR HasAggMR(%+v)", err)
|
||
return
|
||
}
|
||
|
||
// found only one, so update
|
||
if total == 1 {
|
||
return s.dao.UpdateAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID, mrDB)
|
||
} else if total > 1 {
|
||
// found repeated row, this situation will not exist under normal
|
||
log.Warn("SaveDatabaseAggMR aggMR has more rows(%d)", total)
|
||
return
|
||
}
|
||
|
||
// insert row now
|
||
return s.dao.CreateAggregateReviewer(c, mrDB)
|
||
}
|