486 lines
14 KiB
Go
486 lines
14 KiB
Go
|
package service
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"go-common/app/admin/ep/saga/model"
|
||
|
|
"go-common/app/admin/ep/saga/service/utils"
|
||
|
|
"go-common/library/log"
|
||
|
|
|
||
|
|
"github.com/xanzy/go-gitlab"
|
||
|
|
)
|
||
|
|
|
||
|
|
/*-------------------------------------- sync issue ----------------------------------------*/
|
||
|
|
|
||
|
|
// SyncAllIssues ...
|
||
|
|
func (s *Service) SyncAllIssues(projectID int) (totalPage, totalNum int, err error) {
|
||
|
|
var (
|
||
|
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
|
|
syncAllTime = false
|
||
|
|
issues []*gitlab.Issue
|
||
|
|
resp *gitlab.Response
|
||
|
|
since *time.Time
|
||
|
|
until *time.Time
|
||
|
|
)
|
||
|
|
if !syncAllTime {
|
||
|
|
since, until = utils.CalSyncTime()
|
||
|
|
}
|
||
|
|
|
||
|
|
for page := 1; ; page++ {
|
||
|
|
totalPage++
|
||
|
|
if issues, resp, err = s.gitlab.ListProjectIssues(projectID, page, since, until); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, issue := range issues {
|
||
|
|
var (
|
||
|
|
issueAssignees string
|
||
|
|
issueLabels string
|
||
|
|
issueTimeStats string
|
||
|
|
milestoneID int
|
||
|
|
)
|
||
|
|
if issue.Milestone != nil {
|
||
|
|
milestoneID = issue.Milestone.ID
|
||
|
|
}
|
||
|
|
issueAssigneesByte, _ := json.Marshal(issue.Assignees)
|
||
|
|
issueAssignees = string(issueAssigneesByte)
|
||
|
|
issueLabelsByte, _ := json.Marshal(issue.Labels)
|
||
|
|
issueLabels = string(issueLabelsByte)
|
||
|
|
issueTimeStatsByte, _ := json.Marshal(issue.TimeStats)
|
||
|
|
issueTimeStats = string(issueTimeStatsByte)
|
||
|
|
issueDB := &model.StatisticsIssues{
|
||
|
|
ProjectID: projectID,
|
||
|
|
IssueID: issue.ID,
|
||
|
|
IssueIID: issue.IID,
|
||
|
|
MilestoneID: milestoneID,
|
||
|
|
Description: issue.Description,
|
||
|
|
State: issue.State,
|
||
|
|
Assignees: issueAssignees,
|
||
|
|
Upvotes: issue.Upvotes,
|
||
|
|
Downvotes: issue.Downvotes,
|
||
|
|
Labels: issueLabels,
|
||
|
|
Title: issue.Title,
|
||
|
|
UpdatedAt: issue.UpdatedAt,
|
||
|
|
CreatedAt: issue.CreatedAt,
|
||
|
|
ClosedAt: issue.ClosedAt,
|
||
|
|
Subscribed: issue.Subscribed,
|
||
|
|
UserNotesCount: issue.UserNotesCount,
|
||
|
|
DueDate: issue.DueDate,
|
||
|
|
WebURL: issue.WebURL,
|
||
|
|
TimeStats: issueTimeStats,
|
||
|
|
Confidential: issue.Confidential,
|
||
|
|
Weight: issue.Weight,
|
||
|
|
DiscussionLocked: issue.DiscussionLocked,
|
||
|
|
IssueLinkID: issue.IssueLinkID,
|
||
|
|
}
|
||
|
|
|
||
|
|
if issue.Author != nil {
|
||
|
|
issueDB.AuthorID = issue.Author.ID
|
||
|
|
issueDB.AuthorName = issue.Author.Name
|
||
|
|
}
|
||
|
|
if issue.Assignee != nil {
|
||
|
|
issueDB.AssigneeID = issue.Assignee.ID
|
||
|
|
issueDB.AssigneeName = issue.Assignee.Name
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(issueDB.Description) > model.MessageMaxLen {
|
||
|
|
issueDB.Description = issueDB.Description[0 : model.MessageMaxLen-1]
|
||
|
|
}
|
||
|
|
|
||
|
|
if err = s.SaveDatabaseIssue(issueDB); err != nil {
|
||
|
|
log.Error("issue Save Database err: projectID(%d), IssueID(%d)", projectID, issue.ID)
|
||
|
|
err = nil
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
totalNum++
|
||
|
|
}
|
||
|
|
|
||
|
|
if resp.NextPage == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// SaveDatabaseIssue ...
|
||
|
|
func (s *Service) SaveDatabaseIssue(issueDB *model.StatisticsIssues) (err error) {
|
||
|
|
var total int
|
||
|
|
|
||
|
|
if total, err = s.dao.HasIssue(issueDB.ProjectID, issueDB.IssueID); err != nil {
|
||
|
|
log.Error("SaveDatabaseIssue HasIssue(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// update found row
|
||
|
|
if total == 1 {
|
||
|
|
if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
|
||
|
|
if strings.Contains(err.Error(), model.DatabaseErrorText) {
|
||
|
|
issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
|
||
|
|
issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
|
||
|
|
issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
|
||
|
|
issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
|
||
|
|
}
|
||
|
|
if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseIssue UpdateIssue(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
} else if total > 1 {
|
||
|
|
// found repeated row, this situation will not exist under normal
|
||
|
|
log.Warn("SaveDatabaseIssue issue has more rows(%d)", total)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert row now
|
||
|
|
if err = s.dao.CreateIssue(issueDB); err != nil {
|
||
|
|
if strings.Contains(err.Error(), model.DatabaseErrorText) {
|
||
|
|
issueDB.Title = strconv.QuoteToASCII(issueDB.Title)
|
||
|
|
issueDB.Description = strconv.QuoteToASCII(issueDB.Description)
|
||
|
|
issueDB.Title = utils.Unicode2Chinese(issueDB.Title)
|
||
|
|
issueDB.Description = utils.Unicode2Chinese(issueDB.Description)
|
||
|
|
}
|
||
|
|
if err = s.dao.CreateIssue(issueDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseIssue CreateIssue(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
/*-------------------------------------- sync note ----------------------------------------*/
|
||
|
|
|
||
|
|
// SyncProjectNotes ...
|
||
|
|
func (s *Service) SyncProjectNotes(c context.Context, projectID int) (totalPage, totalNum int, err error) {
|
||
|
|
var (
|
||
|
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
|
|
syncAllTime = false
|
||
|
|
resp *gitlab.Response
|
||
|
|
projectInfo *model.ProjectInfo
|
||
|
|
mrs []*model.StatisticsMrs
|
||
|
|
notes []*gitlab.Note
|
||
|
|
since *time.Time
|
||
|
|
until *time.Time
|
||
|
|
)
|
||
|
|
|
||
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if !syncAllTime {
|
||
|
|
since, until = utils.CalSyncTime()
|
||
|
|
}
|
||
|
|
|
||
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, mr := range mrs {
|
||
|
|
for page := 1; ; page++ {
|
||
|
|
totalPage++
|
||
|
|
if notes, resp, err = s.gitlab.ListMRNotes(c, projectID, mr.MRIID, page); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, note := range notes {
|
||
|
|
var (
|
||
|
|
notePosition string
|
||
|
|
notePositionByte []byte
|
||
|
|
)
|
||
|
|
if notePositionByte, err = json.Marshal(note.Position); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
notePosition = string(notePositionByte)
|
||
|
|
noteDB := &model.StatisticsNotes{
|
||
|
|
ProjectID: projectID,
|
||
|
|
ProjectName: projectInfo.Name,
|
||
|
|
MrIID: mr.MRIID,
|
||
|
|
NoteID: note.ID,
|
||
|
|
Body: note.Body,
|
||
|
|
Attachment: note.Attachment,
|
||
|
|
Title: note.Title,
|
||
|
|
FileName: note.FileName,
|
||
|
|
AuthorID: note.Author.ID,
|
||
|
|
AuthorName: note.Author.Name,
|
||
|
|
System: note.System,
|
||
|
|
ExpiresAt: note.ExpiresAt,
|
||
|
|
UpdatedAt: note.UpdatedAt,
|
||
|
|
CreatedAt: note.CreatedAt,
|
||
|
|
NoteableID: note.NoteableID,
|
||
|
|
NoteableType: note.NoteableType,
|
||
|
|
Position: notePosition,
|
||
|
|
Resolvable: note.Resolvable,
|
||
|
|
Resolved: note.Resolved,
|
||
|
|
ResolvedByID: note.ResolvedBy.ID,
|
||
|
|
ResolvedByName: note.ResolvedBy.Username,
|
||
|
|
NoteableIID: note.NoteableIID,
|
||
|
|
}
|
||
|
|
if err = s.SaveDatabaseNote(c, noteDB); err != nil {
|
||
|
|
log.Error("note Save Database err: projectID(%d), NoteID(%d)", projectID, note.ID)
|
||
|
|
err = nil
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
totalNum++
|
||
|
|
}
|
||
|
|
if resp.NextPage == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// HandleNoteDBError ...
|
||
|
|
func (s *Service) HandleNoteDBError(c context.Context, noteDB *model.StatisticsNotes, errText string) (note *model.StatisticsNotes) {
|
||
|
|
if strings.Contains(errText, model.DatabaseErrorText) {
|
||
|
|
noteDB.Title = strconv.QuoteToASCII(noteDB.Title)
|
||
|
|
noteDB.Body = strconv.QuoteToASCII(noteDB.Body)
|
||
|
|
noteDB.Title = utils.Unicode2Chinese(noteDB.Title)
|
||
|
|
noteDB.Body = utils.Unicode2Chinese(noteDB.Body)
|
||
|
|
if len(noteDB.Body) > model.MessageMaxLen {
|
||
|
|
noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
|
||
|
|
}
|
||
|
|
} else if strings.Contains(errText, model.DatabaseMaxLenthErrorText) {
|
||
|
|
noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1]
|
||
|
|
}
|
||
|
|
return noteDB
|
||
|
|
}
|
||
|
|
|
||
|
|
// SaveDatabaseNote ...
|
||
|
|
func (s *Service) SaveDatabaseNote(c context.Context, noteDB *model.StatisticsNotes) (err error) {
|
||
|
|
var total int
|
||
|
|
|
||
|
|
if total, err = s.dao.HasNote(c, noteDB.ProjectID, noteDB.NoteID); err != nil {
|
||
|
|
log.Error("SaveDatabaseNote HasNote(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// update found row
|
||
|
|
if total == 1 {
|
||
|
|
if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
|
||
|
|
noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
|
||
|
|
if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseNote UpdateNote(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
} else if total > 1 {
|
||
|
|
// found repeated row, this situation will not exist under normal
|
||
|
|
log.Warn("SaveDatabaseNote Note has more rows(%d)", total)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert row now
|
||
|
|
if err = s.dao.CreateNote(c, noteDB); err != nil {
|
||
|
|
noteDB = s.HandleNoteDBError(c, noteDB, err.Error())
|
||
|
|
if err = s.dao.CreateNote(c, noteDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseNote CreateNote(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
/*-------------------------------------- sync emoji ----------------------------------------*/
|
||
|
|
|
||
|
|
// SyncProjectAwardEmoji ...
|
||
|
|
func (s *Service) SyncProjectAwardEmoji(c context.Context, projectID int) (totalPage, totalNum int, err error) {
|
||
|
|
|
||
|
|
var (
|
||
|
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
|
|
syncAllTime = false
|
||
|
|
resp *gitlab.Response
|
||
|
|
projectInfo *model.ProjectInfo
|
||
|
|
mrs []*model.StatisticsMrs
|
||
|
|
awardEmojis []*gitlab.AwardEmoji
|
||
|
|
since *time.Time
|
||
|
|
until *time.Time
|
||
|
|
)
|
||
|
|
|
||
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if !syncAllTime {
|
||
|
|
since, until = utils.CalSyncTime()
|
||
|
|
}
|
||
|
|
|
||
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, mr := range mrs {
|
||
|
|
for page := 1; ; page++ {
|
||
|
|
totalPage++
|
||
|
|
if awardEmojis, resp, err = s.gitlab.ListMRAwardEmoji(projectID, mr.MRIID, page); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, awardEmoji := range awardEmojis {
|
||
|
|
awardEmojiDB := &model.StatisticsMRAwardEmojis{
|
||
|
|
ProjectID: projectID,
|
||
|
|
ProjectName: projectInfo.Name,
|
||
|
|
MrIID: mr.MRIID,
|
||
|
|
AwardEmojiID: awardEmoji.ID,
|
||
|
|
Name: awardEmoji.Name,
|
||
|
|
UserID: awardEmoji.User.ID,
|
||
|
|
UserName: awardEmoji.User.Name,
|
||
|
|
CreatedAt: awardEmoji.CreatedAt,
|
||
|
|
UpdatedAt: awardEmoji.UpdatedAt,
|
||
|
|
AwardableID: awardEmoji.AwardableID,
|
||
|
|
AwardableType: awardEmoji.AwardableType,
|
||
|
|
}
|
||
|
|
if err = s.SaveDatabaseAwardEmoji(c, awardEmojiDB); err != nil {
|
||
|
|
log.Error("awardEmoji Save Database err: projectID(%d), AwardEmojiID(%d)", projectID, awardEmoji.ID)
|
||
|
|
err = nil
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
totalNum++
|
||
|
|
}
|
||
|
|
if resp.NextPage == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// SaveDatabaseAwardEmoji ...
|
||
|
|
func (s *Service) SaveDatabaseAwardEmoji(c context.Context, awardEmojiDB *model.StatisticsMRAwardEmojis) (err error) {
|
||
|
|
var total int
|
||
|
|
|
||
|
|
if total, err = s.dao.HasMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID); err != nil {
|
||
|
|
log.Error("SaveDatabaseAwardEmoji HasAwardEmoji(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// update found row
|
||
|
|
if total == 1 {
|
||
|
|
if err = s.dao.UpdateMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID, awardEmojiDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseAwardEmoji UpdateAwardEmoji(%+v)", err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
} else if total > 1 {
|
||
|
|
// found repeated row, this situation will not exist under normal
|
||
|
|
log.Warn("SaveDatabaseAwardEmoji Note has more rows(%d)", total)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert row now
|
||
|
|
if err = s.dao.CreateMRAwardEmoji(c, awardEmojiDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseAwardEmoji CreateAwardEmoji(%+v)", err)
|
||
|
|
return
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
/*-------------------------------------- sync Discussion ----------------------------------------*/
|
||
|
|
|
||
|
|
// SyncProjectDiscussion ...
|
||
|
|
func (s *Service) SyncProjectDiscussion(c context.Context, projectID int) (totalPage, totalNum int, err error) {
|
||
|
|
|
||
|
|
var (
|
||
|
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
|
|
syncAllTime = false
|
||
|
|
resp *gitlab.Response
|
||
|
|
projectInfo *model.ProjectInfo
|
||
|
|
mrs []*model.StatisticsMrs
|
||
|
|
discussions []*gitlab.Discussion
|
||
|
|
since *time.Time
|
||
|
|
until *time.Time
|
||
|
|
)
|
||
|
|
|
||
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
if !syncAllTime {
|
||
|
|
since, until = utils.CalSyncTime()
|
||
|
|
}
|
||
|
|
|
||
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, mr := range mrs {
|
||
|
|
for page := 1; ; page++ {
|
||
|
|
totalPage++
|
||
|
|
if discussions, resp, err = s.gitlab.ListMRDiscussions(projectID, mr.MRIID, page); err != nil {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
for _, discussion := range discussions {
|
||
|
|
var (
|
||
|
|
notesArray []int
|
||
|
|
notesByte []byte
|
||
|
|
notes string
|
||
|
|
)
|
||
|
|
if discussion.Notes != nil {
|
||
|
|
for _, note := range discussion.Notes {
|
||
|
|
notesArray = append(notesArray, note.ID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if notesByte, err = json.Marshal(notesArray); err != nil {
|
||
|
|
notes = model.JsonMarshalErrorText
|
||
|
|
} else {
|
||
|
|
notes = string(notesByte)
|
||
|
|
}
|
||
|
|
discussionDB := &model.StatisticsDiscussions{
|
||
|
|
ProjectID: projectID,
|
||
|
|
ProjectName: projectInfo.Name,
|
||
|
|
MrIID: mr.MRIID,
|
||
|
|
DiscussionID: discussion.ID,
|
||
|
|
IndividualNote: discussion.IndividualNote,
|
||
|
|
Notes: notes,
|
||
|
|
}
|
||
|
|
if err = s.SaveDatabaseDiscussion(c, discussionDB); err != nil {
|
||
|
|
log.Error("discussion Save Database err: projectID(%d), DiscussionID(%s)", projectID, discussion.ID)
|
||
|
|
err = nil
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
totalNum++
|
||
|
|
}
|
||
|
|
if resp.NextPage == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// SaveDatabaseDiscussion ...
|
||
|
|
func (s *Service) SaveDatabaseDiscussion(c context.Context, discussionDB *model.StatisticsDiscussions) (err error) {
|
||
|
|
var total int
|
||
|
|
|
||
|
|
if total, err = s.dao.HasDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID); err != nil {
|
||
|
|
log.Error("SaveDatabaseDiscussion HasDiscussion(%+v)", err)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// update found row
|
||
|
|
if total == 1 {
|
||
|
|
if err = s.dao.UpdateDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID, discussionDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseDiscussion UpdateDiscussion(%+v)", err)
|
||
|
|
}
|
||
|
|
return
|
||
|
|
} else if total > 1 {
|
||
|
|
// found repeated row, this situation will not exist under normal
|
||
|
|
log.Warn("SaveDatabaseDiscussion Note has more rows(%d)", total)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// insert row now
|
||
|
|
if err = s.dao.CreateDiscussion(c, discussionDB); err != nil {
|
||
|
|
log.Error("SaveDatabaseDiscussion CreateDiscussion(%+v)", err)
|
||
|
|
return
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
return
|
||
|
|
}
|