839 lines
24 KiB
Go
839 lines
24 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
"time"
|
||
|
||
"go-common/app/admin/ep/saga/conf"
|
||
"go-common/app/admin/ep/saga/model"
|
||
"go-common/app/admin/ep/saga/service/utils"
|
||
"go-common/app/admin/ep/saga/service/wechat"
|
||
"go-common/library/cache/memcache"
|
||
"go-common/library/log"
|
||
|
||
"github.com/xanzy/go-gitlab"
|
||
)
|
||
|
||
const (
|
||
_gitHTTP = "http://git.bilibili.co/"
|
||
_gitSSH = "git@git-test.bilibili.co:"
|
||
_gitSSHTail = ".git"
|
||
_manualJob = "manual"
|
||
_androidScheduleJob = "daily branch check"
|
||
_iosScheduleJob = "daily:on-schedule"
|
||
)
|
||
|
||
// QueryTeamPipeline query pipeline info according to team.
|
||
func (s *Service) QueryTeamPipeline(c context.Context, req *model.TeamDataRequest) (resp *model.PipelineDataResp, err error) {
|
||
var (
|
||
projectInfo []*model.ProjectInfo
|
||
reqProject = &model.ProjectInfoRequest{}
|
||
|
||
data []*model.PipelineDataTime
|
||
queryDes string
|
||
total int
|
||
succNum int
|
||
key string
|
||
keyNotExist bool
|
||
)
|
||
|
||
if len(req.Department) <= 0 && len(req.Business) <= 0 {
|
||
log.Warn("query department and business are empty!")
|
||
return
|
||
}
|
||
|
||
//get pipeline info from mc
|
||
key = "saga_admin_" + req.Department + "_" + req.Business + "_" + model.KeyTypeConst[3]
|
||
if resp, err = s.dao.GetPipeline(c, key); err != nil {
|
||
if err == memcache.ErrNotFound {
|
||
keyNotExist = true
|
||
} else {
|
||
return
|
||
}
|
||
} else {
|
||
return
|
||
}
|
||
|
||
log.Info("sync team pipeline start => type= %d, Department= %s, Business= %s", req.QueryType, req.Department, req.Business)
|
||
|
||
//query team projects
|
||
reqProject.Department = req.Department
|
||
reqProject.Business = req.Business
|
||
if _, projectInfo, err = s.dao.QueryProjectInfo(false, reqProject); err != nil {
|
||
return
|
||
}
|
||
if len(projectInfo) <= 0 {
|
||
log.Warn("Found no project!")
|
||
return
|
||
}
|
||
|
||
if data, total, succNum, err = s.QueryTeamPipelineByTime(projectInfo, model.LastWeekPerDay); err != nil {
|
||
return
|
||
}
|
||
|
||
successScale := succNum * 100 / total
|
||
queryDes = req.Department + " " + req.Business + " " + "pipeline上一周每天数量"
|
||
resp = &model.PipelineDataResp{
|
||
Department: req.Department,
|
||
Business: req.Business,
|
||
QueryDes: queryDes,
|
||
Total: total,
|
||
SuccessNum: succNum,
|
||
SuccessScale: successScale,
|
||
Data: data,
|
||
}
|
||
|
||
//set pipeline info to mc
|
||
if keyNotExist {
|
||
if err = s.dao.SetPipeline(c, key, resp); err != nil {
|
||
return
|
||
}
|
||
}
|
||
|
||
log.Info("sync team pipeline end")
|
||
return
|
||
}
|
||
|
||
// QueryTeamPipelineByTime ...
|
||
func (s *Service) QueryTeamPipelineByTime(projectInfo []*model.ProjectInfo, queryType int) (resp []*model.PipelineDataTime, allNum, succNum int, err error) {
|
||
var (
|
||
layout = "2006-01-02"
|
||
since time.Time
|
||
until time.Time
|
||
|
||
total int
|
||
success int
|
||
count int
|
||
)
|
||
|
||
if queryType == model.LastWeekPerDay {
|
||
count = model.DayNumPerWeek
|
||
} else {
|
||
log.Warn("Query Type is not in range!")
|
||
return
|
||
}
|
||
|
||
year, month, day := time.Now().Date()
|
||
weekDay := (int)(time.Now().Weekday())
|
||
today := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
|
||
|
||
for i := 0; i < count; i++ {
|
||
since = today.AddDate(0, 0, -weekDay-i)
|
||
until = today.AddDate(0, 0, -weekDay-i+1)
|
||
|
||
totalAll := 0
|
||
successAll := 0
|
||
//log.Info("== start query from: %v, to: %v", since, until)
|
||
for _, project := range projectInfo {
|
||
if total, success, err = s.QueryProjectPipeline(project.ProjectID, "success", since, until); err != nil {
|
||
return
|
||
}
|
||
totalAll = totalAll + total
|
||
successAll = successAll + success
|
||
}
|
||
|
||
perData := &model.PipelineDataTime{
|
||
TotalItem: totalAll,
|
||
SuccessItem: successAll,
|
||
StartTime: since.Format(layout),
|
||
EndTime: until.Format(layout),
|
||
}
|
||
resp = append(resp, perData)
|
||
allNum = allNum + totalAll
|
||
succNum = succNum + successAll
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryProjectPipeline query pipeline info according to project id.
|
||
func (s *Service) QueryProjectPipeline(projectID int, state string, since, until time.Time) (totalNum, stateNum int, err error) {
|
||
var (
|
||
pipelineList gitlab.PipelineList
|
||
pipeline *gitlab.Pipeline
|
||
resp *gitlab.Response
|
||
startQuery bool
|
||
)
|
||
|
||
if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
page := 1
|
||
for page <= resp.TotalPages {
|
||
|
||
if !startQuery {
|
||
if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
if page == 1 && len(pipelineList) <= 0 {
|
||
return
|
||
}
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelineList[0].ID); err != nil {
|
||
return
|
||
}
|
||
|
||
if pipeline.CreatedAt.After(until) {
|
||
page++
|
||
continue
|
||
} else {
|
||
startQuery = true
|
||
page--
|
||
continue
|
||
}
|
||
}
|
||
|
||
if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, v := range pipelineList {
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
|
||
return
|
||
}
|
||
|
||
createTime := pipeline.CreatedAt
|
||
//year, month, day := createTime.Date()
|
||
//log.Info("index: %d createTime: %d, month: %d, day: %d", k, year, month, day)
|
||
|
||
if createTime.After(since) && createTime.Before(until) {
|
||
totalNum = totalNum + 1
|
||
if pipeline.Status == state {
|
||
stateNum = stateNum + 1
|
||
}
|
||
}
|
||
|
||
if createTime.Before(since) {
|
||
return
|
||
}
|
||
}
|
||
page++
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryProjectPipelineNew ...
|
||
func (s *Service) QueryProjectPipelineNew(c context.Context, req *model.PipelineDataReq) (resp *model.PipelineDataAvgResp, err error) {
|
||
var (
|
||
data []*model.PipelineDataAvg
|
||
queryDes string
|
||
total int
|
||
totalStatus int
|
||
avgDurationTime float64
|
||
avgPendingTime float64
|
||
avgRunningTime float64
|
||
)
|
||
|
||
log.Info("QuerySingleProjectData Type: %d", req.Type)
|
||
switch req.Type {
|
||
case model.LastYearPerMonth:
|
||
queryDes = model.LastYearPerMonthNote
|
||
case model.LastMonthPerDay:
|
||
queryDes = model.LastMonthPerDayNote
|
||
case model.LastYearPerDay:
|
||
queryDes = model.LastYearPerDayNote
|
||
default:
|
||
log.Warn("QueryProjectCommit Type is not in range")
|
||
return
|
||
}
|
||
queryDes = req.ProjectName + " pipeline " + req.State + " " + queryDes
|
||
|
||
if data, total, totalStatus, avgDurationTime, avgPendingTime, avgRunningTime, err = s.QueryProjectByTimeNew(c, req, req.Type); err != nil {
|
||
return
|
||
}
|
||
|
||
resp = &model.PipelineDataAvgResp{
|
||
ProjectName: req.ProjectName,
|
||
QueryDes: queryDes,
|
||
Status: req.State,
|
||
Total: total,
|
||
TotalStatus: totalStatus,
|
||
AvgDurationTime: avgDurationTime,
|
||
AvgPendingTime: avgPendingTime,
|
||
AvgRunningTime: avgRunningTime,
|
||
Data: data,
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// QueryProjectByTimeNew ...
|
||
func (s *Service) QueryProjectByTimeNew(c context.Context, req *model.PipelineDataReq, queryType int) (resp []*model.PipelineDataAvg, allNum, allStatusNum int, avgDurationTime, avgPendingTime, avgRunningTime float64, err error) {
|
||
var (
|
||
layout = "2006-01-02"
|
||
since time.Time
|
||
until time.Time
|
||
count int
|
||
pendingTimeListAll []float64
|
||
runningTimeListAll []float64
|
||
durationTimeListAll []float64
|
||
pipelineTime *model.PipelineTime
|
||
perData *model.PipelineDataAvg
|
||
avgTotalTime float64
|
||
totalNum int
|
||
statusNum int
|
||
)
|
||
|
||
year, month, day := time.Now().Date()
|
||
thisMonth := time.Date(year, month, day, 0, 0, 0, 0, time.Local)
|
||
|
||
if queryType == model.LastYearPerMonth {
|
||
count = model.MonthNumPerYear
|
||
} else if queryType == model.LastMonthPerDay {
|
||
//_, _, count = thisMonth.AddDate(0, 0, -1).Date()
|
||
count = model.DayNumPerMonth
|
||
} else if queryType == model.LastYearPerDay {
|
||
count = model.DayNumPerYear
|
||
}
|
||
|
||
for i := count; i >= 1; i-- {
|
||
|
||
if queryType == model.LastYearPerMonth {
|
||
since = thisMonth.AddDate(0, -i, 0)
|
||
until = thisMonth.AddDate(0, -i+1, 0)
|
||
} else if queryType == model.LastMonthPerDay {
|
||
since = thisMonth.AddDate(0, 0, -i)
|
||
until = thisMonth.AddDate(0, 0, -i+1)
|
||
} else if queryType == model.LastYearPerDay {
|
||
since = thisMonth.AddDate(0, 0, -i)
|
||
until = thisMonth.AddDate(0, 0, -i+1)
|
||
}
|
||
|
||
/*if totalNum, statusNum, pipelineTime, err = s.QueryProjectPipelines(c, req, since, until); err != nil {
|
||
log.Error("QueryProjectPipelines err：%+v", err)
|
||
return
|
||
}*/
|
||
if totalNum, statusNum, pipelineTime, err = s.QueryPipelinesFromDB(c, req, since, until); err != nil {
|
||
log.Error("QueryPipelinesFromDB err：%+v", err)
|
||
return
|
||
}
|
||
|
||
avgTotalTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.DurationList)
|
||
avgPendingTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.PendingList)
|
||
avgRunningTime = utils.CalAverageTime(req.StatisticsType, pipelineTime.RunningList)
|
||
|
||
perData = &model.PipelineDataAvg{
|
||
TotalItem: totalNum,
|
||
TotalStatusItem: statusNum,
|
||
AvgDurationTime: avgTotalTime,
|
||
AvgPendingTime: avgPendingTime,
|
||
AvgRunningTime: avgRunningTime,
|
||
MaxDurationTime: pipelineTime.DurationMax,
|
||
MinDurationTime: pipelineTime.DurationMin,
|
||
MaxPendingTime: pipelineTime.PendingMax,
|
||
MinPendingTime: pipelineTime.PendingMin,
|
||
MaxRunningTime: pipelineTime.RunningMax,
|
||
MinRunningTime: pipelineTime.RunningMin,
|
||
StartTime: since.Format(layout),
|
||
EndTime: until.Format(layout),
|
||
}
|
||
resp = append(resp, perData)
|
||
allNum = allNum + totalNum
|
||
allStatusNum = allStatusNum + statusNum
|
||
|
||
pendingTimeListAll = utils.CombineSlice(pendingTimeListAll, pipelineTime.PendingList)
|
||
runningTimeListAll = utils.CombineSlice(runningTimeListAll, pipelineTime.RunningList)
|
||
durationTimeListAll = utils.CombineSlice(durationTimeListAll, pipelineTime.DurationList)
|
||
}
|
||
|
||
avgDurationTime = utils.CalAverageTime(req.StatisticsType, durationTimeListAll)
|
||
avgPendingTime = utils.CalAverageTime(req.StatisticsType, pendingTimeListAll)
|
||
avgRunningTime = utils.CalAverageTime(req.StatisticsType, runningTimeListAll)
|
||
log.Info("avgDurationTime: %v, avgPendingTime: %v, avgRunningTime: %v", avgDurationTime, avgPendingTime, avgRunningTime)
|
||
|
||
return
|
||
}
|
||
|
||
// QueryProjectPipelines ...
|
||
func (s *Service) QueryProjectPipelines(c context.Context, req *model.PipelineDataReq, since, until time.Time) (totalNum, statusNum int, pipelineTime *model.PipelineTime, err error) {
|
||
var (
|
||
pipelineList gitlab.PipelineList
|
||
pipeline *gitlab.Pipeline
|
||
resp *gitlab.Response
|
||
startQuery bool
|
||
meetTime bool
|
||
projectID = req.ProjectID
|
||
pendingTime float64
|
||
runningTime float64
|
||
totalTime float64
|
||
)
|
||
|
||
pipelineTime = &model.PipelineTime{}
|
||
|
||
opt := &gitlab.ListProjectPipelinesOptions{}
|
||
if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
|
||
log.Error("ListProjectPipelines err: %+v", err)
|
||
return
|
||
}
|
||
|
||
page := 1
|
||
for page <= resp.TotalPages {
|
||
opt.ListOptions.Page = page
|
||
if !startQuery && (!since.IsZero() || !until.IsZero()) {
|
||
if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
log.Error("ListProjectPipelines err: %+v", err)
|
||
return
|
||
}
|
||
if page == 1 && len(pipelineList) <= 0 {
|
||
return
|
||
}
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelineList[0].ID); err != nil {
|
||
return
|
||
}
|
||
|
||
if pipeline.CreatedAt.After(until) {
|
||
page++
|
||
continue
|
||
} else {
|
||
startQuery = true
|
||
page--
|
||
continue
|
||
}
|
||
}
|
||
|
||
// start query
|
||
if pipelineList, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
meetTime = true
|
||
for _, v := range pipelineList {
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
|
||
return
|
||
}
|
||
|
||
createTime := pipeline.CreatedAt
|
||
if !since.IsZero() || !until.IsZero() {
|
||
meetTime = createTime.After(since) && createTime.Before(until)
|
||
}
|
||
|
||
//the pipeline is we need
|
||
if meetTime {
|
||
totalNum = totalNum + 1
|
||
|
||
if req.Branch != "" && req.Branch != pipeline.Ref {
|
||
continue
|
||
} else if req.User != "" && req.User != pipeline.User.Name {
|
||
continue
|
||
} else if req.State != "" && req.State != pipeline.Status {
|
||
continue
|
||
}
|
||
|
||
statusNum = statusNum + 1
|
||
|
||
if pipeline.Status != "cancel" {
|
||
if pipeline.StartedAt == nil {
|
||
pendingTime = 0
|
||
runningTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
} else {
|
||
pendingTime = pipeline.StartedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
runningTime = pipeline.FinishedAt.Sub(*pipeline.StartedAt).Seconds()
|
||
}
|
||
totalTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
|
||
pipelineTime.PendingMax, pipelineTime.PendingMin = utils.CalSizeTime(pendingTime, pipelineTime.PendingMax, pipelineTime.PendingMin)
|
||
pipelineTime.RunningMax, pipelineTime.RunningMin = utils.CalSizeTime(runningTime, pipelineTime.RunningMax, pipelineTime.RunningMin)
|
||
pipelineTime.DurationMax, pipelineTime.DurationMin = utils.CalSizeTime(totalTime, pipelineTime.DurationMax, pipelineTime.DurationMin)
|
||
|
||
pipelineTime.PendingList = append(pipelineTime.PendingList, pendingTime)
|
||
pipelineTime.RunningList = append(pipelineTime.RunningList, runningTime)
|
||
pipelineTime.DurationList = append(pipelineTime.DurationList, totalTime)
|
||
}
|
||
}
|
||
|
||
// time is over, so return
|
||
if (!since.IsZero() || !until.IsZero()) && createTime.Before(since) {
|
||
return
|
||
}
|
||
}
|
||
|
||
page++
|
||
}
|
||
return
|
||
}
|
||
|
||
// QueryPipelinesFromDB ...
|
||
func (s *Service) QueryPipelinesFromDB(c context.Context, req *model.PipelineDataReq, since, until time.Time) (totalNum, statusNum int, pipelineTime *model.PipelineTime, err error) {
|
||
var (
|
||
fmtLayout = `%d-%d-%d 00:00:00`
|
||
pipelines []*model.StatisticsPipeline
|
||
projectID = req.ProjectID
|
||
pendingTime float64
|
||
runningTime float64
|
||
totalTime float64
|
||
)
|
||
pipelineTime = &model.PipelineTime{}
|
||
|
||
sinceStr := fmt.Sprintf(fmtLayout, since.Year(), since.Month(), since.Day())
|
||
untilStr := fmt.Sprintf(fmtLayout, until.Year(), until.Month(), until.Day())
|
||
if totalNum, statusNum, pipelines, err = s.dao.QueryPipelinesByTime(projectID, req, sinceStr, untilStr); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, pipeline := range pipelines {
|
||
if pipeline.Status == model.StatusCancel {
|
||
continue
|
||
}
|
||
if pipeline.FinishedAt == nil {
|
||
continue
|
||
}
|
||
|
||
if pipeline.StartedAt == nil {
|
||
pendingTime = 0
|
||
runningTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
} else {
|
||
pendingTime = pipeline.StartedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
runningTime = pipeline.FinishedAt.Sub(*pipeline.StartedAt).Seconds()
|
||
}
|
||
totalTime = pipeline.FinishedAt.Sub(*pipeline.CreatedAt).Seconds()
|
||
|
||
pipelineTime.PendingMax, pipelineTime.PendingMin = utils.CalSizeTime(pendingTime, pipelineTime.PendingMax, pipelineTime.PendingMin)
|
||
pipelineTime.RunningMax, pipelineTime.RunningMin = utils.CalSizeTime(runningTime, pipelineTime.RunningMax, pipelineTime.RunningMin)
|
||
pipelineTime.DurationMax, pipelineTime.DurationMin = utils.CalSizeTime(totalTime, pipelineTime.DurationMax, pipelineTime.DurationMin)
|
||
|
||
pipelineTime.PendingList = append(pipelineTime.PendingList, pendingTime)
|
||
pipelineTime.RunningList = append(pipelineTime.RunningList, runningTime)
|
||
pipelineTime.DurationList = append(pipelineTime.DurationList, totalTime)
|
||
}
|
||
return
|
||
}
|
||
|
||
//alertProjectPipelineProc cron func
|
||
func (s *Service) alertProjectPipelineProc() {
|
||
for _, alert := range conf.Conf.Property.Git.AlertPipeline {
|
||
projectId := alert.ProjectID
|
||
runningTimeout := alert.RunningTimeout
|
||
runningRate := alert.RunningRate
|
||
runningThreshold := alert.RunningThreshold
|
||
pendingTimeout := alert.PendingTimeout
|
||
pendingThreshold := alert.PendingThreshold
|
||
go func() {
|
||
var err error
|
||
if err = s.PipelineAlert(context.TODO(), projectId, runningTimeout, runningThreshold, runningRate, gitlab.Running); err != nil {
|
||
log.Error("PipelineAlert Running (%+v)", err)
|
||
}
|
||
if err = s.PipelineAlert(context.TODO(), projectId, pendingTimeout, pendingThreshold, 0, gitlab.Pending); err != nil {
|
||
log.Error("PipelineAlert Pending (%+v)", err)
|
||
}
|
||
}()
|
||
}
|
||
}
|
||
|
||
//PipelineAlert ...
|
||
func (s *Service) PipelineAlert(c context.Context, projectID, timeout, threshold, rate int, status gitlab.BuildStateValue) (err error) {
|
||
var (
|
||
layout = "2006-01-02 15:04:05"
|
||
pipeline *gitlab.Pipeline
|
||
timeoutNum int
|
||
message string
|
||
pipelineurl string
|
||
durationTime float64
|
||
pipelineSUM int
|
||
timeoutPipeline string
|
||
pipelineList gitlab.PipelineList
|
||
resp *gitlab.Response
|
||
projectInfo *model.ProjectInfo
|
||
userlist = conf.Conf.Property.Git.UserList
|
||
w = wechat.New(s.dao)
|
||
sendMessage = false
|
||
)
|
||
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
return
|
||
}
|
||
repo := projectInfo.Repo
|
||
if len(repo) > len(_gitSSH) {
|
||
repo = repo[len(_gitSSH) : len(repo)-len(_gitSSHTail)]
|
||
repo = _gitHTTP + repo
|
||
}
|
||
timeNow := time.Now().Format(layout)
|
||
message = fmt.Sprintf("[SAGA]Pipeline 告警 %v\n项目：%s\n", timeNow, repo)
|
||
for page := 1; ; page++ {
|
||
if pipelineList, resp, err = s.git.ListProjectPipelines(page, projectID, status); err != nil {
|
||
return
|
||
}
|
||
for _, item := range pipelineList {
|
||
pipelineSUM += 1
|
||
if pipeline, _, err = s.git.GetPipeline(projectID, item.ID); err != nil {
|
||
return
|
||
}
|
||
if status == gitlab.Pending {
|
||
durationTime = pipeline.UpdatedAt.Sub(*pipeline.CreatedAt).Minutes()
|
||
} else if status == gitlab.Running {
|
||
//此处时间计算换成job
|
||
if durationTime, err = s.PipelineRunningTime(projectID, item.ID); err != nil {
|
||
return
|
||
}
|
||
}
|
||
if int(durationTime) >= timeout {
|
||
timeoutNum += 1
|
||
pipelineurl = fmt.Sprintf("%d. %s/pipelines/%d (%vmin)\n", timeoutNum, repo, pipeline.ID, int(durationTime))
|
||
timeoutPipeline += pipelineurl
|
||
}
|
||
}
|
||
if resp.NextPage == 0 {
|
||
break
|
||
}
|
||
}
|
||
|
||
if timeoutPipeline != "" {
|
||
message += fmt.Sprintf(`列表(url|%s时间):%s%s`, status, "\n", timeoutPipeline)
|
||
}
|
||
if pipelineSUM > 0 {
|
||
message += fmt.Sprintf(`状态：%s 总数为%d个`, status, pipelineSUM)
|
||
}
|
||
if status == gitlab.Pending {
|
||
var alertMessage string
|
||
message += fmt.Sprintf(`%s告警：`, "\n")
|
||
if pipelineSUM >= threshold {
|
||
alertMessage = fmt.Sprintf(`[ 数量（%d）>=警戒值（%d） ]`, pipelineSUM, threshold)
|
||
sendMessage = true
|
||
}
|
||
message += alertMessage
|
||
if timeoutNum >= 1 {
|
||
if alertMessage != "" {
|
||
message = message[:strings.LastIndex(message, " ]")] + fmt.Sprintf(`，%s时间>=警戒值（%d） ]`, status, timeout)
|
||
} else {
|
||
message += fmt.Sprintf(`[ %s时间>=警戒值（%d） ]`, status, timeout)
|
||
}
|
||
sendMessage = true
|
||
}
|
||
}
|
||
if status == gitlab.Running && timeoutNum >= threshold {
|
||
sendMessage = true
|
||
message += fmt.Sprintf(`，时间>%dmin为%d个%s告警：[ 数量（%d）>=警戒值（%d) ]`, timeout, timeoutNum, "\n", timeoutNum, threshold)
|
||
if timeoutNum*100/pipelineSUM >= rate {
|
||
message = message[:strings.LastIndex(message, " ]")] + fmt.Sprintf(`，比例（%v%s）>=警戒值%d%s ]`, timeoutNum*100/pipelineSUM, "%", rate, "%")
|
||
}
|
||
}
|
||
if sendMessage {
|
||
return w.PushMsg(c, userlist, message)
|
||
}
|
||
return
|
||
}
|
||
|
||
//PipelineRunningTime ...
|
||
func (s *Service) PipelineRunningTime(projectID, pipelineID int) (durationTime float64, err error) {
|
||
var jobList []*gitlab.Job
|
||
if jobList, _, err = s.git.ListPipelineJobs(nil, projectID, pipelineID); err != nil {
|
||
return
|
||
}
|
||
for _, job := range jobList {
|
||
if job.Status != _manualJob && job.Name != _androidScheduleJob && job.Name != _iosScheduleJob {
|
||
if job.FinishedAt != nil && job.StartedAt != nil {
|
||
durationTime += job.FinishedAt.Sub(*job.StartedAt).Minutes()
|
||
} else if job.StartedAt != nil {
|
||
durationTime += time.Since(*job.StartedAt).Minutes()
|
||
}
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
/*-------------------------------------- sync pipeline ----------------------------------------*/
|
||
|
||
// SyncProjectPipelines ...
|
||
func (s *Service) SyncProjectPipelines(projectID int) (result *model.SyncResult, err error) {
|
||
var (
|
||
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime
|
||
syncAllTime = false
|
||
since *time.Time
|
||
until *time.Time
|
||
projectInfo *model.ProjectInfo
|
||
)
|
||
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil {
|
||
return
|
||
}
|
||
if !syncAllTime {
|
||
|
||
since, until = utils.CalSyncTime()
|
||
log.Info("sync project id(%d), name(%s) pipeline, time since: %v, until: %v", projectID, projectInfo.Name, since, until)
|
||
if result, err = s.SyncProjectPipelinesByTime(projectID, projectInfo.Name, *since, *until); err != nil {
|
||
return
|
||
}
|
||
} else {
|
||
log.Info("sync project id(%d), name(%s) pipeline", projectID, projectInfo.Name)
|
||
if result, err = s.SyncProjectAllPipelines(projectID, projectInfo.Name); err != nil {
|
||
return
|
||
}
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// SyncProjectPipelinesByTime ...
|
||
func (s *Service) SyncProjectPipelinesByTime(projectID int, projectName string, since, until time.Time) (result *model.SyncResult, err error) {
|
||
var (
|
||
pipelines gitlab.PipelineList
|
||
pipeline *gitlab.Pipeline
|
||
resp *gitlab.Response
|
||
startQuery bool
|
||
)
|
||
result = &model.SyncResult{}
|
||
|
||
if _, resp, err = s.gitlab.ListProjectPipelines(1, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
page := 1
|
||
for page <= resp.TotalPages {
|
||
result.TotalPage++
|
||
|
||
if !startQuery {
|
||
if pipelines, resp, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
if page == 1 && len(pipelines) <= 0 {
|
||
return
|
||
}
|
||
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, pipelines[0].ID); err != nil {
|
||
return
|
||
}
|
||
|
||
if pipeline.CreatedAt.After(until) {
|
||
page++
|
||
continue
|
||
} else {
|
||
startQuery = true
|
||
page--
|
||
continue
|
||
}
|
||
}
|
||
|
||
// start query
|
||
if pipelines, _, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, v := range pipelines {
|
||
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
|
||
return
|
||
}
|
||
|
||
createTime := pipeline.CreatedAt
|
||
if createTime.After(since) && createTime.Before(until) {
|
||
|
||
if err = s.structureDBPipeline(projectID, projectName, pipeline); err != nil {
|
||
log.Error("pipeline Save Database err: projectID(%d), PipelineID(%d)", projectID, pipeline.ID)
|
||
err = nil
|
||
|
||
errData := &model.FailData{
|
||
ChildID: pipeline.ID,
|
||
}
|
||
result.FailData = append(result.FailData, errData)
|
||
|
||
continue
|
||
}
|
||
result.TotalNum++
|
||
}
|
||
|
||
if createTime.Before(since) {
|
||
return
|
||
}
|
||
}
|
||
page++
|
||
}
|
||
|
||
return
|
||
}
|
||
|
||
// SyncProjectAllPipelines ...
|
||
func (s *Service) SyncProjectAllPipelines(projectID int, projectName string) (result *model.SyncResult, err error) {
|
||
var (
|
||
pipelines gitlab.PipelineList
|
||
pipeline *gitlab.Pipeline
|
||
resp *gitlab.Response
|
||
)
|
||
result = &model.SyncResult{}
|
||
|
||
for page := 1; ; page++ {
|
||
result.TotalPage++
|
||
if pipelines, resp, err = s.gitlab.ListProjectPipelines(page, projectID, ""); err != nil {
|
||
return
|
||
}
|
||
|
||
for _, v := range pipelines {
|
||
if pipeline, _, err = s.gitlab.GetPipeline(projectID, v.ID); err != nil {
|
||
return
|
||
}
|
||
|
||
if err = s.structureDBPipeline(projectID, projectName, pipeline); err != nil {
|
||
log.Error("pipeline Save Database err: projectID(%d), PipelineID(%d)", projectID, pipeline.ID)
|
||
err = nil
|
||
|
||
errData := &model.FailData{
|
||
ChildID: pipeline.ID,
|
||
}
|
||
result.FailData = append(result.FailData, errData)
|
||
|
||
continue
|
||
}
|
||
result.TotalNum++
|
||
}
|
||
|
||
if resp.NextPage == 0 {
|
||
break
|
||
}
|
||
}
|
||
return
|
||
}
|
||
|
||
// structureDBPipeline ...
|
||
func (s *Service) structureDBPipeline(projectID int, projectName string, pipeline *gitlab.Pipeline) (err error) {
|
||
|
||
statisticPipeline := &model.StatisticsPipeline{
|
||
PipelineID: pipeline.ID,
|
||
ProjectID: projectID,
|
||
ProjectName: projectName,
|
||
Status: pipeline.Status,
|
||
Ref: pipeline.Ref,
|
||
Tag: pipeline.Tag,
|
||
User: pipeline.User.Name,
|
||
CreatedAt: pipeline.CreatedAt,
|
||
UpdatedAt: pipeline.UpdatedAt,
|
||
StartedAt: pipeline.StartedAt,
|
||
FinishedAt: pipeline.FinishedAt,
|
||
CommittedAt: pipeline.CommittedAt,
|
||
Coverage: pipeline.Coverage,
|
||
Duration: pipeline.Duration,
|
||
DurationTime: 0,
|
||
}
|
||
|
||
return s.SaveDatabasePipeline(statisticPipeline)
|
||
}
|
||
|
||
// SaveDatabasePipeline ...
|
||
func (s *Service) SaveDatabasePipeline(pipelineDB *model.StatisticsPipeline) (err error) {
|
||
var total int
|
||
|
||
if total, err = s.dao.HasPipeline(pipelineDB.ProjectID, pipelineDB.PipelineID); err != nil {
|
||
log.Error("SaveDatabasePipeline HasPipeline(%+v)", err)
|
||
return
|
||
}
|
||
|
||
// found only one, so update
|
||
if total == 1 {
|
||
if err = s.dao.UpdatePipeline(pipelineDB.ProjectID, pipelineDB.PipelineID, pipelineDB); err != nil {
|
||
log.Error("SaveDatabasePipeline UpdatePipeline err(%+v)", err)
|
||
return
|
||
}
|
||
return
|
||
} else if total > 1 {
|
||
// found repeated row, this situation will not exist under normal
|
||
log.Warn("SaveDatabasePipeline pipeline has more rows(%d)", total)
|
||
return
|
||
}
|
||
|
||
// insert row now
|
||
if err = s.dao.CreatePipeline(pipelineDB); err != nil {
|
||
log.Error("SaveDatabasePipeline CreatePipeline err(%+v)", err)
|
||
return
|
||
}
|
||
|
||
return
|
||
}
|