359 lines
9.4 KiB
Go
359 lines
9.4 KiB
Go
|
package service
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"context"
|
|||
|
|
"fmt"
|
|||
|
|
"sync"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"go-common/app/admin/ep/marthe/model"
|
|||
|
|
"go-common/library/ecode"
|
|||
|
|
"go-common/library/log"
|
|||
|
|
|
|||
|
|
"github.com/satori/go.uuid"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// RunVersions Run Versions.
|
|||
|
|
func (s *Service) RunVersions(buglyVersionID int64) (rep map[string]interface{}, err error) {
|
|||
|
|
lock := s.getBatchRunLock(buglyVersionID)
|
|||
|
|
lock.Lock()
|
|||
|
|
defer lock.Unlock()
|
|||
|
|
|
|||
|
|
var (
|
|||
|
|
buglyRunVersions []*model.BuglyVersion
|
|||
|
|
uid string
|
|||
|
|
isEnableRun bool
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// 获取可以跑的版本号list
|
|||
|
|
if buglyRunVersions, err = s.GetBatchRunVersions(); err != nil {
|
|||
|
|
log.Error("GetBatchRunVersions err(%v) ", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, buglyRunVersion := range buglyRunVersions {
|
|||
|
|
if buglyRunVersion.ID == buglyVersionID {
|
|||
|
|
isEnableRun = true
|
|||
|
|
uid = uuid.NewV4().String()
|
|||
|
|
|
|||
|
|
// 插入batchrun表
|
|||
|
|
buglyBatchRun := &model.BuglyBatchRun{
|
|||
|
|
BuglyVersionID: buglyRunVersion.ID,
|
|||
|
|
Version: buglyRunVersion.Version,
|
|||
|
|
BatchID: uid,
|
|||
|
|
Status: model.BuglyBatchRunStatusRunning,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if err = s.dao.InsertBuglyBatchRun(buglyBatchRun); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
s.batchRunCache.Do(context.Background(), func(ctx context.Context) {
|
|||
|
|
if err = s.runPerVersion(uid, buglyRunVersion); err != nil {
|
|||
|
|
log.Error("runPerVersion uid(%s) version(%s) err(%v) ", uid, buglyRunVersion.Version, err)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
defer func() {
|
|||
|
|
// batch run 并且更新batch run表
|
|||
|
|
batchStatus := model.BuglyBatchRunStatusDone
|
|||
|
|
errMsg := ""
|
|||
|
|
endTime := time.Now()
|
|||
|
|
|
|||
|
|
if err != nil {
|
|||
|
|
batchStatus = model.BuglyBatchRunStatusFailed
|
|||
|
|
errMsg = err.Error()
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
updateBuglyBatchRun := &model.BuglyBatchRun{
|
|||
|
|
ID: buglyBatchRun.ID,
|
|||
|
|
Status: batchStatus,
|
|||
|
|
ErrorMsg: errMsg,
|
|||
|
|
EndTime: endTime,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if err = s.dao.UpdateBuglyBatchRun(updateBuglyBatchRun); err != nil {
|
|||
|
|
log.Error("runPerVersion UpdateBuglyBatchRun uid(%s) version(%s) err(%v) ", uid, buglyRunVersion.Version, err)
|
|||
|
|
}
|
|||
|
|
}()
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
rep = make(map[string]interface{})
|
|||
|
|
rep["uid"] = uid
|
|||
|
|
rep["enable_to_run"] = isEnableRun
|
|||
|
|
|
|||
|
|
if !isEnableRun {
|
|||
|
|
err = ecode.MartheTaskInRunning
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// BatchRunVersions Batch Run Versions
|
|||
|
|
func (s *Service) BatchRunVersions() (err error) {
|
|||
|
|
var (
|
|||
|
|
buglyRunVersions []*model.BuglyVersion
|
|||
|
|
uid = uuid.NewV4().String()
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
// 获取可以跑的版本号list
|
|||
|
|
if buglyRunVersions, err = s.dao.FindEnableAndReadyVersions(); err != nil {
|
|||
|
|
log.Error("GetBatchRunVersions err(%v) ", err)
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, buglyRunVersion := range buglyRunVersions {
|
|||
|
|
if _, err = s.RunVersions(buglyRunVersion.ID); err != nil {
|
|||
|
|
log.Error("runPerVersion uid(%s) version(%s) err(%v) ", uid, buglyRunVersion.Version, err)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) runPerVersion(uid string, buglyRunVersion *model.BuglyVersion) (err error) {
|
|||
|
|
var (
|
|||
|
|
buglyRet *model.BugRet
|
|||
|
|
buglyCookie *model.BuglyCookie
|
|||
|
|
requestPageCnt int
|
|||
|
|
c = context.Background()
|
|||
|
|
lastRunTime time.Time
|
|||
|
|
buglyBatchRun *model.BuglyBatchRun
|
|||
|
|
buglyProject *model.BuglyProject
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
log.Info("start run version: [%s] batchId: [%s]", buglyRunVersion.Version, uid)
|
|||
|
|
|
|||
|
|
if buglyProject, err = s.dao.QueryBuglyProject(buglyRunVersion.BuglyProjectID); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if buglyProject.ID == 0 {
|
|||
|
|
err = ecode.NothingFound
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//get issue total count
|
|||
|
|
bugIssueRequest := &model.BugIssueRequest{
|
|||
|
|
ProjectID: buglyProject.ProjectID,
|
|||
|
|
PlatformID: buglyProject.PlatformID,
|
|||
|
|
Version: buglyRunVersion.Version,
|
|||
|
|
ExceptionType: buglyProject.ExceptionType,
|
|||
|
|
StartNum: 0,
|
|||
|
|
Rows: 1,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//get last run time
|
|||
|
|
if buglyBatchRun, err = s.dao.QueryLastSuccessBatchRunByVersion(buglyRunVersion.Version); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
if buglyBatchRun.ID > 0 {
|
|||
|
|
lastRunTime = buglyBatchRun.CTime
|
|||
|
|
} else {
|
|||
|
|
loc, _ := time.LoadLocation("Local")
|
|||
|
|
if lastRunTime, err = time.ParseInLocation(model.TimeLayout, model.TimeLayout, loc); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//get enable cookie
|
|||
|
|
if buglyCookie, err = s.GetEnableCookie(); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// if cookie, update cookie as expired
|
|||
|
|
defer func() {
|
|||
|
|
if err != nil && err == ecode.MartheCookieExpired {
|
|||
|
|
s.DisableCookie(c, buglyCookie.ID)
|
|||
|
|
}
|
|||
|
|
}()
|
|||
|
|
|
|||
|
|
if buglyRet, err = s.dao.BuglyIssueAndRetry(c, buglyCookie, bugIssueRequest); err != nil || len(buglyRet.BugIssues) < 1 {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
//获取issue count 和 page 上限
|
|||
|
|
requestPageCnt = int(buglyRet.NumFound/s.c.Bugly.IssuePageSize) + 1
|
|||
|
|
requestPageCntMax := s.c.Bugly.IssueCountUpper / s.c.Bugly.IssuePageSize
|
|||
|
|
if requestPageCnt > requestPageCntMax {
|
|||
|
|
requestPageCnt = requestPageCntMax
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// update or add issue
|
|||
|
|
for i := 0; i < requestPageCnt; i++ {
|
|||
|
|
innerBreak := false
|
|||
|
|
|
|||
|
|
bugIssueRequest.StartNum = s.c.Bugly.IssuePageSize * i
|
|||
|
|
bugIssueRequest.Rows = s.c.Bugly.IssuePageSize
|
|||
|
|
|
|||
|
|
var ret *model.BugRet
|
|||
|
|
if ret, err = s.dao.BuglyIssueAndRetry(c, buglyCookie, bugIssueRequest); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
loc, _ := time.LoadLocation("Local")
|
|||
|
|
issueLink := "/v2/crash-reporting/errors/%s/%s/report?pid=%s&searchType=detail&version=%s&start=0&date=all"
|
|||
|
|
for _, issueDto := range ret.BugIssues {
|
|||
|
|
var (
|
|||
|
|
issueTime time.Time
|
|||
|
|
bugIssueDetail *model.BugIssueDetail
|
|||
|
|
tagStr string
|
|||
|
|
bugDetail = "no detail"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
tmpTime := []rune(issueDto.LastTime)
|
|||
|
|
issueTime, _ = time.ParseInLocation(model.TimeLayout, string(tmpTime[:len(tmpTime)-4]), loc)
|
|||
|
|
|
|||
|
|
//issue时间早于库里面最新时间的，跳出
|
|||
|
|
if issueTime.Before(lastRunTime) {
|
|||
|
|
innerBreak = true
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var tmpBuglyIssue *model.BuglyIssue
|
|||
|
|
|
|||
|
|
if tmpBuglyIssue, err = s.dao.GetBuglyIssue(issueDto.IssueID, issueDto.Version); err != nil {
|
|||
|
|
log.Error("d.GetSaveIssues url(%s) err(%v)", "GetSaveIssues", err)
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, bugTag := range issueDto.Tags {
|
|||
|
|
tagStr = tagStr + bugTag.TagName + ","
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if tmpBuglyIssue.ID != 0 {
|
|||
|
|
//update
|
|||
|
|
issueRecord := &model.BuglyIssue{
|
|||
|
|
IssueNo: issueDto.IssueID,
|
|||
|
|
Title: issueDto.Title,
|
|||
|
|
LastTime: issueTime,
|
|||
|
|
HappenTimes: issueDto.Count,
|
|||
|
|
UserTimes: issueDto.UserCount,
|
|||
|
|
Version: issueDto.Version,
|
|||
|
|
Tags: tagStr,
|
|||
|
|
}
|
|||
|
|
s.dao.UpdateBuglyIssue(issueRecord)
|
|||
|
|
} else {
|
|||
|
|
//create
|
|||
|
|
if bugIssueDetail, err = s.dao.BuglyIssueDetailAndRetry(c, buglyCookie, buglyProject.ProjectID, buglyProject.PlatformID, issueDto.IssueID); err == nil {
|
|||
|
|
bugDetail = bugIssueDetail.CallStack
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
issueURL := s.c.Bugly.Host + fmt.Sprintf(issueLink, buglyProject.ProjectID, issueDto.IssueID, buglyProject.PlatformID, buglyRunVersion.Version)
|
|||
|
|
issueRecord := &model.BuglyIssue{
|
|||
|
|
IssueNo: issueDto.IssueID,
|
|||
|
|
Title: issueDto.Title,
|
|||
|
|
LastTime: issueTime,
|
|||
|
|
HappenTimes: issueDto.Count,
|
|||
|
|
UserTimes: issueDto.UserCount,
|
|||
|
|
Version: issueDto.Version,
|
|||
|
|
Tags: tagStr,
|
|||
|
|
Detail: bugDetail,
|
|||
|
|
ExceptionMsg: issueDto.ExceptionMsg,
|
|||
|
|
KeyStack: issueDto.KeyStack,
|
|||
|
|
IssueLink: issueURL,
|
|||
|
|
ProjectID: buglyProject.ProjectID,
|
|||
|
|
}
|
|||
|
|
s.dao.InsertBuglyIssue(issueRecord)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if innerBreak {
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
log.Info("end run version: [%s] batchId: [%s]", buglyRunVersion.Version, uid)
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// GetBatchRunVersions Get Batch Run Versions.
|
|||
|
|
func (s *Service) GetBatchRunVersions() (buglyRunVersions []*model.BuglyVersion, err error) {
|
|||
|
|
var (
|
|||
|
|
buglyVersions []*model.BuglyVersion
|
|||
|
|
buglyBatchRuns []*model.BuglyBatchRun
|
|||
|
|
)
|
|||
|
|
if buglyVersions, err = s.dao.FindEnableAndReadyVersions(); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if buglyBatchRuns, err = s.dao.QueryBuglyBatchRunsByStatus(model.BuglyBatchRunStatusRunning); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 排除正在执行的版本
|
|||
|
|
for _, buglyVersion := range buglyVersions {
|
|||
|
|
var isVersionRun bool
|
|||
|
|
for _, buglyBatchRun := range buglyBatchRuns {
|
|||
|
|
if buglyBatchRun.Version == buglyVersion.Version {
|
|||
|
|
isVersionRun = true
|
|||
|
|
break
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if !isVersionRun {
|
|||
|
|
buglyRunVersions = append(buglyRunVersions, buglyVersion)
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// DisableBatchRunOverTime Disable Batch Run OverTime.
|
|||
|
|
func (s *Service) DisableBatchRunOverTime() (err error) {
|
|||
|
|
var (
|
|||
|
|
buglyBatchRuns []*model.BuglyBatchRun
|
|||
|
|
tapdBugRecords []*model.TapdBugRecord
|
|||
|
|
timeNow = time.Now()
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
//清 未完成 disable batch run
|
|||
|
|
if buglyBatchRuns, err = s.dao.QueryBuglyBatchRunsByStatus(model.BuglyBatchRunStatusRunning); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, buglyBatchRun := range buglyBatchRuns {
|
|||
|
|
if timeNow.Sub(buglyBatchRun.CTime).Hours() > float64(s.c.Scheduler.BatchRunOverHourTime) {
|
|||
|
|
updateBuglyBatchRun := &model.BuglyBatchRun{
|
|||
|
|
ID: buglyBatchRun.ID,
|
|||
|
|
Status: model.BuglyBatchRunStatusFailed,
|
|||
|
|
ErrorMsg: "over time",
|
|||
|
|
EndTime: timeNow,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if err = s.dao.UpdateBuglyBatchRun(updateBuglyBatchRun); err != nil {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 清 未完成 disable insert tapd bug
|
|||
|
|
if tapdBugRecords, err = s.dao.QueryTapdBugRecordByStatus(model.InsertBugStatusRunning); err != nil {
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for _, tapdBugRecord := range tapdBugRecords {
|
|||
|
|
if timeNow.Sub(tapdBugRecord.CTime).Hours() > float64(s.c.Scheduler.BatchRunOverHourTime) {
|
|||
|
|
tapdBugRecord.Status = model.InsertBugStatusFailed
|
|||
|
|
if err = s.dao.UpdateTapdBugRecord(tapdBugRecord); err != nil {
|
|||
|
|
continue
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func (s *Service) getBatchRunLock(buglyVersionId int64) (batchRunLock *sync.Mutex) {
|
|||
|
|
s.syncBatchRunLock.Lock()
|
|||
|
|
defer s.syncBatchRunLock.Unlock()
|
|||
|
|
|
|||
|
|
var ok bool
|
|||
|
|
if batchRunLock, ok = s.mapBatchRunLocks[buglyVersionId]; !ok {
|
|||
|
|
batchRunLock = new(sync.Mutex)
|
|||
|
|
s.mapBatchRunLocks[buglyVersionId] = batchRunLock
|
|||
|
|
}
|
|||
|
|
return
|
|||
|
|
}
|