256 lines
7.5 KiB
Go
256 lines
7.5 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"time"
|
|
|
|
"go-common/app/job/main/appstatic/model"
|
|
"go-common/library/log"
|
|
)
|
|
|
|
const (
|
|
errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
|
|
_zipType = "application/zip"
|
|
_tmpSlash = "*"
|
|
// file type
|
|
_fullPackage = 0
|
|
_diffPackge = 1
|
|
_calculationInProgress = 2
|
|
)
|
|
|
|
// Sync modified season data to the license owner
|
|
func (s *Service) calDiffproc() {
|
|
defer s.waiter.Done()
|
|
var (
|
|
file *model.ResourceFile
|
|
err error
|
|
)
|
|
for {
|
|
if s.daoClosed {
|
|
log.Info("DB closed!")
|
|
return
|
|
}
|
|
time.Sleep(time.Duration(s.c.Cfg.Diff.FreDiff))
|
|
// check recently created files
|
|
if file, err = s.dao.DiffNew(ctx); err != nil {
|
|
log.Error(errFormat, "callDiffproc", "diffNew", err)
|
|
continue
|
|
}
|
|
// if recently created file not found, we retry failed files
|
|
if file == nil {
|
|
if file, err = s.dao.DiffRetry(ctx); err != nil {
|
|
log.Error(errFormat, "callDiffproc", "DiffRetry", err)
|
|
continue
|
|
}
|
|
if file == nil {
|
|
log.Error(errFormat, "callDiffproc", "DiffRetry", "No Diff To Calculate")
|
|
continue
|
|
}
|
|
}
|
|
// begin calculate
|
|
log.Info("Calculate Diff for (fileID:%d, ResID: %d, FileName:%s)", file.ID, file.ResourceID, file.Name)
|
|
if err := s.calDiff(file); err != nil {
|
|
log.Error(errFormat, "calDiffproc", "calDiff", err)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
// get the file info from DB and download the file to local
|
|
func (s *Service) catchFile(file *model.ResourceFile, version int) (newPath string, err error) {
|
|
var (
|
|
size int64
|
|
newPkg *model.ResourceFile
|
|
)
|
|
// get the new package info
|
|
if newPkg, err = s.getFile(file.ResourceID, _fullPackage, version); err != nil {
|
|
log.Error(errFormat, "calDiff", fmt.Sprintf("getFileNew %v", file), err)
|
|
s.chgFStatus(file.ID, _diffPackge)
|
|
return
|
|
}
|
|
// save the new package
|
|
newPkg.Name = strings.Replace(newPkg.Name, "/", _tmpSlash, 1) // avoid open file error
|
|
newPath = fmt.Sprintf("%s/%s", s.c.Cfg.Diff.Folder, newPkg.Name) // combine the local path
|
|
if size, err = s.dao.DownloadFile(ctx, newPkg.URL, newPath); err != nil {
|
|
log.Error(errFormat, "calDiff", fmt.Sprintf("saveFile %v", file), err)
|
|
s.chgFStatus(file.ID, _diffPackge)
|
|
return
|
|
}
|
|
// check result file
|
|
if _, err = os.Stat(newPath); os.IsNotExist(err) {
|
|
log.Error(errFormat, "diffCmd", "IsNotExist", newPath+" - File Not Exist")
|
|
return
|
|
}
|
|
log.Info("Save File From URL [%s] in [%s], Size: %d", newPkg.URL, newPath, size)
|
|
return
|
|
}
|
|
|
|
// calculate the diff for one file struct and upload the result to BFS and fill the URL
|
|
func (s *Service) calDiff(file *model.ResourceFile) (err error) {
|
|
var (
|
|
newPath string
|
|
oldPath string
|
|
patchPath string
|
|
patchURL string
|
|
patchFInfo *model.FileInfo
|
|
)
|
|
// update the status of the file, to avoid being picked by another
|
|
if err = s.chgFStatus(file.ID, _calculationInProgress); err != nil {
|
|
log.Error(errFormat, "calDiff", "chgFStatus", err)
|
|
return
|
|
}
|
|
// save the new file
|
|
if newPath, err = s.catchFile(file, 0); err != nil {
|
|
log.Error(errFormat, "callDiff", "catchFile", err)
|
|
return
|
|
}
|
|
// save the old file
|
|
if oldPath, err = s.catchFile(file, int(file.FromVer)); err != nil {
|
|
log.Error(errFormat, "callDiff", "catchFile", err)
|
|
return
|
|
}
|
|
// exec bsdiff to get the patch file and upload it
|
|
file.Name = strings.Replace(file.Name, "/", _tmpSlash, 1)
|
|
patchPath = s.c.Cfg.Diff.Folder + "/" + file.Name
|
|
if patchFInfo, patchURL, err = s.diffCmd(file.Name, patchPath, newPath, oldPath); err != nil {
|
|
log.Error(errFormat, "calDiff", "diffCmd", err)
|
|
s.chgFStatus(file.ID, _diffPackge)
|
|
return
|
|
}
|
|
log.Info("Upload Path File From [%s] to [%s], Size: %d", patchPath, patchURL)
|
|
// save the url to the file
|
|
if err = s.dao.SaveFile(ctx, file.ID, &model.FileInfo{
|
|
Name: patchFInfo.Name,
|
|
Size: patchFInfo.Size,
|
|
Md5: patchFInfo.Md5,
|
|
URL: patchURL}); err != nil {
|
|
log.Error(errFormat, "calDiff", "updateURL", err)
|
|
return
|
|
}
|
|
// delete all the packages used
|
|
if err = delPkgs(newPath, oldPath, patchPath); err != nil {
|
|
log.Error(errFormat, "calDiff", "delPkgs", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// delete all the packages used to generate the diff pkg
|
|
func delPkgs(newPath string, oldPath string, patchPath string) (err error) {
|
|
if err = deleteFile(newPath); err != nil {
|
|
log.Error(errFormat, "delPkgs", "NewPath", err)
|
|
return
|
|
}
|
|
if err = deleteFile(oldPath); err != nil {
|
|
log.Error(errFormat, "delPkgs", "oldPath", err)
|
|
return
|
|
}
|
|
if err = deleteFile(patchPath); err != nil {
|
|
log.Error(errFormat, "delPkgs", "patchPath", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// delete one file
|
|
func deleteFile(path string) (err error) {
|
|
// check file
|
|
if _, err = os.Stat(path); os.IsNotExist(err) {
|
|
err = fmt.Errorf("File %s Not exist", path)
|
|
log.Error(errFormat, "deleteFile", "IsNotExist", "File Not Exist")
|
|
return
|
|
}
|
|
if err = os.Remove(path); err != nil {
|
|
log.Error(errFormat, "deleteFile", "Remove", err)
|
|
return
|
|
}
|
|
log.Info("Delete %s Succesfully", path)
|
|
return
|
|
}
|
|
|
|
// execute the bsdiff command to have the result
|
|
func (s *Service) diffCmd(patchName string, patchPath string, newPath string, oldPath string) (patchFInfo *model.FileInfo, location string, err error) {
|
|
var (
|
|
content []byte
|
|
fInfo os.FileInfo
|
|
)
|
|
begin := time.Now()
|
|
cmd := exec.Command("bsdiff", oldPath, newPath, patchPath)
|
|
// exec Command
|
|
if err = cmd.Run(); err != nil {
|
|
log.Error(errFormat, "diffCmd", "cmdRun", err)
|
|
return
|
|
}
|
|
timeCost := time.Since(begin)
|
|
log.Info("BSDiff Command Finished. Time Cost: %v, Cmd:%s %s %s %s", timeCost, "bsdiff", oldPath, newPath, patchPath)
|
|
// check patch file
|
|
if fInfo, err = os.Stat(patchPath); os.IsNotExist(err) {
|
|
log.Error(errFormat, "diffCmd", "patchFileCheck", "File Not Exist")
|
|
return
|
|
}
|
|
log.Info("Patch File Generated, Name: %s, Size: %s.", fInfo.Name(), fInfo.Size())
|
|
// read patch file and upload
|
|
if content, err = ioutil.ReadFile(patchPath); err != nil {
|
|
log.Error(errFormat, "diffCmd", "ReadFile_Patch", err)
|
|
return
|
|
}
|
|
if patchFInfo, err = s.ParseFile(content); err != nil {
|
|
log.Error(errFormat, "diffCmd", "ParsePatchFile", err)
|
|
return
|
|
}
|
|
patchFInfo.Name = rename(patchName, patchFInfo.Md5)
|
|
// upload patch file to bfs
|
|
location, err = s.Upload(context.Background(), patchFInfo.Name, _zipType, time.Now().Unix(), content)
|
|
if err != nil {
|
|
log.Error(errFormat, "diffCmd", "UploadPatch", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// split the patchName, pick the ModID (res[0]), VersionInfo (res[1]), Insert the md5 inside
|
|
func rename(patchName string, md5 string) (newName string) {
|
|
res := strings.Split(patchName, _tmpSlash)
|
|
if len(res) != 2 {
|
|
log.Error("patchName %s can't split", patchName)
|
|
return patchName
|
|
}
|
|
return res[0] + "_" + md5 + "/" + res[1]
|
|
}
|
|
|
|
// get file object ( struct )
|
|
func (s *Service) getFile(resID int, fileType int, version int) (file *model.ResourceFile, err error) {
|
|
file = &model.ResourceFile{}
|
|
var (
|
|
res *model.Resource // current version
|
|
resHis *model.Resource // history version
|
|
poolID int
|
|
)
|
|
if res, err = s.dao.ParseResID(ctx, resID); err != nil {
|
|
log.Error("[getFile]-[findPool %d]-Error(%v)", resID, err)
|
|
return
|
|
}
|
|
poolID = int(res.PoolID)
|
|
if version != 0 { // full pkg of the history version
|
|
if resHis, err = s.dao.ParseResVer(ctx, poolID, version); err != nil {
|
|
log.Error("[getFile]-[findVersion]-Error(%v)", err)
|
|
return
|
|
}
|
|
resID = int(resHis.ID)
|
|
}
|
|
if file, err = s.dao.ReadyFile(ctx, resID, fileType); err != nil {
|
|
log.Error(errFormat, "getUrl", "First", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
// change the file's status
|
|
func (s *Service) chgFStatus(fileID int, status int) (err error) {
|
|
if err = s.dao.UpdateStatus(ctx, status, fileID); err != nil {
|
|
log.Error(errFormat, "chgFStatus", "update", err)
|
|
}
|
|
return
|
|
}
|