Create & Init Project...

app/job/main/appstatic/service/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cal_diff.go",
+        "push.go",
+        "service.go",
+        "upbfs.go",
+    ],
+    importpath = "go-common/app/job/main/appstatic/service",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/appstatic/conf:go_default_library",
+        "//app/job/main/appstatic/dao/caldiff:go_default_library",
+        "//app/job/main/appstatic/dao/push:go_default_library",
+        "//app/job/main/appstatic/model:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/appstatic/service/cal_diff.go

@@ -0,0 +1,255 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+const (
+	errFormat = "Func:[%s] - Step:[%s] - Error:[%v]"
+	_zipType  = "application/zip"
+	_tmpSlash = "*"
+	// file type
+	_fullPackage           = 0
+	_diffPackge            = 1
+	_calculationInProgress = 2
+)
+
+// Sync modified season data to the license owner
+func (s *Service) calDiffproc() {
+	defer s.waiter.Done()
+	var (
+		file *model.ResourceFile
+		err  error
+	)
+	for {
+		if s.daoClosed {
+			log.Info("DB closed!")
+			return
+		}
+		time.Sleep(time.Duration(s.c.Cfg.Diff.FreDiff))
+		// check recently created files
+		if file, err = s.dao.DiffNew(ctx); err != nil {
+			log.Error(errFormat, "callDiffproc", "diffNew", err)
+			continue
+		}
+		// if recently created file not found, we retry failed files
+		if file == nil {
+			if file, err = s.dao.DiffRetry(ctx); err != nil {
+				log.Error(errFormat, "callDiffproc", "DiffRetry", err)
+				continue
+			}
+			if file == nil {
+				log.Error(errFormat, "callDiffproc", "DiffRetry", "No Diff To Calculate")
+				continue
+			}
+		}
+		// begin calculate
+		log.Info("Calculate Diff for (fileID:%d, ResID: %d, FileName:%s)", file.ID, file.ResourceID, file.Name)
+		if err := s.calDiff(file); err != nil {
+			log.Error(errFormat, "calDiffproc", "calDiff", err)
+			continue
+		}
+	}
+}
+
+// get the file info from DB and download the file to local
+func (s *Service) catchFile(file *model.ResourceFile, version int) (newPath string, err error) {
+	var (
+		size   int64
+		newPkg *model.ResourceFile
+	)
+	// get the new package info
+	if newPkg, err = s.getFile(file.ResourceID, _fullPackage, version); err != nil {
+		log.Error(errFormat, "calDiff", fmt.Sprintf("getFileNew %v", file), err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	// save the new package
+	newPkg.Name = strings.Replace(newPkg.Name, "/", _tmpSlash, 1)    // avoid open file error
+	newPath = fmt.Sprintf("%s/%s", s.c.Cfg.Diff.Folder, newPkg.Name) // combine the local path
+	if size, err = s.dao.DownloadFile(ctx, newPkg.URL, newPath); err != nil {
+		log.Error(errFormat, "calDiff", fmt.Sprintf("saveFile %v", file), err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	// check result file
+	if _, err = os.Stat(newPath); os.IsNotExist(err) {
+		log.Error(errFormat, "diffCmd", "IsNotExist", newPath+" - File Not Exist")
+		return
+	}
+	log.Info("Save File From URL [%s] in [%s], Size: %d", newPkg.URL, newPath, size)
+	return
+}
+
+// calculate the diff for one file struct and upload the result to BFS and fill the URL
+func (s *Service) calDiff(file *model.ResourceFile) (err error) {
+	var (
+		newPath    string
+		oldPath    string
+		patchPath  string
+		patchURL   string
+		patchFInfo *model.FileInfo
+	)
+	// update the status of the file, to avoid being picked by another
+	if err = s.chgFStatus(file.ID, _calculationInProgress); err != nil {
+		log.Error(errFormat, "calDiff", "chgFStatus", err)
+		return
+	}
+	// save the new file
+	if newPath, err = s.catchFile(file, 0); err != nil {
+		log.Error(errFormat, "callDiff", "catchFile", err)
+		return
+	}
+	// save the old file
+	if oldPath, err = s.catchFile(file, int(file.FromVer)); err != nil {
+		log.Error(errFormat, "callDiff", "catchFile", err)
+		return
+	}
+	// exec bsdiff to get the patch file and upload it
+	file.Name = strings.Replace(file.Name, "/", _tmpSlash, 1)
+	patchPath = s.c.Cfg.Diff.Folder + "/" + file.Name
+	if patchFInfo, patchURL, err = s.diffCmd(file.Name, patchPath, newPath, oldPath); err != nil {
+		log.Error(errFormat, "calDiff", "diffCmd", err)
+		s.chgFStatus(file.ID, _diffPackge)
+		return
+	}
+	log.Info("Upload Path File From [%s] to [%s], Size: %d", patchPath, patchURL)
+	// save the url to the file
+	if err = s.dao.SaveFile(ctx, file.ID, &model.FileInfo{
+		Name: patchFInfo.Name,
+		Size: patchFInfo.Size,
+		Md5:  patchFInfo.Md5,
+		URL:  patchURL}); err != nil {
+		log.Error(errFormat, "calDiff", "updateURL", err)
+		return
+	}
+	// delete all the packages used
+	if err = delPkgs(newPath, oldPath, patchPath); err != nil {
+		log.Error(errFormat, "calDiff", "delPkgs", err)
+	}
+	return
+}
+
+// delete all the packages used to generate the diff pkg
+func delPkgs(newPath string, oldPath string, patchPath string) (err error) {
+	if err = deleteFile(newPath); err != nil {
+		log.Error(errFormat, "delPkgs", "NewPath", err)
+		return
+	}
+	if err = deleteFile(oldPath); err != nil {
+		log.Error(errFormat, "delPkgs", "oldPath", err)
+		return
+	}
+	if err = deleteFile(patchPath); err != nil {
+		log.Error(errFormat, "delPkgs", "patchPath", err)
+	}
+	return
+}
+
+// delete one file
+func deleteFile(path string) (err error) {
+	// check file
+	if _, err = os.Stat(path); os.IsNotExist(err) {
+		err = fmt.Errorf("File %s Not exist", path)
+		log.Error(errFormat, "deleteFile", "IsNotExist", "File Not Exist")
+		return
+	}
+	if err = os.Remove(path); err != nil {
+		log.Error(errFormat, "deleteFile", "Remove", err)
+		return
+	}
+	log.Info("Delete %s Succesfully", path)
+	return
+}
+
+// execute the bsdiff command to have the result
+func (s *Service) diffCmd(patchName string, patchPath string, newPath string, oldPath string) (patchFInfo *model.FileInfo, location string, err error) {
+	var (
+		content []byte
+		fInfo   os.FileInfo
+	)
+	begin := time.Now()
+	cmd := exec.Command("bsdiff", oldPath, newPath, patchPath)
+	// exec Command
+	if err = cmd.Run(); err != nil {
+		log.Error(errFormat, "diffCmd", "cmdRun", err)
+		return
+	}
+	timeCost := time.Since(begin)
+	log.Info("BSDiff Command Finished. Time Cost: %v, Cmd:%s %s %s %s", timeCost, "bsdiff", oldPath, newPath, patchPath)
+	// check patch file
+	if fInfo, err = os.Stat(patchPath); os.IsNotExist(err) {
+		log.Error(errFormat, "diffCmd", "patchFileCheck", "File Not Exist")
+		return
+	}
+	log.Info("Patch File Generated, Name: %s, Size: %s.", fInfo.Name(), fInfo.Size())
+	// read patch file and upload
+	if content, err = ioutil.ReadFile(patchPath); err != nil {
+		log.Error(errFormat, "diffCmd", "ReadFile_Patch", err)
+		return
+	}
+	if patchFInfo, err = s.ParseFile(content); err != nil {
+		log.Error(errFormat, "diffCmd", "ParsePatchFile", err)
+		return
+	}
+	patchFInfo.Name = rename(patchName, patchFInfo.Md5)
+	// upload patch file to bfs
+	location, err = s.Upload(context.Background(), patchFInfo.Name, _zipType, time.Now().Unix(), content)
+	if err != nil {
+		log.Error(errFormat, "diffCmd", "UploadPatch", err)
+	}
+	return
+}
+
+// split the patchName, pick the ModID (res[0]), VersionInfo (res[1]), Insert the md5 inside
+func rename(patchName string, md5 string) (newName string) {
+	res := strings.Split(patchName, _tmpSlash)
+	if len(res) != 2 {
+		log.Error("patchName %s can't split", patchName)
+		return patchName
+	}
+	return res[0] + "_" + md5 + "/" + res[1]
+}
+
+// get file object ( struct )
+func (s *Service) getFile(resID int, fileType int, version int) (file *model.ResourceFile, err error) {
+	file = &model.ResourceFile{}
+	var (
+		res    *model.Resource // current version
+		resHis *model.Resource // history version
+		poolID int
+	)
+	if res, err = s.dao.ParseResID(ctx, resID); err != nil {
+		log.Error("[getFile]-[findPool %d]-Error(%v)", resID, err)
+		return
+	}
+	poolID = int(res.PoolID)
+	if version != 0 { // full pkg of the history version
+		if resHis, err = s.dao.ParseResVer(ctx, poolID, version); err != nil {
+			log.Error("[getFile]-[findVersion]-Error(%v)", err)
+			return
+		}
+		resID = int(resHis.ID)
+	}
+	if file, err = s.dao.ReadyFile(ctx, resID, fileType); err != nil {
+		log.Error(errFormat, "getUrl", "First", err)
+	}
+	return
+}
+
+// change the file's status
+func (s *Service) chgFStatus(fileID int, status int) (err error) {
+	if err = s.dao.UpdateStatus(ctx, status, fileID); err != nil {
+		log.Error(errFormat, "chgFStatus", "update", err)
+	}
+	return
+}

app/job/main/appstatic/service/push.go

@@ -0,0 +1,124 @@
+package service
+
+import (
+	"time"
+
+	"go-common/library/log"
+)
+
+const (
+	_platIOS     = "ios"
+	_platAndroid = "android"
+	_platAll     = ""
+	_vIPad       = "ipad"
+	_vIPhone     = "iphone"
+	_vAndroid    = "android"
+	_vAndroidB   = "android_b"
+)
+
+func (s *Service) push(resIDs map[string]int64) {
+	var (
+		now     = time.Now().Unix()
+		timeout = s.c.Cfg.Push.Timeout
+		err     error
+		msg     string
+	)
+	for resID, timeV := range resIDs {
+		finish := false
+		needPush := false
+		// distinguish whether the resource is ready to push. calc finish or timeout
+		if finish, err = s.pushDao.DiffFinish(ctx, resID); err != nil { // check whether diff cal finish
+			continue
+		}
+		if now-timeV > timeout { // check whether it's already timeout
+			needPush = true
+			log.Info("CallPush [%v] Because of Timeout", resID)
+		} else if finish {
+			needPush = true
+			log.Info("CallPush [%v] Because of DiffFinish", resID)
+		} else {
+			log.Info("CallPush Jump [%v]", resID)
+			continue
+		}
+		// prepare api call
+		if msg, err = s.pushDao.PushMsg(ctx, resID); err != nil { // prepare msg
+			continue
+		}
+		if needPush {
+			if err = s.pushDao.CallRefresh(ctx); err != nil {
+				log.Error("CallPush [%d] app-resource refresh error [%v]", resID, err)
+				continue
+			}
+			time.Sleep(time.Duration(s.c.Cfg.Push.Pause))
+			if err = s.pushDao.CallPush(ctx, s.platform(resID), msg, ""); err != nil {
+				log.Error("CallPush [%v] Error [%v]", resID, err)
+				continue
+			}
+			log.Info("CallPush [%v] Succ, Platform: %s, Delete Key", resID)
+			if err = s.pushDao.ZRem(ctx, resID); err != nil {
+				continue
+			}
+		}
+	}
+}
+
+// distinguish the resource's platform info
+func (s *Service) platform(resID string) (platform string) {
+	var (
+		err          error
+		ios, android bool
+		mobiAPPs     []string
+	)
+	platform = _platAll // default value
+	if mobiAPPs, err = s.pushDao.Platform(ctx, resID); err != nil {
+		return
+	}
+	for _, value := range mobiAPPs {
+		switch value {
+		case _vAndroid:
+			android = true
+		case _vAndroidB:
+			android = true
+		case _vIPad:
+			ios = true
+		case _vIPhone:
+			ios = true
+		default:
+			log.Error("ResourceID %d, Limit Wrong Value %s", resID, value)
+		}
+	}
+	if ios && !android {
+		return _platIOS
+	}
+	if !ios && android {
+		return _platAndroid
+	}
+	return // other case like all or none, just return the default value
+}
+
+func (s *Service) pushproc() {
+	var (
+		resIDs map[string]int64
+		err    error
+	)
+	defer s.waiter.Done()
+	for {
+		if s.daoClosed {
+			log.Info("DB closed!")
+			return
+		}
+		time.Sleep(time.Duration(s.c.Cfg.Push.Fre))
+		// pick to push resIDs from redis
+		if resIDs, err = s.pushDao.ZrangeList(ctx); err != nil {
+			log.Error("Get ToPush List Err %v", err)
+			continue
+		}
+		if len(resIDs) == 0 {
+			log.Info("No ToPush Data, Sleep")
+			continue
+		}
+		// push the data
+		log.Info("ToPush Treat Data: %d", len(resIDs))
+		s.push(resIDs)
+	}
+}

app/job/main/appstatic/service/service.go

@@ -0,0 +1,47 @@
+package service
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/job/main/appstatic/conf"
+	"go-common/app/job/main/appstatic/dao/caldiff"
+	"go-common/app/job/main/appstatic/dao/push"
+	"go-common/library/log"
+)
+
+var ctx = context.Background()
+
+// Service .
+type Service struct {
+	c         *conf.Config
+	dao       *caldiff.Dao
+	pushDao   *push.Dao
+	waiter    *sync.WaitGroup
+	daoClosed bool
+}
+
+// New creates a Service instance.
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:       c,
+		dao:     caldiff.New(c),
+		pushDao: push.New(c),
+		waiter:  new(sync.WaitGroup),
+	}
+	s.waiter.Add(1)
+	go s.pushproc()
+	s.waiter.Add(1)
+	go s.calDiffproc()
+	return
+}
+
+// Close releases resources which owned by the Service instance.
+func (s *Service) Close() (err error) {
+	log.Info("Close dao!")
+	s.daoClosed = true
+	log.Info("Wait waiter!")
+	s.waiter.Wait()
+	log.Info("appstatic-job has been closed.")
+	return
+}

app/job/main/appstatic/service/upbfs.go

@@ -0,0 +1,40 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"net/http"
+
+	"go-common/app/job/main/appstatic/model"
+	"go-common/library/log"
+)
+
+// ParseFile analyses file info
+func (s *Service) ParseFile(content []byte) (file *model.FileInfo, err error) {
+	fType := http.DetectContentType(content)
+	// file md5
+	md5hash := md5.New()
+	if _, err = io.Copy(md5hash, bytes.NewReader(content)); err != nil {
+		log.Error("resource uploadFile.Copy error(%v)", err)
+		return
+	}
+	md5 := md5hash.Sum(nil)
+	fMd5 := hex.EncodeToString(md5[:])
+	file = &model.FileInfo{
+		Md5:  fMd5,
+		Type: fType,
+		Size: int64(len(content)),
+	}
+	return
+}
+
+// Upload can upload a file object: store the info in Redis, and transfer the file to Bfs
+func (s *Service) Upload(c context.Context, fileName string, fileType string, timing int64, body []byte) (location string, err error) {
+	if location, err = s.dao.Upload(c, fileName, fileType, timing, body, s.c.Bfs); err != nil { // bfs
+		log.Error("s.upload.UploadBfs() error(%v)", err)
+	}
+	return
+}