Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@ -0,0 +1,59 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"add_ver_test.go",
"publish_test.go",
"service_test.go",
"upbfs_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = [
"//app/admin/main/appstatic/conf:go_default_library",
"//app/admin/main/appstatic/model:go_default_library",
"//library/time:go_default_library",
"//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"add_ver.go",
"publish.go",
"push.go",
"service.go",
"upbfs.go",
],
importpath = "go-common/app/admin/main/appstatic/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/admin/main/appstatic/conf:go_default_library",
"//app/admin/main/appstatic/dao:go_default_library",
"//app/admin/main/appstatic/model:go_default_library",
"//library/log:go_default_library",
"//vendor/github.com/jinzhu/gorm:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,355 @@
package service
import (
"database/sql"
"fmt"
"go-common/app/admin/main/appstatic/model"
"go-common/library/log"
"github.com/jinzhu/gorm"
)
const (
// file type
_fullPackage = 0
_diffPackge = 1
// diff file name format
_diffFormat = "Mod_%d/V_%d-V_%d.bspatch"
// limit column
deviceCol = "device"
mobiAppCol = "mobi_app"
platCol = "plat"
buildCol = "build"
sysverCol = "sysver"
scaleCol = "scale"
levelCol = "level"
archCol = "arch"
// condition column
_bk = "bk"
_wt = "wt"
buildLtCdt = "lt"
buildGtCdt = "gt"
buildLeCdt = "le"
buildGeCdt = "ge"
_valid = 1
)
// GenerateVer generates a new version ( resource ) and cover the diff logic
func (s *Service) GenerateVer(resName string, limitData *model.Limit, fInfo *model.FileInfo, pool *model.ResourcePool, defPkg int) (resID int, version int, err error) {
// create a new version
var tx = s.DB.Begin()
var resource = tx.Create(transResource(resName, pool.ID))
if err = resource.Error; err != nil {
log.Error("GenerateVer DBCreate Resource Error(%v)", err)
tx.Rollback()
return
}
resID = int(resource.Value.(*model.Resource).ID)
log.Info("Resource Generated: ID = %d", resID)
// create the full package in File Table
if err = tx.Create(transFile(fInfo, resID)).Error; err != nil {
log.Error("GenerateVer DBCreate ResourceFile Error(%v)", err)
tx.Rollback()
return
}
// create the resource config
var config = tx.Create(transConfig(int64(resID), limitData))
if err = config.Error; err != nil {
log.Error("GenerateVer DBCreate ResoureConfig Error(%v)", err)
tx.Rollback()
return
}
configID := int64(config.Value.(*model.ResourceConfig).ID)
log.Info("Resource Config Generated: ID = %d", configID)
// create the resource limits
limits := createLimit(configID, limitData)
if len(limits) != 0 {
for _, v := range limits {
if err = tx.Create(v).Error; err != nil {
log.Error("GenerateVer DCreate ResourceLimit (%v) Error(%v)", v, err)
tx.Rollback()
return
}
}
} else {
log.Error("[GenerateVer]-[createLimit]-No limit to create")
}
// commit the transaction
tx.Commit()
log.Info("Transaction Committed, ResID: %d", resID)
// treat the default package setting
if defPkg == 1 {
if err = s.DefaultPkg(resID, pool.ID, configID); err != nil {
log.Error("defaultPkg Error (%v)", err)
return
}
}
// defines the version of this resource
if version, err = s.defineVer(resID, pool.ID); err != nil {
log.Error("defineVer Error (%v)", err)
return
}
// create diff records
if err = s.createDiff(resID); err != nil {
log.Error("[GenerateVer]-[createDiff]-Error(%v)", err)
return
}
return
}
// DefaultPkg sets the resID's config as default package, and resets the other resources' config
func (s *Service) DefaultPkg(resID int, poolID int64, confID int64) (err error) {
var (
tx = s.DB.Begin()
rows *sql.Rows
rid int // resource id
)
// find out all the resources under the same pool, and put them as non-default pkg
if rows, err = s.DB.Model(&model.Resource{}).Where("pool_id = ?", poolID).Select("id").Rows(); err != nil {
return
}
for rows.Next() {
if err = rows.Scan(&rid); err != nil {
tx.Rollback()
return
}
if err = tx.Model(&model.ResourceConfig{}).Where("resource_id = ?", rid).Update("default_package", 0).Error; err != nil {
tx.Rollback()
return
}
}
// defines the new package as the default pkg
if err = tx.Model(&model.ResourceConfig{}).Where("id = ?", confID).Update("default_package", 1).Error; err != nil {
tx.Rollback()
return
}
tx.Commit()
return
}
// defines the version of the resource after the transaction commited
func (s *Service) defineVer(resID int, poolID int64) (version int, err error) {
var maxVer = model.Resource{}
if err = s.DB.Where("id < ?", resID).Where("pool_id = ?", poolID).Order("version desc").First(&maxVer).Error; err == gorm.ErrRecordNotFound {
err = nil
}
if err != nil {
log.Error("GenerateVer DBFind ResourceVer (%d) Error(%v)", resID, err)
return
}
version = int(maxVer.Version) + 1
if err = s.DB.Model(&model.Resource{}).Where("id = ?", resID).Update("version", version).Error; err != nil {
log.Error("GenerateVer DBUpdate ResourceVer (%d)-(%d) Error(%v)", resID, maxVer.Version+1, err)
return
}
return
}
// create diff packages for the latest version with the history versions
func (s *Service) createDiff(resID int) (err error) {
var (
prodVers, testVers []int64
currRes *model.Resource
)
// pick history versions to calculate diff
if prodVers, testVers, currRes, err = s.pickDiff(resID); err != nil {
return
}
// put diff packages in our DB
if err = s.putDiff(resID, mergeSlice(prodVers, testVers), currRes); err != nil {
return
}
return
}
// pick history versions to calculate diff
func (s *Service) pickDiff(resID int) (prodVers []int64, testVers []int64, currRes *model.Resource, err error) {
var (
VersProd = []*model.Resource{} // prod
VersTest = []*model.Resource{} // test
res = model.Resource{}
)
if err = s.DB.Where("id = ?", resID).First(&res).Error; err != nil {
log.Error("[createDiff]-[FindCurrentRes]-Error(%v)", err)
return
}
currRes = &res
poolID := currRes.PoolID
// calculate prod diffs
if err = s.DB.Joins("LEFT JOIN resource_config ON resource.id = resource_config.resource_id").
Where("resource.pool_id = ?", poolID).
Where("resource.id < ?", resID).
Where("resource_config.valid = ?", _valid).
Order("resource.version desc").Limit(s.c.Cfg.HistoryVer).
Select("resource.*").
Find(&VersProd).Error; err != nil {
log.Error("[createDiff]-[FindHistoryVers]-Error(%v)", err)
return
}
log.Info("Get Prod History Versions: %d", len(VersProd))
// calculate test diffs
if err = s.DB.Joins("LEFT JOIN resource_config ON resource.id = resource_config.resource_id").
Where("resource.pool_id = ?", poolID).
Where("resource.id < ?", resID).
Where("resource_config.valid != ?", _valid).
Where("resource_config.valid_test = ?", _valid).
Order("resource.version desc").Limit(s.c.Cfg.HistoryVer).
Select("resource.*").
Find(&VersTest).Error; err != nil {
log.Error("[createDiff]-[FindHistoryVers]-Error(%v)", err)
return
}
log.Info("Get Test History Versions: %d", len(VersTest))
// merge slices
prodVers = pickVersion(VersProd)
testVers = pickVersion(VersTest)
return
}
// put diff package in our DB
func (s *Service) putDiff(resID int, historyVers []int64, currRes *model.Resource) (err error) {
for _, v := range historyVers {
var diffPkg = &model.ResourceFile{
Name: fmt.Sprintf(_diffFormat, currRes.PoolID, v, currRes.Version),
FromVer: v,
ResourceID: resID,
FileType: _diffPackge,
}
if err = s.DB.Create(diffPkg).Error; err != nil {
log.Error("[createDiff]-[createDiffPkg]-Error(%v)", err)
return
}
}
log.Info("[createDiff]-Create (%d) Diff Pkg for ResID:(%d)", len(historyVers), resID)
return
}
// pick resource version
func pickVersion(s1 []*model.Resource) (res []int64) {
if len(s1) > 0 {
for _, v := range s1 {
res = append(res, v.Version)
}
}
return
}
// merge int64 slices
func mergeSlice(s1 []int64, s2 []int64) []int64 {
slice := make([]int64, len(s1)+len(s2))
copy(slice, s1)
copy(slice[len(s1):], s2)
return slice
}
// transform to Resource struct
func transResource(resMame string, id int64) *model.Resource {
return &model.Resource{
Name: resMame,
Version: 0, // will be updated after the transaction commits
PoolID: id,
}
}
// transform to File struct
func transFile(fInfo *model.FileInfo, resID int) *model.ResourceFile {
return &model.ResourceFile{
Name: fInfo.Name,
Type: fInfo.Type,
Md5: fInfo.Md5,
Size: int(fInfo.Size),
URL: fInfo.URL,
ResourceID: resID,
FileType: _fullPackage,
}
}
// transform to Config struct
func transConfig(resID int64, limitData *model.Limit) *model.ResourceConfig {
var cfg = &model.ResourceConfig{
ResourceID: resID,
Valid: 0,
IsDeleted: 0,
IsWifi: limitData.IsWifi,
}
if limitData.TimeRange != nil {
cfg.Etime = limitData.TimeRange.Etime
cfg.Stime = limitData.TimeRange.Stime
}
return cfg
}
// createLimit
func createLimit(configID int64, limitData *model.Limit) (res []*model.ResourceLimit) {
// create device limit
if len(limitData.Device) != 0 {
generateDevice(limitData.Device, &res, configID, deviceCol, _bk)
}
// create plat limit
if len(limitData.Plat) != 0 {
generateDevice(limitData.Plat, &res, configID, platCol, _wt)
}
// create mobi_app limit
if len(limitData.MobiApp) != 0 {
generateDevice(limitData.MobiApp, &res, configID, mobiAppCol, _wt)
}
// scale, level, arch
if len(limitData.Level) != 0 {
generateDevice(limitData.Level, &res, configID, levelCol, _wt)
}
if len(limitData.Scale) != 0 {
generateDevice(limitData.Scale, &res, configID, scaleCol, _wt)
}
if len(limitData.Arch) != 0 {
generateDevice(limitData.Arch, &res, configID, archCol, _wt)
}
// create build & sysver limit
if build := limitData.Build; build != nil {
generateBuild(build, configID, &res, buildCol)
}
if sysver := limitData.Sysver; sysver != nil {
generateBuild(sysver, configID, &res, sysverCol)
}
log.Info("createLimit creates %d limits", len(res))
return
}
// generate build-like limits data (json, range), insert them into the slice 'res'
func generateBuild(build *model.Build, configID int64, res *[]*model.ResourceLimit, column string) {
if build.GT != 0 {
*res = append(*res, transBuild(buildGtCdt, configID, build.GT, column))
}
if build.LT != 0 {
*res = append(*res, transBuild(buildLtCdt, configID, build.LT, column))
}
if build.GE != 0 {
*res = append(*res, transBuild(buildGeCdt, configID, build.GE, column))
}
if build.LE != 0 {
*res = append(*res, transBuild(buildLeCdt, configID, build.LE, column))
}
}
// generate device-like limits data([]string), insert them into the slice 'res'
func generateDevice(device []string, res *[]*model.ResourceLimit, configID int64, col string, cdt string) {
for _, v := range device {
*res = append(*res, &model.ResourceLimit{
ConfigID: configID,
Column: col,
Condition: cdt,
Value: v,
IsDeleted: 0,
})
}
}
func transBuild(condition string, configID int64, value int, column string) *model.ResourceLimit {
return &model.ResourceLimit{
ConfigID: configID,
Column: column,
Condition: condition,
Value: fmt.Sprintf("%d", value),
IsDeleted: 0,
}
}

View File

@ -0,0 +1,14 @@
package service
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_DefaultPkg(t *testing.T) {
Convey("AddFile should return without err", t, WithService(func(svf *Service) {
err := svf.DefaultPkg(74, 3, 54)
So(err, ShouldBeNil)
}))
}

View File

@ -0,0 +1,64 @@
package service
import (
"context"
"go-common/app/admin/main/appstatic/model"
"go-common/library/log"
)
// GendDiff picks the already generated diff packages
func (s *Service) GendDiff(resID int) (generated map[int64]int64, err error) {
generated = make(map[int64]int64)
genVers := []*model.Ver{}
if err = s.DB.Where("file_type IN (1,2)"). // 1=diff pkg, 2=diff pkg calculation in progress
Where("is_deleted = 0").Where("resource_id = ?", resID).Select("id, from_ver").Find(&genVers).Error; err != nil {
log.Error("generatedDiff Error %v", err)
return
}
for _, v := range genVers {
generated[v.FromVer] = v.ID
}
return
}
// Publish returns the second trigger result
func (s *Service) Publish(ctx context.Context, resID int) (data *model.PubResp, err error) {
var (
prodVers, testVers []int64 // the history versions that we should generate for
currRes *model.Resource
generated map[int64]int64
prodMore, testMore []int64
)
// pick history versions to calculate diff
if prodVers, testVers, currRes, err = s.pickDiff(resID); err != nil {
return
}
// pick already generated diff packages
if generated, err = s.GendDiff(resID); err != nil {
return
}
// filter already generated
for _, v := range prodVers {
if _, ok := generated[v]; !ok {
prodMore = append(prodMore, v)
}
}
for _, v := range testVers {
if _, ok := generated[v]; !ok {
testMore = append(testMore, v)
}
}
// put diff packages in our DB
if err = s.putDiff(resID, mergeSlice(prodMore, testMore), currRes); err != nil {
return
}
data = &model.PubResp{
CurrVer: currRes.Version,
DiffProd: prodMore,
DiffTest: testMore,
}
// add the publish resID into to push list
err = s.newPush(ctx, resID)
return
}

View File

@ -0,0 +1,27 @@
package service
import (
"context"
"testing"
"fmt"
. "github.com/smartystreets/goconvey/convey"
)
func TestService_GendDiff(t *testing.T) {
Convey("TestService_GendDiff", t, WithService(func(svf *Service) {
generated, err := svf.GendDiff(57)
So(err, ShouldBeNil)
So(len(generated), ShouldBeGreaterThan, 0)
fmt.Println(generated)
}))
}
func TestService_Publish(t *testing.T) {
Convey("TestService_Publish", t, WithService(func(svf *Service) {
data, err := svf.Publish(context.Background(), 57)
So(err, ShouldBeNil)
fmt.Println(data)
}))
}

View File

@ -0,0 +1,15 @@
package service
import (
"context"
"go-common/library/log"
)
// add a new resource ID into the to push list
func (s *Service) newPush(ctx context.Context, resID int) (err error) {
if err = s.dao.ZAddPush(ctx, resID); err != nil {
log.Error("NewPush Redis for ResID: %d, Error: %v", resID, err)
}
return
}

View File

@ -0,0 +1,61 @@
package service
import (
"context"
"sync"
"go-common/app/admin/main/appstatic/conf"
"go-common/app/admin/main/appstatic/dao"
"go-common/library/log"
"github.com/jinzhu/gorm"
)
// Service biz service def.
type Service struct {
c *conf.Config
dao *dao.Dao
DB *gorm.DB
waiter *sync.WaitGroup
daoClosed bool // logic close the dao's DB
MaxSize int64 // max size supported for the file to upload
}
// New new a Service and return.
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
daoClosed: false,
waiter: new(sync.WaitGroup),
}
s.DB = s.dao.DB
if s.c.Cfg.Storage == "nas" {
s.MaxSize = 200 * 1024 * 1024 // 200M NAS
} else {
s.MaxSize = 20 * 1024 * 1024 // 20M BFS
}
return s
}
// Ping check dao health.
func (s *Service) Ping(c context.Context) (err error) {
return
}
// Wait wait all closed.
func (s *Service) Wait() {
if s.dao != nil {
s.daoClosed = true
log.Info("Dao is logically closed!")
}
log.Info("Wait waiter!")
s.waiter.Wait()
}
// Close close all dao.
func (s *Service) Close() {
log.Info("Close Dao physically!")
s.dao.Close()
log.Info("Service Closed!")
}

View File

@ -0,0 +1,70 @@
package service
import (
"context"
"flag"
"fmt"
"path/filepath"
"testing"
"time"
"go-common/app/admin/main/appstatic/conf"
"go-common/app/admin/main/appstatic/model"
. "github.com/smartystreets/goconvey/convey"
)
var srv *Service
func init() {
dir, _ := filepath.Abs("../cmd/appstatic-admin-test.toml")
flag.Set("conf", dir)
conf.Init()
srv = New(conf.Conf)
time.Sleep(time.Second)
}
func WithService(f func(s *Service)) func() {
return func() {
f(srv)
}
}
func TestService_Ping(t *testing.T) {
Convey("Ping", t, WithService(func(svf *Service) {
svf.Ping(context.Background())
fmt.Println("service ping successfully")
}))
}
func TestService_Wait(t *testing.T) {
Convey("Ping", t, WithService(func(svf *Service) {
svf.Wait()
fmt.Println("service wait successfully")
}))
}
func TestService_Close(t *testing.T) {
Convey("Close", t, WithService(func(svf *Service) {
svf.Close()
fmt.Println("service closed successfully")
}))
}
func TestService_GenerateVer(t *testing.T) {
Convey("Generate Version", t, WithService(func(svf *Service) {
resID, version, err := svf.GenerateVer("myTestRes", &model.Limit{}, &model.FileInfo{
Name: "testResFile",
Size: 333,
Type: "application/zip",
Md5: "333",
URL: "www.bilibili.com",
}, &model.ResourcePool{
ID: 1,
Name: "resourcefile",
}, 1)
So(err, ShouldBeNil)
So(resID, ShouldBeGreaterThan, 0)
So(version, ShouldBeGreaterThan, 0)
}))
}

View File

@ -0,0 +1,74 @@
package service
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"io"
"net/http"
"go-common/app/admin/main/appstatic/model"
"go-common/library/log"
)
// Upload can upload a file object: store the info in Redis, and transfer the file to Bfs
func (s *Service) Upload(c context.Context, fileName string, fileType string, timing int64, body []byte) (location string, err error) {
if s.c.Cfg.Storage == "nas" { // nas
if location, err = s.dao.UploadNas(c, fileName, body, s.c.Nas); err != nil {
log.Error("s.upload.UploadNas() error(%v)", err)
}
return
}
if location, err = s.dao.Upload(c, fileName, fileType, timing, body, s.c.Bfs); err != nil { // bfs
log.Error("s.upload.UploadBfs() error(%v)", err)
}
return
}
// AddFile inserts file info into DB and updates its resource version+1
func (s *Service) AddFile(c context.Context, file *model.ResourceFile, version int) (err error) {
if err = s.DB.Create(file).Error; err != nil {
log.Error("resSrv.DB.Create error(%v)", err)
return
}
// the resource containing the file updates its version
if err = s.DB.Model(&model.Resource{}).Where("id = ?", file.ResourceID).Update("version", version+1).Error; err != nil {
log.Error("resSrv.Update Version error(%v)", err)
return
}
return nil
}
// ParseFile analyses file info
func (s *Service) ParseFile(content []byte) (file *model.FileInfo, err error) {
fType := http.DetectContentType(content)
// file md5
md5hash := md5.New()
if _, err = io.Copy(md5hash, bytes.NewReader(content)); err != nil {
log.Error("resource uploadFile.Copy error(%v)", err)
return
}
md5 := md5hash.Sum(nil)
fMd5 := hex.EncodeToString(md5[:])
file = &model.FileInfo{
Md5: fMd5,
Type: fType,
Size: int64(len(content)),
}
return
}
// TypeCheck checks whether the file type is allowed
func (s *Service) TypeCheck(fType string) (canAllow bool) {
allowed := s.c.Cfg.Filetypes
if len(allowed) == 0 {
return true
}
for _, v := range allowed {
if v == fType {
return true
}
}
return false
}

View File

@ -0,0 +1,78 @@
package service
import (
"context"
"fmt"
"testing"
"time"
"go-common/app/admin/main/appstatic/model"
xtime "go-common/library/time"
. "github.com/smartystreets/goconvey/convey"
)
var (
c = context.TODO()
now = time.Now().Unix()
)
func TestService_AddFile(t *testing.T) {
Convey("AddFile should return without err", t, WithService(func(svf *Service) {
// simulate resource & file data
res := &model.Resource{}
res.ID = 1
resFile := &model.ResourceFile{
ID: 0,
Name: "tName",
Type: "image/png",
Md5: "d41d8cd98f00b204e9800998ecf8427e",
Size: 20,
URL: "http://uat-i0.hdslb.com/bfs/app-static/timg.png",
ResourceID: 2,
Ctime: xtime.Time(now),
Mtime: xtime.Time(now),
}
// DB insert
err := svf.AddFile(c, resFile, 3)
// testing
So(err, ShouldBeNil)
}))
}
func TestService_Upload(t *testing.T) {
Convey("Upload should return without err", t, WithService(func(svf *Service) {
// simulate file content
str2 := "Testing File Content"
data2 := []byte(str2)
// upload to bfs
location, err := svf.Upload(c, "", "image/png", now, data2)
// testing
So(err, ShouldBeNil)
So(location, ShouldNotBeNil)
fmt.Println(location)
}))
}
func TestService_ParseFile(t *testing.T) {
Convey("ParseFile can get file info", t, WithService(func(svf *Service) {
// simulate file content
str2 := "Testing File Content"
data2 := []byte(str2)
// upload to bfs
fInfo, err := svf.ParseFile(data2)
// testing
So(err, ShouldBeNil)
So(fInfo.Size, ShouldBeGreaterThan, 0)
So(fInfo.Type, ShouldNotBeBlank)
So(fInfo.Md5, ShouldNotBeBlank)
}))
}
func TestService_TypeCheck(t *testing.T) {
Convey("File Type Check", t, WithService(func(svf *Service) {
So(svf.TypeCheck("application/zip"), ShouldBeTrue)
So(svf.TypeCheck("application/xml"), ShouldBeTrue)
So(svf.TypeCheck("application/json"), ShouldBeFalse)
}))
}