Create & Init Project...

This commit is contained in:
2019-04-22 18:49:16 +08:00
commit fc4fa37393
25440 changed files with 4054998 additions and 0 deletions

View File

@@ -0,0 +1,34 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"apk.go",
"helper.go",
"unzip.go",
"writer.go",
],
importpath = "go-common/app/admin/main/macross/tools",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//library/log:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,378 @@
package tools
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"math"
"os"
)
const (
apkSigBlockMinSize uint32 = 32
// https://android.googlesource.com/platform/build/+/android-7.1.2_r27/tools/signapk/src/com/android/signapk/ApkSignerV2.java
// APK_SIGNING_BLOCK_MAGIC = {
// 0x41, 0x50, 0x4b, 0x20, 0x53, 0x69, 0x67, 0x20,
// 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x34, 0x32 }
apkSigBlockMagicHi = 0x3234206b636f6c42 // LITTLE_ENDIAN, High
apkSigBlockMagicLo = 0x20676953204b5041 // LITTLE_ENDIAN, Low
apkChannelBlockID = 0x71777777
// https://en.wikipedia.org/wiki/Zip_(file_format)
// https://android.googlesource.com/platform/build/+/android-7.1.2_r27/tools/signapk/src/com/android/signapk/ZipUtils.java
zipEocdRecSig = 0x06054b50
zipEocdRecMinSize = 22
zipEocdCentralDirSizeFieldOffset = 12
zipEocdCentralDirOffsetFieldOffset = 16
zipEocdCommentLengthFieldOffset = 20
)
// ChannelInfo for apk
type ChannelInfo struct {
Channel string
Extras map[string]string
raw []byte
}
// ChannelInfo to string
func (c *ChannelInfo) String() string {
b := c.Bytes()
if b == nil {
return ""
}
return string(b)
}
// Bytes for ChannelInfo to byte array
func (c *ChannelInfo) Bytes() []byte {
if c.raw != nil {
return c.raw
}
if len(c.Channel) == 0 && c.Extras == nil {
return nil
}
var buf bytes.Buffer
buf.WriteByte('{')
if len(c.Channel) != 0 {
buf.WriteString("\"channel\":")
buf.WriteByte('"')
buf.WriteString(c.Channel)
buf.WriteByte('"')
buf.WriteByte(',')
}
if c.Extras != nil {
for k, v := range c.Extras {
buf.WriteByte('"')
buf.WriteString(k)
buf.WriteByte('"')
buf.WriteByte(':')
buf.WriteByte('"')
buf.WriteString(v)
buf.WriteByte('"')
buf.WriteByte(',')
}
}
if buf.Len() > 2 {
buf.Truncate(buf.Len() - 1)
}
buf.WriteByte('}')
return buf.Bytes()
}
func readChannelInfo(file string) (c ChannelInfo, err error) {
block, err := readChannelBlock(file)
if err != nil {
return c, err
}
if block != nil {
var bundle map[string]string
err := json.Unmarshal(block, &bundle)
if err != nil {
return c, err
}
c.Channel = bundle["channel"]
delete(bundle, "channel")
c.Extras = bundle
c.raw = block
}
return c, nil
}
// read block associated to apkChannelBlockID
func readChannelBlock(file string) ([]byte, error) {
m, err := readIDValues(file, apkChannelBlockID)
if err != nil {
return nil, err
}
return m[apkChannelBlockID], nil
}
func readIDValues(file string, ids ...uint32) (map[uint32][]byte, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
eocd, offset, err := findEndOfCentralDirectoryRecord(f)
if err != nil {
return nil, err
}
if offset <= 0 {
return nil, errors.New("Cannot find EOCD record, maybe a broken zip file")
}
centralDirOffset := getEocdCentralDirectoryOffset(eocd)
block, _, err := findApkSigningBlock(f, centralDirOffset)
if err != nil {
return nil, err
}
return findIDValuesInApkSigningBlock(block, ids...)
}
// End of central directory record (EOCD)
//
// Offset Bytes Description[23]
// 0 4 End of central directory signature = 0x06054b50
// 4 2 Number of this disk
// 6 2 Disk where central directory starts
// 8 2 Number of central directory records on this disk
// 10 2 Total number of central directory records
// 12 4 Size of central directory (bytes)
// 16 4 Offset of start of central directory, relative to start of archive
// 20 2 Comment length (n)
// 22 n Comment
// For a zip with no archive comment, the
// end-of-central-directory record will be 22 bytes long, so
// we expect to find the EOCD marker 22 bytes from the end.
func findEndOfCentralDirectoryRecord(f *os.File) ([]byte, int64, error) {
fi, err := f.Stat()
if err != nil {
return nil, -1, err
}
if fi.Size() < zipEocdRecMinSize {
// No space for EoCD record in the file.
return nil, -1, nil
}
// Optimization: 99.99% of APKs have a zero-length comment field in the EoCD record and thus
// the EoCD record offset is known in advance. Try that offset first to avoid unnecessarily
// reading more data.
ret, offset, err := findEOCDRecord(f, 0)
if err != nil {
return nil, -1, err
}
if ret != nil && offset != -1 {
return ret, offset, nil
}
// EoCD does not start where we expected it to. Perhaps it contains a non-empty comment
// field. Expand the search. The maximum size of the comment field in EoCD is 65535 because
// the comment length field is an unsigned 16-bit number.
return findEOCDRecord(f, math.MaxUint16)
}
func findEOCDRecord(f *os.File, maxCommentSize uint16) ([]byte, int64, error) {
if maxCommentSize > math.MaxUint16 {
return nil, -1, os.ErrInvalid
}
fi, err := f.Stat()
if err != nil {
return nil, -1, err
}
fileSize := fi.Size()
if fileSize < zipEocdRecMinSize {
// No space for EoCD record in the file.
return nil, -1, nil
}
// Lower maxCommentSize if the file is too small.
if s := uint16(fileSize - zipEocdRecMinSize); maxCommentSize > s {
maxCommentSize = s
}
maxEocdSize := zipEocdRecMinSize + maxCommentSize
bufOffsetInFile := fileSize - int64(maxEocdSize)
buf := make([]byte, maxEocdSize)
n, e := f.ReadAt(buf, bufOffsetInFile)
if e != nil {
return nil, -1, err
}
eocdOffsetInFile :=
func() int64 {
eocdWithEmptyCommentStartPosition := n - zipEocdRecMinSize
for expectedCommentLength := uint16(0); expectedCommentLength < maxCommentSize; expectedCommentLength++ {
eocdStartPos := eocdWithEmptyCommentStartPosition - int(expectedCommentLength)
if getUint32(buf, eocdStartPos) == zipEocdRecSig {
n := eocdStartPos + zipEocdCommentLengthFieldOffset
actualCommentLength := getUint16(buf, n)
if actualCommentLength == expectedCommentLength {
return int64(eocdStartPos)
}
}
}
return -1
}()
if eocdOffsetInFile == -1 {
// No EoCD record found in the buffer
return nil, -1, nil
}
// EoCD found
return buf[eocdOffsetInFile:], bufOffsetInFile + eocdOffsetInFile, nil
}
func getEocdCentralDirectoryOffset(buf []byte) uint32 {
return getUint32(buf, zipEocdCentralDirOffsetFieldOffset)
}
func getEocdCentralDirectorySize(buf []byte) uint32 {
return getUint32(buf, zipEocdCentralDirSizeFieldOffset)
}
func setEocdCentralDirectoryOffset(eocd []byte, offset uint32) {
putUint32(offset, eocd, zipEocdCentralDirOffsetFieldOffset)
}
func isExpected(ids []uint32, test uint32) bool {
for _, id := range ids {
if id == test {
return true
}
}
return false
}
func findIDValuesInApkSigningBlock(block []byte, ids ...uint32) (map[uint32][]byte, error) {
ret := make(map[uint32][]byte)
position := 8
limit := len(block) - 24
entryCount := 0
for limit > position { // has remaining bytes
entryCount++
if limit-position < 8 { // but not enough
return nil, fmt.Errorf("APK Signing Block broken on entry #%d", entryCount)
}
length := int(getUint64(block, position))
position += 8
if length < 4 || length > limit-position {
return nil, fmt.Errorf("APK Signing Block broken on entry #%d,"+
" size out of range: length=%d, remaining=%d", entryCount, length, limit-position)
}
nextEntryPosition := position + length
id := getUint32(block, position)
position += 4
if len(ids) == 0 || isExpected(ids, id) {
ret[id] = block[position : position+length-4]
}
position = nextEntryPosition
}
return ret, nil
}
// Find the APK Signing Block. The block immediately precedes the Central Directory.
//
// FORMAT:
// uint64: size (excluding this field)
// repeated ID-value pairs:
// uint64: size (excluding this field)
// uint32: ID
// (size - 4) bytes: value
// uint64: size (same as the one above)
// uint128: magic
func findApkSigningBlock(f *os.File, centralDirOffset uint32) (block []byte, offset int64, err error) {
if centralDirOffset < apkSigBlockMinSize {
return block, offset, fmt.Errorf("APK too small for APK Signing Block."+
" ZIP Central Directory offset: %d", centralDirOffset)
}
// Read the footer of APK signing block
// 24 = sizeof(uint128) + sizeof(uint64)
footer := make([]byte, 24)
_, err = f.ReadAt(footer, int64(centralDirOffset-24))
if err != nil {
return
}
// Read the magic and block size
var blockSizeInFooter = getUint64(footer, 0)
if blockSizeInFooter < 24 || blockSizeInFooter > uint64(math.MaxInt32-8 /* ID-value size field*/) {
return block, offset, fmt.Errorf("APK Signing Block size out of range: %d", blockSizeInFooter)
}
if getUint64(footer, 8) != apkSigBlockMagicLo ||
getUint64(footer, 16) != apkSigBlockMagicHi {
return block, offset, errors.New("No APK Signing Block before ZIP Central Directory")
}
totalSize := blockSizeInFooter + 8 /* APK signing block size field*/
offset = int64(uint64(centralDirOffset) - totalSize)
if offset <= 0 {
return block, offset, fmt.Errorf("invalid offset for APK Signing Block %d", offset)
}
block = make([]byte, totalSize)
_, err = f.ReadAt(block, offset)
if err != nil {
return
}
blockSizeInHeader := getUint64(block, 0)
if blockSizeInHeader != blockSizeInFooter {
return nil, offset, fmt.Errorf("APK Signing Block sizes in header "+
"and footer are mismatched! Except %d but %d", blockSizeInFooter, blockSizeInHeader)
}
return block, offset, nil
}
// FORMAT:
// uint64: size (excluding this field)
// repeated ID-value pairs:
// uint64: size (excluding this field)
// uint32: ID
// (size - 4) bytes: value
// uint64: size (same as the one above)
// uint128: magic
func makeSigningBlockWithChannelInfo(info ChannelInfo, signingBlock []byte) ([]byte, int, error) {
signingBlockSize := getUint64(signingBlock, 0)
signingBlockLen := len(signingBlock)
if n := uint64(signingBlockLen - 8); signingBlockSize != n {
return nil, 0, fmt.Errorf("APK Signing Block is illegal! Expect size %d but %d", signingBlockSize, n)
}
channelValue := info.Bytes()
channelValueSize := uint64(4 + len(channelValue))
resultSize := 8 + signingBlockSize + 8 + channelValueSize
newBlock := make([]byte, resultSize)
position := 0
putUint64(resultSize-8, newBlock, position)
position += 8
// copy raw id-value pairs
n, _ := copyBytes(signingBlock, 8, newBlock, position, int(signingBlockSize)-16-8)
position += n
putUint64(channelValueSize, newBlock, position)
position += 8
putUint32(apkChannelBlockID, newBlock, position)
position += 4
n, _ = copyBytes(channelValue, 0, newBlock, position, len(channelValue))
position += n
putUint64(resultSize-8, newBlock, position)
position += 8
copyBytes(signingBlock, signingBlockLen-16, newBlock, int(resultSize-16), 16)
position += 16
if position != int(resultSize) {
return nil, -1, fmt.Errorf("count mismatched ! %d vs %d", position, resultSize)
}
return newBlock, int(resultSize) - signingBlockLen, nil
}
func makeEocd(origin []byte, newCentralDirOffset uint32) []byte {
eocd := make([]byte, len(origin))
copy(eocd, origin)
setEocdCentralDirectoryOffset(eocd, newCentralDirOffset)
return eocd
}

View File

@@ -0,0 +1,78 @@
package tools
import (
"fmt"
"os"
"path/filepath"
)
//LittleEndian
func getUint16(b []byte, offset int) uint16 {
_ = b[offset+1] // early bounds check
return uint16(b[offset+0]) |
uint16(b[offset+1])<<8
}
//LittleEndian
func getUint32(b []byte, offset int) uint32 {
_ = b[offset+3] // early bounds check
return uint32(b[offset+0]) |
uint32(b[offset+1])<<8 |
uint32(b[offset+2])<<16 |
uint32(b[offset+3])<<24
}
//LittleEndian
func getUint64(b []byte, offset int) uint64 {
_ = b[offset+7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[offset+0]) |
uint64(b[offset+1])<<8 |
uint64(b[offset+2])<<16 |
uint64(b[offset+3])<<24 |
uint64(b[offset+4])<<32 |
uint64(b[offset+5])<<40 |
uint64(b[offset+6])<<48 |
uint64(b[offset+7])<<56
}
//LittleEndian
func putUint64(v uint64, b []byte, offset int) {
_ = b[offset+7] // early bounds check to guarantee safety of writes below
b[offset+0] = byte(v)
b[offset+1] = byte(v >> 8)
b[offset+2] = byte(v >> 16)
b[offset+3] = byte(v >> 24)
b[offset+4] = byte(v >> 32)
b[offset+5] = byte(v >> 40)
b[offset+6] = byte(v >> 48)
b[offset+7] = byte(v >> 56)
}
//LittleEndian
func putUint32(v uint32, b []byte, offset int) {
_ = b[offset+3]
b[offset+0] = byte(v)
b[offset+1] = byte(v >> 8)
b[offset+2] = byte(v >> 16)
b[offset+3] = byte(v >> 24)
}
func copyBytes(src []byte, srcStart int, dst []byte, dstStart int, count int) (int, error) {
if len(src) < srcStart+count || len(dst) < dstStart+count {
return -1, fmt.Errorf("Array index out of bounds")
}
for i := 0; i < count; i++ {
dst[dstStart+i] = src[srcStart+i]
}
return count, nil
}
func fileNameAndExt(path string) (string, string) {
name := filepath.Base(path)
for i := len(name) - 1; i >= 0 && !os.IsPathSeparator(name[i]); i-- {
if name[i] == '.' {
return name[:i], name[i:]
}
}
return name, ""
}

View File

@@ -0,0 +1,60 @@
package tools
import (
"archive/zip"
"go-common/library/log"
"io"
"os"
"path/filepath"
"strings"
)
// Unzip unzip a file to the target directory.
func Unzip(filePath string, targetDir string) (err error) {
reader, err := zip.OpenReader(filePath)
if err != nil {
log.Error("ozip.OpenReader(%s) error(%v)", filePath, err)
return
}
defer reader.Close()
err = os.MkdirAll(targetDir, 0755)
if err != nil {
log.Error("os.MkdirAll(%s) error(%v)", targetDir, err)
return
}
for _, file := range reader.File {
path := filepath.Join(targetDir, file.Name)
if strings.Contains(path, "__MACOSX") {
continue
}
if file.FileInfo().IsDir() {
os.MkdirAll(path, file.Mode())
continue
} else {
os.MkdirAll(filepath.Dir(path), 0755)
}
fileReader, err := file.Open()
if err != nil {
log.Error("file.Open() error(%v)", err)
return err
}
defer fileReader.Close()
targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
if err != nil {
log.Error("os.OpenFile() error(%v)", err)
return err
}
defer targetFile.Close()
if _, err := io.Copy(targetFile, fileReader); err != nil {
log.Error("io.Copy() error(%v)", err)
return err
}
}
return
}

View File

@@ -0,0 +1,225 @@
package tools
import (
"fmt"
"go-common/library/log"
"os"
"path/filepath"
"time"
)
type zipSections struct {
beforeSigningBlock []byte
signingBlock []byte
signingBlockOffset int64
centraDir []byte
centralDirOffset int64
eocd []byte
eocdOffset int64
}
type transform func(*zipSections) (*zipSections, error)
func (z *zipSections) writeTo(output string, transform transform) (err error) {
f, err := os.Create(output)
if err != nil {
return
}
defer f.Close()
newZip, err := transform(z)
if err != nil {
return
}
for _, s := range [][]byte{
newZip.beforeSigningBlock,
newZip.signingBlock,
newZip.centraDir,
newZip.eocd} {
_, err := f.Write(s)
if err != nil {
return err
}
}
return
}
var _debug bool
// GenerateChannelApk generate a specified channel apk
func GenerateChannelApk(out string, channel string, extras map[string]string, input string, force bool, debug bool) (output string, err error) {
_debug = debug
if len(input) == 0 {
err = fmt.Errorf("Error: no input file specified")
log.Error("no input file specified, error(%v)", err)
return
}
if _, err = os.Stat(input); os.IsNotExist(err) {
return
}
if len(out) == 0 {
out = filepath.Dir(input)
} else {
var fi os.FileInfo
err = os.MkdirAll(out, 0755)
if err != nil {
log.Error("os.MkdirAll(%s) error(%v)", out, err)
return
}
fi, err = os.Stat(out)
println("error %v", err)
if os.IsNotExist(err) || !fi.IsDir() {
err = fmt.Errorf("Error: output %s is neither exist nor a dir", out)
log.Error("output %s is neither exist nor a dir, error(%v)", out, err)
return
}
}
if channel == "" {
err = fmt.Errorf("Error: no channel specified")
log.Error("no channel specified, error(%v)", err)
return
}
//TODO: add new option for generating new channel from channelled apk
if c, _ := readChannelInfo(input); len(c.Channel) != 0 {
err = fmt.Errorf("Error: file %s is registered a channel block %s", filepath.Base(input), c.String())
log.Error("file %s is registered a channel block %s, error(%v)", filepath.Base(input), c.String(), err)
return
}
var start time.Time
if _debug {
start = time.Now()
}
z, err := newZipSections(input)
if err != nil {
err = fmt.Errorf("Error occurred on parsing apk %s, %s", input, err)
log.Error("Error occurred on parsing apk %s, error(%v)", input, err)
return
}
name, ext := fileNameAndExt(input)
output = filepath.Join(out, name+"-"+channel+ext)
c := ChannelInfo{Channel: channel, Extras: extras}
err = gen(c, z, output, force)
if err != nil {
err = fmt.Errorf("Error occurred on generating channel %s, %s", channel, err)
log.Error("Error occurred on generating channel %s, error(%v)", input, err)
return
}
if _debug {
println("Consume", time.Since(start).String())
} else {
println("Done!")
}
return
}
func newZipSections(input string) (z zipSections, err error) {
in, err := os.Open(input)
if err != nil {
return
}
defer in.Close()
// read eocd
eocd, eocdOffset, err := findEndOfCentralDirectoryRecord(in)
if err != nil {
return
}
centralDirOffset := getEocdCentralDirectoryOffset(eocd)
centralDirSize := getEocdCentralDirectorySize(eocd)
z.eocd = eocd
z.eocdOffset = eocdOffset
z.centralDirOffset = int64(centralDirOffset)
// read signing block
signingBlock, signingBlockOffset, err := findApkSigningBlock(in, centralDirOffset)
if err != nil {
return
}
z.signingBlock = signingBlock
z.signingBlockOffset = signingBlockOffset
// read bytes before signing block
//TODO: waste too large memory
if signingBlockOffset >= 64*1024*1024 {
fmt.Print("Warning: maybe waste large memory on processing this apk! ")
fmt.Println("Before APK Signing Block bytes size is", signingBlockOffset/1024/1024, "MB")
}
beforeSigningBlock := make([]byte, signingBlockOffset)
n, err := in.ReadAt(beforeSigningBlock, 0)
if err != nil {
return
}
if int64(n) != signingBlockOffset {
return z, fmt.Errorf("Read bytes count mismatched! Expect %d, but %d", signingBlockOffset, n)
}
z.beforeSigningBlock = beforeSigningBlock
centralDir := make([]byte, centralDirSize)
n, err = in.ReadAt(centralDir, int64(centralDirOffset))
if uint32(n) != centralDirSize {
return z, fmt.Errorf("Read bytes count mismatched! Expect %d, but %d", centralDirSize, n)
}
z.centraDir = centralDir
if _debug {
fmt.Printf("signingBlockOffset=%d, signingBlockLenth=%d\n"+
"centralDirOffset=%d, centralDirSize=%d\n"+
"eocdOffset=%d, eocdLenthe=%d\n",
signingBlockOffset,
len(signingBlock),
centralDirOffset,
centralDirSize,
eocdOffset,
len(eocd))
}
return
}
func gen(info ChannelInfo, sections zipSections, output string, force bool) (err error) {
fi, err := os.Stat(output)
if err != nil && !os.IsNotExist(err) {
return err
}
if fi != nil {
if !force {
return fmt.Errorf("file already exists %s", output)
}
println("Force generating channel", info.Channel)
}
var s time.Time
if _debug {
s = time.Now()
}
err = sections.writeTo(output, newTransform(info))
if _debug {
fmt.Printf(" write %s consume %s", output, time.Since(s).String())
fmt.Println()
}
return
}
func newTransform(info ChannelInfo) transform {
return func(zip *zipSections) (*zipSections, error) {
newBlock, diffSize, err := makeSigningBlockWithChannelInfo(info, zip.signingBlock)
if err != nil {
return nil, err
}
newzip := new(zipSections)
newzip.beforeSigningBlock = zip.beforeSigningBlock
newzip.signingBlock = newBlock
newzip.signingBlockOffset = zip.signingBlockOffset
newzip.centraDir = zip.centraDir
newzip.centralDirOffset = zip.centralDirOffset
newzip.eocdOffset = zip.eocdOffset
newzip.eocd = makeEocd(zip.eocd, uint32(int64(diffSize)+zip.centralDirOffset))
return newzip, nil
}
}