Create & Init Project...

app/service/main/dapper/pkg/diskqueue/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "bucket_test.go",
+        "diskqueue_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bucket.go",
+        "diskqueue.go",
+    ],
+    importpath = "go-common/app/service/main/dapper/pkg/diskqueue",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/main/dapper/pkg/diskqueue/bucket.go

@@ -0,0 +1,200 @@
+package diskqueue
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+const (
+	_blockByte int32 = 512
+	_lenByte   int32 = 2
+	_dataByte        = _blockByte - _lenByte
+)
+
+var errBucketFull = errors.New("bucket is full or not enough")
+var fullHeader = []byte{1, 254}
+var nextHeader = []byte{1, 255}
+var magicHeader = []byte{'D', 'Q'}
+
+type memBucketPool struct {
+	cap  int32
+	pool sync.Pool
+}
+
+func newMemBucketPool(bucketByte int32) *memBucketPool {
+	return &memBucketPool{
+		pool: sync.Pool{New: func() interface{} {
+			return make([]byte, bucketByte)
+		}},
+		cap: bucketByte / _blockByte,
+	}
+}
+
+func (m *memBucketPool) new() *memBucket {
+	data := m.pool.Get().([]byte)
+	return &memBucket{data: data, cap: m.cap}
+}
+
+func (m *memBucketPool) free(bucket *memBucket) {
+	m.pool.Put(bucket.data)
+}
+
+type memBucket struct {
+	sync.Mutex
+	cap     int32
+	readAt  int32
+	writeAt int32
+	data    []byte
+}
+
+func (m *memBucket) push(p []byte) error {
+	m.Lock()
+	defer m.Unlock()
+	length := int32(len(p))
+	if length > _dataByte*(m.cap-m.writeAt) {
+		return errBucketFull
+	}
+	// if p length < blockbyte write it direct
+	if length < _dataByte {
+		ds := m.writeAt * _blockByte
+		binary.BigEndian.PutUint16(m.data[ds:], uint16(length))
+		copy(m.data[ds+_lenByte:], p)
+		m.writeAt++
+		return nil
+	}
+	// loop write block
+	blocks := length / _dataByte
+	re := length % _dataByte
+	var i int32
+	for i = 0; i < blocks-1; i++ {
+		ds := m.writeAt * _blockByte
+		copy(m.data[ds:], nextHeader)
+		ps := i * _dataByte
+		copy(m.data[ds+_lenByte:], p[ps:ps+_dataByte])
+		m.writeAt++
+	}
+	var nh []byte
+	if re == 0 {
+		nh = fullHeader
+	} else {
+		nh = nextHeader
+	}
+	ds := m.writeAt * _blockByte
+	copy(m.data[ds:], nh)
+	ps := (blocks - 1) * _dataByte
+	copy(m.data[ds+_lenByte:], p[ps:ps+_dataByte])
+	m.writeAt++
+	if re != 0 {
+		ds := m.writeAt * _blockByte
+		binary.BigEndian.PutUint16(m.data[ds:], uint16(re))
+		copy(m.data[ds+_lenByte:], p[blocks*_dataByte:])
+		m.writeAt++
+	}
+	return nil
+}
+
+func (m *memBucket) pop() ([]byte, error) {
+	m.Lock()
+	defer m.Unlock()
+	if m.readAt >= m.writeAt {
+		return nil, io.EOF
+	}
+	ret := make([]byte, 0, _blockByte)
+	for m.readAt < m.writeAt {
+		ds := m.readAt * _blockByte
+		m.readAt++
+		l := int32(binary.BigEndian.Uint16(m.data[ds : ds+_lenByte]))
+		if l <= _dataByte {
+			ret = append(ret, m.data[ds+_lenByte:ds+_lenByte+l]...)
+			break
+		}
+		ret = append(ret, m.data[ds+_lenByte:ds+_blockByte]...)
+	}
+	return ret, nil
+}
+
+func (m *memBucket) dump(w io.Writer) (int, error) {
+	header := make([]byte, 10)
+	copy(header, magicHeader)
+	binary.BigEndian.PutUint32(header[2:6], uint32(m.readAt))
+	binary.BigEndian.PutUint32(header[6:10], uint32(m.writeAt))
+	n1, err := w.Write(header)
+	if err != nil {
+		return n1, err
+	}
+	n2, err := w.Write(m.data[:m.writeAt*_blockByte])
+	return n1 + n2, err
+}
+
+func newFileBucket(fpath string) (*fileBucket, error) {
+	fp, err := os.Open(fpath)
+	if err != nil {
+		return nil, err
+	}
+	header := make([]byte, 10)
+	n, err := fp.Read(header)
+	if err != nil {
+		return nil, err
+	}
+	if n != 10 {
+		return nil, fmt.Errorf("expect read 10 byte header get: %d", n)
+	}
+	if !bytes.Equal(header[:2], magicHeader) {
+		return nil, fmt.Errorf("invalid magic %s", header[:2])
+	}
+	readAt := int32(binary.BigEndian.Uint32(header[2:6]))
+	writeAt := int32(binary.BigEndian.Uint32(header[6:10]))
+	if _, err = fp.Seek(int64(readAt*_blockByte), os.SEEK_CUR); err != nil {
+		return nil, err
+	}
+	return &fileBucket{
+		fp:      fp,
+		readAt:  readAt,
+		writeAt: writeAt,
+		bufRd:   bufio.NewReader(fp),
+	}, nil
+}
+
+type fileBucket struct {
+	sync.Mutex
+	fp      *os.File
+	readAt  int32
+	writeAt int32
+	bufRd   *bufio.Reader
+}
+
+func (f *fileBucket) pop() ([]byte, error) {
+	f.Lock()
+	defer f.Unlock()
+	if f.readAt >= f.writeAt {
+		return nil, io.EOF
+	}
+	ret := make([]byte, 0, _blockByte)
+	block := make([]byte, _blockByte)
+	for f.readAt < f.writeAt {
+		n, err := f.bufRd.Read(block)
+		if err != nil {
+			return nil, err
+		}
+		if int32(n) != _blockByte {
+			return nil, fmt.Errorf("expect read %d byte data get %d", _blockByte, n)
+		}
+		l := int32(binary.BigEndian.Uint16(block[:2]))
+		if l <= _dataByte {
+			ret = append(ret, block[2:2+l]...)
+			break
+		}
+		ret = append(ret, block[2:_blockByte]...)
+	}
+	return ret, nil
+}
+
+func (f *fileBucket) close() error {
+	return f.fp.Close()
+}

app/service/main/dapper/pkg/diskqueue/bucket_test.go

@@ -0,0 +1,105 @@
+package diskqueue
+
+import (
+	"crypto/rand"
+	"io"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func Test_membucket(t *testing.T) {
+	cap := int32(16)
+	data := make([]byte, _blockByte*cap)
+	mb := &memBucket{
+		cap:  cap,
+		data: data,
+	}
+	t.Run("test push & pop small data", func(t *testing.T) {
+		p := []byte("hello world")
+		err := mb.push(p)
+		if err != nil {
+			t.Error(err)
+		}
+		ret, err := mb.pop()
+		if err != nil {
+			t.Error(err)
+		} else {
+			if !reflect.DeepEqual(ret, p) {
+				t.Errorf("%s not equal %s", ret, p)
+			}
+		}
+	})
+	t.Run("test push & pop big data", func(t *testing.T) {
+		p := make([]byte, 1890)
+		rand.Read(p)
+		err := mb.push(p)
+		if err != nil {
+			t.Error(err)
+		}
+		ret, err := mb.pop()
+		if err != nil {
+			t.Error(err)
+		} else {
+			if !reflect.DeepEqual(ret, p) {
+				t.Logf("buf: %v", mb.data)
+				t.Errorf("%v not equal %v", ret, p)
+			}
+		}
+	})
+	t.Run("push big data", func(t *testing.T) {
+		p := make([]byte, _blockByte*cap*2)
+		err := mb.push(p)
+		if err != errBucketFull {
+			t.Errorf("expect err == errBucketFull get: %v", err)
+		}
+	})
+	t.Run("pop io.EOF", func(t *testing.T) {
+		_, err := mb.pop()
+		if err != io.EOF {
+			t.Errorf("expect err == io.EOF get: %v", err)
+		}
+	})
+}
+
+func Test_fileBucket(t *testing.T) {
+	fpath := "bucket.bin"
+	defer os.RemoveAll(fpath)
+	cap := int32(16)
+	data := make([]byte, _blockByte*cap)
+	mb := &memBucket{
+		cap:  cap,
+		data: data,
+	}
+	d1 := []byte("hello world")
+	for i := 0; i < 10; i++ {
+		mb.push(d1)
+	}
+	fp, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	mb.dump(fp)
+	fp.Close()
+	fb, err := newFileBucket(fpath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	count := 0
+	for {
+		ret, err := fb.pop()
+		if err != nil {
+			if err != io.EOF {
+				t.Error(err)
+			}
+			break
+		}
+		count++
+		if !reflect.DeepEqual(ret, d1) {
+			t.Errorf("%v not equal %v", ret, d1)
+		}
+	}
+	if count != 10 {
+		t.Errorf("expect 10 data get %d", count)
+	}
+}

app/service/main/dapper/pkg/diskqueue/diskqueue.go

@@ -0,0 +1,460 @@
+package diskqueue
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+const (
+	// max memroy use equal to BucketByte * (MemBucket + DynamicMemBucket)
+	_defaultBucketByte       = _blockByte * 2 * 1024 * 16 // 16MB
+	_defaultMemBucket        = 1
+	_defaultDynamicMemBucket = 3
+	_filePrefix              = "disk_queue_"
+)
+
+// node status
+const (
+	_inmem int8 = iota
+	_indisk
+	_freed
+)
+
+var _globalID int64
+
+// ErrQueueFull .
+var ErrQueueFull = errors.New("error queue is full, can't create new membucket")
+
+func nextNodeID() int64 {
+	return atomic.AddInt64(&_globalID, 1)
+}
+
+// DiskQueue disk queue
+type DiskQueue interface {
+	Push(p []byte) error
+	Pop() ([]byte, error)
+	Close() error
+}
+
+// Option Ringbuffer option
+type Option func(opt *option)
+
+// SetBucketByte bucketbyte
+func SetBucketByte(n int) Option {
+	return func(opt *option) {
+		opt.bucketByte = (int32(n) / _blockByte) * _blockByte
+	}
+}
+
+// SetMemBucket set the number of mem bucket
+func SetMemBucket(n int) Option {
+	return func(opt *option) {
+		opt.memBucket = int32(n)
+	}
+}
+
+// SetDynamicMemBucket set the number of dynamic mem bucket
+func SetDynamicMemBucket(n int) Option {
+	return func(opt *option) {
+		opt.dynamicMemBucket = int32(n)
+	}
+}
+
+// SetMaxBucket set the number of max bucket 0 represent unlimit
+func SetMaxBucket(n int) Option {
+	return func(opt *option) {
+		opt.maxBucket = int32(n)
+	}
+}
+
+type option struct {
+	bucketByte       int32
+	memBucket        int32
+	maxBucket        int32
+	dynamicMemBucket int32
+	fpath            string
+}
+
+func (o option) validate() error {
+	if o.bucketByte <= 0 {
+		return fmt.Errorf("bucket byte must > 0")
+	}
+	if o.memBucket <= 0 {
+		return fmt.Errorf("mem bucket must > 0")
+	}
+	if o.dynamicMemBucket <= 0 {
+		return fmt.Errorf("dynamic mem bucket must > 0")
+	}
+	return nil
+}
+
+var _defaultOpt = option{
+	bucketByte:       _defaultBucketByte,
+	memBucket:        _defaultMemBucket,
+	dynamicMemBucket: _defaultDynamicMemBucket,
+}
+
+// New Ringbuffer
+func New(fpath string, options ...Option) (DiskQueue, error) {
+	info, err := os.Stat(fpath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("stat %s error: %s", fpath, err)
+		}
+		if err = os.MkdirAll(fpath, 0755); err != nil {
+			return nil, fmt.Errorf("fpath %s not exists try create directry error: %s", fpath, err)
+		}
+	} else if !info.IsDir() {
+		return nil, fmt.Errorf("fpath: %s already exists and not a directory", fpath)
+	}
+	// TODO: check permission
+	opt := _defaultOpt
+	opt.fpath = fpath
+	for _, fn := range options {
+		fn(&opt)
+	}
+	if err = opt.validate(); err != nil {
+		return nil, err
+	}
+	b := &base{
+		opt: opt,
+	}
+	if opt.maxBucket == 0 {
+		return &queue{base: b}, b.init()
+	}
+	return nil, nil
+}
+
+type node struct {
+	id       int64
+	mx       sync.Mutex
+	flushing bool
+	bucket   *memBucket
+	next     *node
+	fpath    string
+	fbucket  *fileBucket
+	kind     int8
+}
+
+func (n *node) setFlushing(flushing bool) {
+	n.mx.Lock()
+	n.flushing = flushing
+	n.mx.Unlock()
+}
+
+func (n *node) pop() ([]byte, error) {
+	n.mx.Lock()
+	defer n.mx.Unlock()
+	if n.bucket != nil {
+		return n.bucket.pop()
+	}
+	var err error
+	if n.fbucket == nil {
+		if n.fbucket, err = newFileBucket(n.fpath); err != nil {
+			return nil, err
+		}
+	}
+	return n.fbucket.pop()
+}
+
+type base struct {
+	opt       option
+	head      *node
+	tail      *node
+	pool      *memBucketPool
+	length    int32
+	memBucket int32
+}
+
+func (b *base) init() error {
+	b.pool = newMemBucketPool(b.opt.bucketByte)
+	if loaded, err := b.loadFromFile(); err != nil || loaded {
+		return err
+	}
+	current := &node{
+		id:     nextNodeID(),
+		bucket: b.pool.new(),
+	}
+	b.head = current
+	b.tail = current
+	return nil
+}
+
+func (b *base) loadFromFile() (bool, error) {
+	infos, err := ioutil.ReadDir(b.opt.fpath)
+	if err != nil {
+		return false, fmt.Errorf("readdir %s error: %s", b.opt.fpath, err)
+	}
+	var files []string
+	for _, info := range infos {
+		if info.IsDir() || !strings.HasPrefix(info.Name(), _filePrefix) {
+			continue
+		}
+		files = append(files, path.Join(b.opt.fpath, info.Name()))
+	}
+	if len(files) == 0 {
+		return false, nil
+	}
+	nodeID := func(name string) int64 {
+		id, err := strconv.ParseInt(path.Base(name)[len(_filePrefix):], 10, 64)
+		if err != nil {
+			panic(fmt.Errorf("invalid file name: %s error: %s", name, err))
+		}
+		return id
+	}
+	sort.Slice(files, func(i int, j int) bool {
+		return nodeID(files[i]) < nodeID(files[j])
+	})
+	_globalID = nodeID(files[len(files)-1])
+	current := &node{
+		id:    nodeID(files[0]),
+		fpath: files[0],
+		kind:  _indisk,
+	}
+	b.head = current
+	for _, file := range files[1:] {
+		next := &node{
+			id:    nodeID(file),
+			fpath: file,
+			kind:  _indisk,
+		}
+		current.next = next
+		current = next
+	}
+	b.memBucket = 1
+	next := &node{
+		id:     nextNodeID(),
+		bucket: b.pool.new(),
+	}
+	current.next = next
+	current = next
+	b.tail = current
+	return true, nil
+}
+
+type queue struct {
+	*base
+	mx     sync.Mutex
+	closed bool
+	lastID int64
+	wg     sync.WaitGroup
+}
+
+func (q *queue) Push(p []byte) (err error) {
+	if len(p) >= int(q.opt.bucketByte) {
+		return fmt.Errorf("data too large")
+	}
+	if q.closed {
+		return fmt.Errorf("queue already closed")
+	}
+	for {
+		err = q.tail.bucket.push(p)
+		if err == nil {
+			atomic.AddInt32(&q.length, 1)
+			return
+		}
+		if err == errBucketFull {
+			if err = q.moveTail(); err != nil {
+				return err
+			}
+			continue
+		}
+		return
+	}
+}
+
+func (q *queue) moveTail() error {
+	bucket := atomic.LoadInt32(&q.memBucket)
+	if bucket >= q.opt.memBucket+q.opt.dynamicMemBucket {
+		return fmt.Errorf("can't assign memory bucket any more")
+	}
+	if bucket >= q.opt.maxBucket {
+		q.notifyStore()
+	}
+	// take tail snapshot
+	p := q.tail
+	// lock queue
+	q.mx.Lock()
+	defer q.mx.Unlock()
+	// tail alreay changed
+	if p != q.tail {
+		return nil
+	}
+	atomic.AddInt32(&q.memBucket, 1)
+	n := &node{
+		id:     nextNodeID(),
+		bucket: q.pool.new(),
+		kind:   _inmem,
+	}
+	// move to new tail
+	q.tail.next = n
+	q.tail = n
+	return nil
+}
+
+func (q *queue) notifyStore() {
+	n := q.head
+	for n.next != nil {
+		read := q.head
+		if n.id > q.lastID && n.kind != _indisk && n != read {
+			q.lastID = n.id
+			go q.storeNode(n)
+			return
+		}
+		n = n.next
+	}
+}
+
+func (q *queue) Pop() (data []byte, err error) {
+	defer func() {
+		if err != nil {
+			atomic.AddInt32(&q.length, -1)
+		}
+	}()
+	if q.closed {
+		return nil, fmt.Errorf("queue already closed")
+	}
+	data, err = q.head.pop()
+	if err != nil {
+		if err == io.EOF {
+			if err = q.moveHead(); err != nil {
+				return nil, err
+			}
+			return q.head.pop()
+		}
+		return nil, err
+	}
+	return data, nil
+}
+
+func (q *queue) moveHead() error {
+	tail := q.tail
+	if q.head == tail {
+		return io.EOF
+	}
+	// move head to next
+	q.mx.Lock()
+	head := q.head
+	q.head = q.head.next
+	q.mx.Unlock()
+	// reset head to new read node
+	q.freeNode(head)
+	return nil
+}
+
+func (q *queue) freeNode(n *node) {
+	n.mx.Lock()
+	defer n.mx.Unlock()
+	if n.flushing {
+		n.kind = _freed
+		return
+	}
+	if n.bucket != nil {
+		q.freeBucket(n.bucket)
+		n.bucket = nil
+	}
+	if n.fbucket != nil {
+		n.fbucket.close()
+	}
+	if n.fpath != "" {
+		if err := os.Remove(n.fpath); err != nil {
+			//fmt.Fprintf(os.Stderr, "[ERROR] diskqueue: remove file %s error: %s", n.fpath, err)
+		}
+	}
+}
+
+func (q *queue) storeNode(n *node) (err error) {
+	fpath := storePath(q.opt.fpath, n)
+	q.wg.Add(1)
+	defer q.wg.Done()
+	n.setFlushing(true)
+	// if node already free return direct
+	if n.bucket == nil {
+		return
+	}
+	// if node be freed just release membucket
+	if n.kind == _freed {
+		q.freeBucket(n.bucket)
+		return
+	}
+	// store bucket to disk
+	if err = store(fpath, n); err != nil {
+		fmt.Fprintf(os.Stderr, "[ERROR] diskqueue: store node error: %s", err)
+	}
+	n.fpath = fpath
+	n.setFlushing(false)
+	if n.kind == _freed {
+		q.freeBucket(n.bucket)
+		n.bucket = nil
+		if err := os.Remove(fpath); err != nil {
+			//fmt.Fprintf(os.Stderr, "[ERROR] diskqueue: remove file %s error: %s", n.fpath, err)
+		}
+		return
+	}
+	n.kind = _indisk
+	q.mx.Lock()
+	if q.head != n {
+		q.freeBucket(n.bucket)
+		n.bucket = nil
+	}
+	q.mx.Unlock()
+	return
+}
+
+func (q *queue) freeBucket(bucket *memBucket) {
+	q.pool.free(bucket)
+	atomic.AddInt32(&q.memBucket, -1)
+}
+
+func (q *queue) Close() error {
+	// set closed
+	q.closed = true
+	// wait all store goroutines finish
+	q.wg.Wait()
+	var messages []string
+	// store all leave node
+	current := q.head
+	for current != nil {
+		if current.kind == _inmem && current.bucket != nil {
+			fpath := storePath(q.opt.fpath, current)
+			if err := store(fpath, current); err != nil {
+				messages = append(messages, err.Error())
+			}
+		}
+		current = current.next
+	}
+	if len(messages) == 0 {
+		return nil
+	}
+	return fmt.Errorf("close queue error: %s", strings.Join(messages, "; "))
+}
+
+func store(fpath string, n *node) (err error) {
+	// ignore empty bucket
+	if n.bucket.writeAt == n.bucket.readAt {
+		return nil
+	}
+	var fp *os.File
+	fp, err = os.OpenFile(fpath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("open file %s error: %s", fpath, err)
+	}
+	_, err = n.bucket.dump(fp)
+	if err != nil {
+		return fmt.Errorf("dump data to file %s error: %s", fpath, err)
+	}
+	return
+}
+
+func storePath(base string, n *node) string {
+	return path.Join(base, _filePrefix+strconv.FormatInt(n.id, 10))
+}

app/service/main/dapper/pkg/diskqueue/diskqueue_test.go

@@ -0,0 +1,206 @@
+package diskqueue
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	mrand "math/rand"
+	"os"
+	"os/exec"
+	"sync"
+	"testing"
+	"time"
+)
+
+func init() {
+	mrand.Seed(time.Now().UnixNano())
+}
+
+func TestDiskQueuePushPopMem(t *testing.T) {
+	dirname := "testdata/d1"
+	defer os.RemoveAll(dirname)
+	queue, err := New(dirname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	N := 10
+	p := []byte("hello world")
+	for i := 0; i < N; i++ {
+		if err := queue.Push(p); err != nil {
+			t.Error(err)
+		}
+	}
+	count := 0
+	for {
+		data, err := queue.Pop()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Error(err)
+		}
+		if !bytes.Equal(data, p) {
+			t.Errorf("invalid data: %s", data)
+		}
+		count++
+	}
+	if count != N {
+		t.Errorf("wrong count %d", count)
+	}
+}
+
+func TestDiskQueueDisk(t *testing.T) {
+	data := make([]byte, 2233)
+	rand.Read(data)
+	count := 1024 * 256
+	dirname := "testdata/d2"
+	defer os.RemoveAll(dirname)
+	t.Run("test write disk", func(t *testing.T) {
+		queue, err := New(dirname)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := 0; i < count; i++ {
+			if err := queue.Push(data); err != nil {
+				time.Sleep(time.Second)
+				if err := queue.Push(data); err != nil {
+					t.Error(err)
+				}
+			}
+		}
+		queue.Close()
+	})
+	t.Run("test read disk", func(t *testing.T) {
+		n := 0
+		queue, err := New(dirname)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for {
+			ret, err := queue.Pop()
+			if err == io.EOF {
+				break
+			}
+			if !bytes.Equal(data, ret) {
+				t.Errorf("invalid data unequal")
+			}
+			n++
+		}
+		if n != count {
+			t.Errorf("want %d get %d", count, n)
+		}
+	})
+}
+
+func TestDiskQueueTrans(t *testing.T) {
+	dirname := "testdata/d3"
+	defer os.RemoveAll(dirname)
+	queue, err := New(dirname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data := make([]byte, 1890)
+	rand.Read(data)
+	cycles := 512
+	var wg sync.WaitGroup
+	wg.Add(2)
+	done := false
+	writed := 0
+	readed := 0
+	go func() {
+		defer wg.Done()
+		for i := 0; i < cycles; i++ {
+			ms := mrand.Intn(40) + 10
+			time.Sleep(time.Duration(ms) * time.Millisecond)
+			for i := 0; i < 128; i++ {
+				if err := queue.Push(data); err != nil {
+					t.Error(err)
+				} else {
+					writed++
+				}
+			}
+		}
+		done = true
+	}()
+	go func() {
+		defer wg.Done()
+		for {
+			ret, err := queue.Pop()
+			if err == io.EOF && done {
+				break
+			}
+			if err == io.EOF {
+				ms := mrand.Intn(10)
+				time.Sleep(time.Duration(ms) * time.Millisecond)
+				continue
+			}
+			if !bytes.Equal(ret, data) {
+				t.Fatalf("invalid data, data length: %d, want: %d, data: %v, want: %v", len(ret), len(data), ret, data)
+			}
+			readed++
+		}
+	}()
+	wg.Wait()
+	os.RemoveAll(dirname)
+	if writed != readed {
+		t.Errorf("readed: %d != writed: %d", readed, writed)
+	}
+}
+
+func TestEmpty(t *testing.T) {
+	dirname := "testdata/d4"
+	defer os.RemoveAll(dirname)
+	queue, err := New(dirname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 5; i++ {
+		_, err := queue.Pop()
+		if err != io.EOF {
+			t.Errorf("expect err == io.EOF, get %v", err)
+		}
+	}
+}
+
+func TestEmptyCache(t *testing.T) {
+	datadir := "testdata/emptycache"
+	dirname := "testdata/de"
+	if err := exec.Command("cp", "-r", datadir, dirname).Run(); err != nil {
+		t.Error(err)
+	}
+	defer os.RemoveAll(dirname)
+	queue, err := New(dirname)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 5; i++ {
+		_, err := queue.Pop()
+		if err != io.EOF {
+			t.Errorf("expect err == io.EOF, get %v", err)
+		}
+	}
+}
+
+func BenchmarkDiskQueue(b *testing.B) {
+	queue, err := New("testdata/d5")
+	if err != nil {
+		b.Fatal(err)
+	}
+	done := make(chan bool, 1)
+	go func() {
+		for {
+			if _, err := queue.Pop(); err != nil {
+				if err == io.EOF {
+					break
+				}
+			}
+		}
+		done <- true
+	}()
+	data := make([]byte, 768)
+	rand.Read(data)
+	for i := 0; i < b.N; i++ {
+		queue.Push(data)
+	}
+	<-done
+}