Merge pull request #2 from chrislusf/master

merge

17 files changed, 80 insertions, 41 deletions

diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go
index 1869a563e..fbb59e9c0 100644
--- a/go/storage/cdb_map.go
+++ b/go/storage/cdb_map.go
@@ -1,13 +1,14 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/util"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/tgulacsi/go-cdb"
 	"os"
 	"path/filepath"
+
+	"github.com/chrislusf/weed-fs/go/util"
+	"github.com/tgulacsi/go-cdb"
 )
 
 // CDB-backed read-only needle map
diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go
index cff7dfa61..ed690f44f 100644
--- a/go/storage/cdb_map_test.go
+++ b/go/storage/cdb_map_test.go
@@ -1,11 +1,12 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
 	"math/rand"
 	"os"
 	"runtime"
 	"testing"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 var testIndexFilename string = "../../test/sample.idx"
diff --git a/go/storage/compact_map.go b/go/storage/compact_map.go
index 9cfc3e8f7..6ac18b012 100644
--- a/go/storage/compact_map.go
+++ b/go/storage/compact_map.go
@@ -1,7 +1,5 @@
 package storage
 
-import ()
-
 type NeedleValue struct {
 	Key    Key
 	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go
index ef43de25b..f74684225 100644
--- a/go/storage/compact_map_perf_test.go
+++ b/go/storage/compact_map_perf_test.go
@@ -1,11 +1,12 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
 	"log"
 	"os"
 	"testing"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 func TestMemoryUsage(t *testing.T) {
diff --git a/go/storage/compress.go b/go/storage/compress.go
index a353c9d3a..5efc9e1f1 100644
--- a/go/storage/compress.go
+++ b/go/storage/compress.go
@@ -2,11 +2,12 @@ package storage
 
 import (
 	"bytes"
-	"github.com/chrislusf/weed-fs/go/glog"
 	"compress/flate"
 	"compress/gzip"
 	"io/ioutil"
 	"strings"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 /*
diff --git a/go/storage/crc.go b/go/storage/crc.go
index 7aa400959..af25b9e53 100644
--- a/go/storage/crc.go
+++ b/go/storage/crc.go
@@ -1,9 +1,10 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/util"
 	"fmt"
 	"hash/crc32"
+
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 var table = crc32.MakeTable(crc32.Castagnoli)
diff --git a/go/storage/file_id.go b/go/storage/file_id.go
index ec566826c..f6e36a98c 100644
--- a/go/storage/file_id.go
+++ b/go/storage/file_id.go
@@ -1,11 +1,12 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
 	"encoding/hex"
 	"errors"
 	"strings"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 type FileId struct {
diff --git a/go/storage/needle.go b/go/storage/needle.go
index daede321b..11610dd80 100644
--- a/go/storage/needle.go
+++ b/go/storage/needle.go
@@ -1,11 +1,9 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/images"
-	"github.com/chrislusf/weed-fs/go/util"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"mime"
 	"net/http"
@@ -13,6 +11,10 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/images"
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 const (
@@ -23,6 +25,7 @@ const (
 )
 
 /*
+* A Needle means a uploaded and stored file.
 * Needle file size is limited to 4GB for now.
  */
 type Needle struct {
@@ -44,6 +47,11 @@ type Needle struct {
 	Padding  []byte `comment:"Aligned to 8 bytes"`
 }
 
+func (n *Needle) String() (str string) {
+	str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
+	return
+}
+
 func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
 	form, fe := r.MultipartReader()
 	if fe != nil {
diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go
index dca2e6c5d..504ca1552 100644
--- a/go/storage/needle_map.go
+++ b/go/storage/needle_map.go
@@ -1,11 +1,12 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
 	"fmt"
 	"io"
 	"os"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 type NeedleMapper interface {
diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go
index bf452ba37..663b5abbd 100644
--- a/go/storage/needle_read_write.go
+++ b/go/storage/needle_read_write.go
@@ -1,12 +1,13 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/util"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/util"
 )
 
 const (
diff --git a/go/storage/store.go b/go/storage/store.go
index e7a9dac94..65eed1d0e 100644
--- a/go/storage/store.go
+++ b/go/storage/store.go
@@ -1,10 +1,6 @@
 package storage
 
 import (
-	proto "code.google.com/p/goprotobuf/proto"
-	"github.com/chrislusf/weed-fs/go/glog"
-	"github.com/chrislusf/weed-fs/go/operation"
-	"github.com/chrislusf/weed-fs/go/util"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -12,6 +8,11 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
+
+	"github.com/chrislusf/weed-fs/go/glog"
+	"github.com/chrislusf/weed-fs/go/operation"
+	"github.com/chrislusf/weed-fs/go/util"
+	"github.com/golang/protobuf/proto"
 )
 
 const (
@@ -32,6 +33,10 @@ type MasterNodes struct {
 	lastNode int
 }
 
+func (mn *MasterNodes) String() string {
+	return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode)
+}
+
 func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) {
 	mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1}
 	return
@@ -64,6 +69,9 @@ func (mn *MasterNodes) findMaster() (string, error) {
 	return mn.nodes[mn.lastNode], nil
 }
 
+/*
+ * A VolumeServer contains one Store
+ */
 type Store struct {
 	Port            int
 	Ip              string
@@ -76,6 +84,11 @@ type Store struct {
 	masterNodes     *MasterNodes
 }
 
+func (s *Store) String() (str string) {
+	str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes)
+	return
+}
+
 func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int) (s *Store) {
 	s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
 	s.Locations = make([]*DiskLocation, 0)
diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go
index 3527e4f59..209e3b4b3 100644
--- a/go/storage/store_vacuum.go
+++ b/go/storage/store_vacuum.go
@@ -1,9 +1,10 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
 	"fmt"
 	"strconv"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
diff --git a/go/storage/volume.go b/go/storage/volume.go
index de79e9107..a1eccd62c 100644
--- a/go/storage/volume.go
+++ b/go/storage/volume.go
@@ -2,7 +2,6 @@ package storage
 
 import (
 	"bytes"
-	"github.com/chrislusf/weed-fs/go/glog"
 	"errors"
 	"fmt"
 	"io"
@@ -10,6 +9,8 @@ import (
 	"path"
 	"sync"
 	"time"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 type Volume struct {
@@ -32,6 +33,10 @@ func NewVolume(dirname string, collection string, id VolumeId, replicaPlacement
 	e = v.load(true, true)
 	return
 }
+func (v *Volume) String() string {
+	return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly)
+}
+
 func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId) (v *Volume, e error) {
 	v = &Volume{dir: dirname, Collection: collection, Id: id}
 	v.SuperBlock = SuperBlock{}
@@ -72,7 +77,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error {
 
 	if e != nil {
 		if !os.IsPermission(e) {
-			return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error())
+			return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e)
 		}
 	}
 
@@ -92,12 +97,12 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error {
 		if v.readOnly {
 			glog.V(1).Infoln("open to read file", fileName+".idx")
 			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
-				return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error())
+				return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e)
 			}
 		} else {
 			glog.V(1).Infoln("open to write file", fileName+".idx")
 			if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
-				return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error())
+				return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e)
 			}
 		}
 		glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly)
@@ -115,7 +120,7 @@ func (v *Volume) Size() int64 {
 	if e == nil {
 		return stat.Size()
 	}
-	glog.V(0).Infof("Failed to read file size %s %s", v.dataFile.Name(), e.Error())
+	glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e)
 	return -1
 }
 func (v *Volume) Close() {
@@ -134,7 +139,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
 		oldNeedle := new(Needle)
 		oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
 		if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
-			n.Size = oldNeedle.Size
+			n.DataSize = oldNeedle.DataSize
 			return true
 		}
 	}
@@ -164,12 +169,13 @@ func (v *Volume) write(n *Needle) (size uint32, err error) {
 	v.accessLock.Lock()
 	defer v.accessLock.Unlock()
 	if v.isFileUnchanged(n) {
-		size = n.Size
+		size = n.DataSize
 		glog.V(4).Infof("needle is unchanged!")
 		return
 	}
 	var offset int64
 	if offset, err = v.dataFile.Seek(0, 2); err != nil {
+		glog.V(0).Infof("faile to seek the end of file: %v", err)
 		return
 	}
 
@@ -177,21 +183,21 @@ func (v *Volume) write(n *Needle) (size uint32, err error) {
 	if offset%NeedlePaddingSize != 0 {
 		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
 		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
-			glog.V(4).Infof("failed to align in datafile %s: %s", v.dataFile.Name(), err.Error())
+			glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
 			return
 		}
 	}
 
 	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
 		if e := v.dataFile.Truncate(offset); e != nil {
-			err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile.Name(), e.Error())
+			err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e)
 		}
 		return
 	}
 	nv, ok := v.nm.Get(n.Id)
 	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
 		if _, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil {
-			glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error())
+			glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
 		}
 	}
 	if v.lastModifiedTime < n.LastModified {
@@ -292,13 +298,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
 	offset := int64(SuperBlockSize)
 	n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
 	if e != nil {
-		err = fmt.Errorf("cannot read needle header: %s", e)
+		err = fmt.Errorf("cannot read needle header: %v", e)
 		return
 	}
 	for n != nil {
 		if readNeedleBody {
 			if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
-				err = fmt.Errorf("cannot read needle body: %s", err)
+				err = fmt.Errorf("cannot read needle body: %v", err)
 				return
 			}
 		}
@@ -310,7 +316,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
 			if err == io.EOF {
 				return nil
 			}
-			return fmt.Errorf("cannot read needle header: %s", err)
+			return fmt.Errorf("cannot read needle header: %v", err)
 		}
 	}
 
@@ -360,7 +366,7 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) {
 	defer indexFile.Close()
 	glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName)
 	if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil {
-		glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error())
+		glog.V(0).Infof("error converting %s.idx to %s.cdb: %v", fileName, fileName, e)
 		return false
 	}
 	return true
diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go
index 6410c1784..bc8049ea4 100644
--- a/go/storage/volume_info.go
+++ b/go/storage/volume_info.go
@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"fmt"
 	"github.com/chrislusf/weed-fs/go/operation"
 )
 
@@ -36,3 +37,7 @@ func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err er
 	vi.Ttl = LoadTTLFromUint32(*m.Ttl)
 	return vi, nil
 }
+
+func (vi VolumeInfo) String() string {
+	return fmt.Sprintf("Id:%s, Size:%d, ReplicaPlacement:%s, Collection:%s, Version:%v, FileCount:%d, DeleteCount:%d, DeletedByteCount:%d, ReadOnly:%v", vi.Id, vi.Size, vi.ReplicaPlacement, vi.Collection, vi.Version, vi.FileCount, vi.DeleteCount, vi.DeletedByteCount, vi.ReadOnly)
+}
diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go
index a7e86b1c3..57e0deea9 100644
--- a/go/storage/volume_super_block.go
+++ b/go/storage/volume_super_block.go
@@ -1,9 +1,10 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
 	"fmt"
 	"os"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 const (
diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go
index b348434d2..7e026a61d 100644
--- a/go/storage/volume_vacuum.go
+++ b/go/storage/volume_vacuum.go
@@ -1,10 +1,11 @@
 package storage
 
 import (
-	"github.com/chrislusf/weed-fs/go/glog"
 	"fmt"
 	"os"
 	"time"
+
+	"github.com/chrislusf/weed-fs/go/glog"
 )
 
 func (v *Volume) garbageLevel() float64 {
diff --git a/go/storage/volume_version.go b/go/storage/volume_version.go
index 9702ae904..2e9f58aa2 100644
--- a/go/storage/volume_version.go
+++ b/go/storage/volume_version.go
@@ -1,7 +1,5 @@
 package storage
 
-import ()
-
 type Version uint8
 
 const (