diff options
| -rw-r--r-- | weed/storage/needle_read_write.go | 9 | ||||
| -rw-r--r-- | weed/storage/needle_read_write_test.go | 62 | ||||
| -rw-r--r-- | weed/storage/volume_read_write.go | 8 |
3 files changed, 71 insertions, 8 deletions
diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go index 6441cb501..c99395f8b 100644 --- a/weed/storage/needle_read_write.go +++ b/weed/storage/needle_read_write.go @@ -3,6 +3,7 @@ package storage import ( "errors" "fmt" + "io" "os" "github.com/chrislusf/seaweedfs/weed/glog" @@ -27,8 +28,8 @@ func (n *Needle) DiskSize(version Version) int64 { return getActualSize(n.Size, version) } -func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32, actualSize int64, err error) { - if end, e := w.Seek(0, 2); e == nil { +func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) { + if end, e := w.Seek(0, io.SeekEnd); e == nil { defer func(w *os.File, off int64) { if err != nil { if te := w.Truncate(end); te != nil { @@ -36,7 +37,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32 } } }(w, end) - offset = Offset(end) + offset = uint64(end) } else { err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) return @@ -175,7 +176,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version } n.ParseNeedleHeader(bytes) if n.Size != size { - return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size) + return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size) } switch version { case Version1: diff --git a/weed/storage/needle_read_write_test.go b/weed/storage/needle_read_write_test.go new file mode 100644 index 000000000..8bd9205c1 --- /dev/null +++ b/weed/storage/needle_read_write_test.go @@ -0,0 +1,62 @@ +package storage + +import ( + "crypto/rand" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestAppend(t *testing.T) { + n := &Needle{ + + Cookie: types.Cookie(123), // Cookie Cookie `comment:"random number to mitigate brute force lookups"` + Id: types.NeedleId(123), // Id NeedleId `comment:"needle id"` + Size: 8, // Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"` + DataSize: 4, // DataSize uint32 `comment:"Data size"` //version2 + Data: []byte("abcd"), // Data []byte `comment:"The actual file data"` + Flags: 0, // Flags byte `comment:"boolean flags"` //version2 + NameSize: 0, // NameSize uint8 //version2 + Name: nil, // Name []byte `comment:"maximum 256 characters"` //version2 + MimeSize: 0, // MimeSize uint8 //version2 + Mime: nil, // Mime []byte `comment:"maximum 256 characters"` //version2 + PairsSize: 0, // PairsSize uint16 //version2 + Pairs: nil, // Pairs []byte `comment:"additional name value pairs, json format, maximum 6 + LastModified: 123, // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes + Ttl: nil, // Ttl *TTL + Checksum: 123, // Checksum CRC `comment:"CRC32 to check integrity"` + AppendAtNs: 123, // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3 + Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"` + } + + tempFile, err := ioutil.TempFile("", ".dat") + if err != nil { + t.Errorf("Fail TempFile. %v", err) + return + } + + /* + uint8 : 0 to 255 + uint16 : 0 to 65535 + uint32 : 0 to 4294967295 + uint64 : 0 to 18446744073709551615 + int8 : -128 to 127 + int16 : -32768 to 32767 + int32 : -2147483648 to 2147483647 + int64 : -9223372036854775808 to 9223372036854775807 + */ + + fileSize := int64(4294967295) + 10000 + io.CopyN(tempFile, rand.Reader, fileSize) + defer func() { + tempFile.Close() + os.Remove(tempFile.Name()) + }() + + offset, _, _, _ := n.Append(tempFile, CurrentVersion) + if offset != uint64(fileSize) { + t.Errorf("Fail to Append Needle.") + } +} diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index 32062db46..66f90634f 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -76,7 +76,7 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) { return } -func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) { +func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) { glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String()) if v.readOnly { err = fmt.Errorf("%s is read-only", v.dataFile.Name()) @@ -96,8 +96,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) } nv, ok := v.nm.Get(n.Id) - if !ok || Offset(nv.Offset)*NeedlePaddingSize < offset { - if err = v.nm.Put(n.Id, offset/NeedlePaddingSize, n.Size); err != nil { + if !ok || uint64(nv.Offset)*NeedlePaddingSize < offset { + if err = v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size); err != nil { glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } } @@ -124,7 +124,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) { if err != nil { return size, err } - if err = v.nm.Delete(n.Id, offset/NeedlePaddingSize); err != nil { + if err = v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize)); err != nil { return size, err } return size, err |