diff options
| -rw-r--r-- | weed-fs/src/pkg/storage/needle.go | 60 | ||||
| -rw-r--r-- | weed-fs/src/pkg/storage/needle_read_write.go | 52 | ||||
| -rw-r--r-- | weed-fs/src/pkg/storage/volume.go | 3 | ||||
| -rw-r--r-- | weed-fs/src/pkg/storage/volume_version.go | 12 |
4 files changed, 73 insertions, 54 deletions
diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index 28dfbbed6..ffa9beb06 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -2,13 +2,10 @@ package storage import ( "encoding/hex" - "errors" "fmt" - "io" "io/ioutil" "mime" "net/http" - "os" "pkg/util" "strconv" "strings" @@ -89,53 +86,14 @@ func (n *Needle) ParsePath(fid string) { } } } -func (n *Needle) Append(w io.Writer) uint32 { - header := make([]byte, 16) - util.Uint32toBytes(header[0:4], n.Cookie) - util.Uint64toBytes(header[4:12], n.Id) - n.Size = uint32(len(n.Data)) - util.Uint32toBytes(header[12:16], n.Size) - w.Write(header) - w.Write(n.Data) - rest := 8 - ((n.Size + 16 + 4) % 8) - util.Uint32toBytes(header[0:4], n.Checksum.Value()) - w.Write(header[0 : rest+4]) - return n.Size -} -func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) { - bytes := make([]byte, size+16+4) - ret, e := r.Read(bytes) - n.Cookie = util.BytesToUint32(bytes[0:4]) - n.Id = util.BytesToUint64(bytes[4:12]) - n.Size = util.BytesToUint32(bytes[12:16]) - n.Data = bytes[16 : 16+size] - checksum := util.BytesToUint32(bytes[16+size : 16+size+4]) - if checksum != NewCRC(n.Data).Value() { - return 0, errors.New("CRC error! Data On Disk Corrupted!") - } - return ret, e -} -func ReadNeedle(r *os.File) (*Needle, uint32) { - n := new(Needle) - bytes := make([]byte, 16) - count, e := r.Read(bytes) - if count <= 0 || e != nil { - return nil, 0 - } - n.Cookie = util.BytesToUint32(bytes[0:4]) - n.Id = util.BytesToUint64(bytes[4:12]) - n.Size = util.BytesToUint32(bytes[12:16]) - rest := 8 - ((n.Size + 16 + 4) % 8) - return n, n.Size + 4 + rest -} func ParseKeyHash(key_hash_string string) (uint64, uint32) { - key_hash_bytes, khe := hex.DecodeString(key_hash_string) - key_hash_len := len(key_hash_bytes) - if khe != nil || key_hash_len <= 4 { - println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) - return 0, 0 - } - key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) - hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) - return key, hash + key_hash_bytes, khe := hex.DecodeString(key_hash_string) + key_hash_len := len(key_hash_bytes) + if khe != nil || key_hash_len <= 4 { + println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) + return 0, 0 + } + key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) + hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) + return key, hash } diff --git a/weed-fs/src/pkg/storage/needle_read_write.go b/weed-fs/src/pkg/storage/needle_read_write.go new file mode 100644 index 000000000..bd0bebd0b --- /dev/null +++ b/weed-fs/src/pkg/storage/needle_read_write.go @@ -0,0 +1,52 @@ +package storage + +import ( + "errors" + "io" + "os" + "pkg/util" +) + +func (n *Needle) Append(w io.Writer) uint32 { + header := make([]byte, 16) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) + n.Size = uint32(len(n.Data)) + util.Uint32toBytes(header[12:16], n.Size) + w.Write(header) + w.Write(n.Data) + rest := 8 - ((n.Size + 16 + 4) % 8) + util.Uint32toBytes(header[0:4], n.Checksum.Value()) + w.Write(header[0 : rest+4]) + return n.Size +} +func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) { + if version == Version1 { + bytes := make([]byte, size+16+4) + ret, e := r.Read(bytes) + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Id = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:16]) + n.Data = bytes[16 : 16+size] + checksum := util.BytesToUint32(bytes[16+size : 16+size+4]) + if checksum != NewCRC(n.Data).Value() { + return 0, errors.New("CRC error! Data On Disk Corrupted!") + } + return ret, e + }else if version == Version2 { + } + return 0, errors.New("Unsupported Version!") +} +func ReadNeedle(r *os.File) (*Needle, uint32) { + n := new(Needle) + bytes := make([]byte, 16) + count, e := r.Read(bytes) + if count <= 0 || e != nil { + return nil, 0 + } + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Id = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:16]) + rest := 8 - ((n.Size + 16 + 4) % 8) + return n, n.Size + 4 + rest +} diff --git a/weed-fs/src/pkg/storage/volume.go b/weed-fs/src/pkg/storage/volume.go index c1931d281..f5a810986 100644 --- a/weed-fs/src/pkg/storage/volume.go +++ b/weed-fs/src/pkg/storage/volume.go @@ -8,11 +8,8 @@ import ( "sync" ) -type Version uint8 - const ( SuperBlockSize = 8 - CurrentVersion = Version(1) ) type Volume struct { diff --git a/weed-fs/src/pkg/storage/volume_version.go b/weed-fs/src/pkg/storage/volume_version.go new file mode 100644 index 000000000..974a8079c --- /dev/null +++ b/weed-fs/src/pkg/storage/volume_version.go @@ -0,0 +1,12 @@ +package storage + +import ( +) + +type Version uint8 + +const ( + CurrentVersion = Version(1) + Version1 = Version(1) + Version2 = Version(2) +) |