aboutsummaryrefslogtreecommitdiff
path: root/go/storage
diff options
context:
space:
mode:
Diffstat (limited to 'go/storage')
-rw-r--r--go/storage/cdb_map.go3
-rw-r--r--go/storage/compact_map_perf_test.go2
-rw-r--r--go/storage/compress.go2
-rw-r--r--go/storage/needle.go21
-rw-r--r--go/storage/needle_map.go22
-rw-r--r--go/storage/needle_read_write.go2
-rw-r--r--go/storage/store.go8
-rw-r--r--go/storage/volume.go2
8 files changed, 40 insertions, 22 deletions
diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go
index 451d8727a..a5480ac11 100644
--- a/go/storage/cdb_map.go
+++ b/go/storage/cdb_map.go
@@ -76,6 +76,9 @@ func (m cdbMap) FileCount() int {
func (m *cdbMap) DeletedCount() int {
return m.DeletionCounter
}
+func (m *cdbMap) NextFileKey(count int) (uint64) {
+ return 0
+}
func getMetric(c *cdb.Cdb, m *mapMetric) error {
data, err := c.Data([]byte{'M'})
diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go
index d97e7a66b..460085dee 100644
--- a/go/storage/compact_map_perf_test.go
+++ b/go/storage/compact_map_perf_test.go
@@ -1,8 +1,8 @@
package storage
import (
- "code.google.com/p/weed-fs/go/util"
"code.google.com/p/weed-fs/go/glog"
+ "code.google.com/p/weed-fs/go/util"
"os"
"testing"
)
diff --git a/go/storage/compress.go b/go/storage/compress.go
index eaf3c9ad8..b557b3411 100644
--- a/go/storage/compress.go
+++ b/go/storage/compress.go
@@ -2,9 +2,9 @@ package storage
import (
"bytes"
+ "code.google.com/p/weed-fs/go/glog"
"compress/flate"
"compress/gzip"
- "code.google.com/p/weed-fs/go/glog"
"io/ioutil"
"strings"
)
diff --git a/go/storage/needle.go b/go/storage/needle.go
index 54799df4e..e23f4e67b 100644
--- a/go/storage/needle.go
+++ b/go/storage/needle.go
@@ -1,10 +1,10 @@
package storage
import (
+ "code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"encoding/hex"
"io/ioutil"
- "code.google.com/p/weed-fs/go/glog"
"mime"
"net/http"
"path"
@@ -86,19 +86,19 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
fileName = fileName[:len(fileName)-3]
}
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
- return
+ return
}
func NewNeedle(r *http.Request) (n *Needle, e error) {
- fname, mimeType, isGzipped := "", "", false
+ fname, mimeType, isGzipped := "", "", false
n = new(Needle)
fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r)
if e != nil {
- return
+ return
+ }
+ if len(fname) < 256 {
+ n.Name = []byte(fname)
+ n.SetHasName()
}
- if len(fname) < 256 {
- n.Name = []byte(fname)
- n.SetHasName()
- }
if len(mimeType) < 256 {
n.Mime = []byte(mimeType)
n.SetHasMime()
@@ -108,7 +108,7 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
}
if n.LastModified == 0 {
n.LastModified = uint64(time.Now().Unix())
- n.SetHasLastModifiedDate()
+ n.SetHasLastModifiedDate()
}
n.Checksum = NewCRC(n.Data)
@@ -127,9 +127,6 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
func (n *Needle) ParsePath(fid string) {
length := len(fid)
if length <= 8 {
- if length > 0 {
- glog.V(0).Infoln("Invalid fid", fid, "length", length)
- }
return
}
delta := ""
diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go
index a242cd673..29b71ae52 100644
--- a/go/storage/needle_map.go
+++ b/go/storage/needle_map.go
@@ -2,6 +2,7 @@ package storage
import (
"bufio"
+ "code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"fmt"
"io"
@@ -18,6 +19,7 @@ type NeedleMapper interface {
FileCount() int
DeletedCount() int
Visit(visit func(NeedleValue) error) (err error)
+ NextFileKey(count int) uint64
}
type mapMetric struct {
@@ -25,6 +27,7 @@ type mapMetric struct {
FileCounter int `json:"FileCounter"`
DeletionByteCounter uint64 `json:"DeletionByteCounter"`
FileByteCounter uint64 `json:"FileByteCounter"`
+ MaximumFileKey uint64 `json:"MaxFileKey"`
}
type NeedleMap struct {
@@ -53,23 +56,27 @@ const (
func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewNeedleMap(file)
e := walkIndexFile(file, func(key uint64, offset, size uint32) error {
+ if key > nm.MaximumFileKey {
+ nm.MaximumFileKey = key
+ }
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
if offset > 0 {
oldSize := nm.m.Set(Key(key), offset, size)
- //glog.V(0).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
+ glog.V(4).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
if oldSize > 0 {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
} else {
oldSize := nm.m.Delete(Key(key))
- //glog.V(0).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
+ glog.V(4).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
return nil
})
+ glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
return nm, e
}
@@ -163,3 +170,14 @@ func (nm NeedleMap) DeletedCount() int {
func (nm *NeedleMap) Visit(visit func(NeedleValue) error) (err error) {
return nm.m.Visit(visit)
}
+func (nm NeedleMap) MaxFileKey() uint64 {
+ return nm.MaximumFileKey
+}
+func (nm NeedleMap) NextFileKey(count int) (ret uint64) {
+ if count <= 0 {
+ return 0
+ }
+ ret = nm.MaximumFileKey
+ nm.MaximumFileKey += uint64(count)
+ return
+}
diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go
index 0b833c0a9..d525a10d5 100644
--- a/go/storage/needle_read_write.go
+++ b/go/storage/needle_read_write.go
@@ -1,11 +1,11 @@
package storage
import (
+ "code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"errors"
"fmt"
"io"
- "code.google.com/p/weed-fs/go/glog"
"os"
)
diff --git a/go/storage/store.go b/go/storage/store.go
index 143d2b706..bea3bd34f 100644
--- a/go/storage/store.go
+++ b/go/storage/store.go
@@ -1,11 +1,11 @@
package storage
import (
+ "code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"encoding/json"
"fmt"
"io/ioutil"
- "code.google.com/p/weed-fs/go/glog"
"net/url"
"strconv"
"strings"
@@ -97,10 +97,10 @@ func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", replicationType =", replicationType)
if volume, err := NewVolume(location.directory, vid, replicationType); err == nil {
- location.volumes[vid] = volume
- return nil
+ location.volumes[vid] = volume
+ return nil
} else {
- return err
+ return err
}
}
return fmt.Errorf("No more free space left")
diff --git a/go/storage/volume.go b/go/storage/volume.go
index 6097aeec5..094134356 100644
--- a/go/storage/volume.go
+++ b/go/storage/volume.go
@@ -64,7 +64,7 @@ func (v *Volume) load(alsoLoadIndex bool) error {
v.dataFile, e = os.Open(fileName + ".dat")
v.readOnly = true
} else {
- return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
+ return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
}
if e != nil {
if !os.IsPermission(e) {
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 18:58:29 +0000