aboutsummaryrefslogtreecommitdiff
path: root/go/storage
diff options
context:
space:
mode:
Diffstat (limited to 'go/storage')
-rw-r--r--go/storage/cdb_map.go5
-rw-r--r--go/storage/cdb_map_test.go3
-rw-r--r--go/storage/compact_map.go2
-rw-r--r--go/storage/compact_map_perf_test.go5
-rw-r--r--go/storage/compress.go3
-rw-r--r--go/storage/crc.go3
-rw-r--r--go/storage/file_id.go5
-rw-r--r--go/storage/needle.go14
-rw-r--r--go/storage/needle_map.go5
-rw-r--r--go/storage/needle_read_write.go5
-rw-r--r--go/storage/store.go21
-rw-r--r--go/storage/store_vacuum.go3
-rw-r--r--go/storage/volume.go34
-rw-r--r--go/storage/volume_info.go5
-rw-r--r--go/storage/volume_super_block.go3
-rw-r--r--go/storage/volume_vacuum.go3
-rw-r--r--go/storage/volume_version.go2
17 files changed, 80 insertions, 41 deletions
diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go
index 1869a563e..fbb59e9c0 100644
--- a/go/storage/cdb_map.go
+++ b/go/storage/cdb_map.go
@@ -1,13 +1,14 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"fmt"
- "github.com/tgulacsi/go-cdb"
"os"
"path/filepath"
+
+ "github.com/chrislusf/weed-fs/go/util"
+ "github.com/tgulacsi/go-cdb"
)
// CDB-backed read-only needle map
diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go
index cff7dfa61..ed690f44f 100644
--- a/go/storage/cdb_map_test.go
+++ b/go/storage/cdb_map_test.go
@@ -1,11 +1,12 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
"math/rand"
"os"
"runtime"
"testing"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
var testIndexFilename string = "../../test/sample.idx"
diff --git a/go/storage/compact_map.go b/go/storage/compact_map.go
index 9cfc3e8f7..6ac18b012 100644
--- a/go/storage/compact_map.go
+++ b/go/storage/compact_map.go
@@ -1,7 +1,5 @@
package storage
-import ()
-
type NeedleValue struct {
Key Key
Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go
index ef43de25b..f74684225 100644
--- a/go/storage/compact_map_perf_test.go
+++ b/go/storage/compact_map_perf_test.go
@@ -1,11 +1,12 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/util"
"log"
"os"
"testing"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/util"
)
func TestMemoryUsage(t *testing.T) {
diff --git a/go/storage/compress.go b/go/storage/compress.go
index a353c9d3a..5efc9e1f1 100644
--- a/go/storage/compress.go
+++ b/go/storage/compress.go
@@ -2,11 +2,12 @@ package storage
import (
"bytes"
- "github.com/chrislusf/weed-fs/go/glog"
"compress/flate"
"compress/gzip"
"io/ioutil"
"strings"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
/*
diff --git a/go/storage/crc.go b/go/storage/crc.go
index 7aa400959..af25b9e53 100644
--- a/go/storage/crc.go
+++ b/go/storage/crc.go
@@ -1,9 +1,10 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/util"
"fmt"
"hash/crc32"
+
+ "github.com/chrislusf/weed-fs/go/util"
)
var table = crc32.MakeTable(crc32.Castagnoli)
diff --git a/go/storage/file_id.go b/go/storage/file_id.go
index ec566826c..f6e36a98c 100644
--- a/go/storage/file_id.go
+++ b/go/storage/file_id.go
@@ -1,11 +1,12 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/util"
"encoding/hex"
"errors"
"strings"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/util"
)
type FileId struct {
diff --git a/go/storage/needle.go b/go/storage/needle.go
index daede321b..11610dd80 100644
--- a/go/storage/needle.go
+++ b/go/storage/needle.go
@@ -1,11 +1,9 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/images"
- "github.com/chrislusf/weed-fs/go/util"
"encoding/hex"
"errors"
+ "fmt"
"io/ioutil"
"mime"
"net/http"
@@ -13,6 +11,10 @@ import (
"strconv"
"strings"
"time"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/images"
+ "github.com/chrislusf/weed-fs/go/util"
)
const (
@@ -23,6 +25,7 @@ const (
)
/*
+* A Needle means a uploaded and stored file.
* Needle file size is limited to 4GB for now.
*/
type Needle struct {
@@ -44,6 +47,11 @@ type Needle struct {
Padding []byte `comment:"Aligned to 8 bytes"`
}
+func (n *Needle) String() (str string) {
+ str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime)
+ return
+}
+
func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
form, fe := r.MultipartReader()
if fe != nil {
diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go
index dca2e6c5d..504ca1552 100644
--- a/go/storage/needle_map.go
+++ b/go/storage/needle_map.go
@@ -1,11 +1,12 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/util"
"fmt"
"io"
"os"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/util"
)
type NeedleMapper interface {
diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go
index bf452ba37..663b5abbd 100644
--- a/go/storage/needle_read_write.go
+++ b/go/storage/needle_read_write.go
@@ -1,12 +1,13 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/util"
"errors"
"fmt"
"io"
"os"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/util"
)
const (
diff --git a/go/storage/store.go b/go/storage/store.go
index e7a9dac94..65eed1d0e 100644
--- a/go/storage/store.go
+++ b/go/storage/store.go
@@ -1,10 +1,6 @@
package storage
import (
- proto "code.google.com/p/goprotobuf/proto"
- "github.com/chrislusf/weed-fs/go/glog"
- "github.com/chrislusf/weed-fs/go/operation"
- "github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"fmt"
@@ -12,6 +8,11 @@ import (
"math/rand"
"strconv"
"strings"
+
+ "github.com/chrislusf/weed-fs/go/glog"
+ "github.com/chrislusf/weed-fs/go/operation"
+ "github.com/chrislusf/weed-fs/go/util"
+ "github.com/golang/protobuf/proto"
)
const (
@@ -32,6 +33,10 @@ type MasterNodes struct {
lastNode int
}
+func (mn *MasterNodes) String() string {
+ return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode)
+}
+
func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) {
mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1}
return
@@ -64,6 +69,9 @@ func (mn *MasterNodes) findMaster() (string, error) {
return mn.nodes[mn.lastNode], nil
}
+/*
+ * A VolumeServer contains one Store
+ */
type Store struct {
Port int
Ip string
@@ -76,6 +84,11 @@ type Store struct {
masterNodes *MasterNodes
}
+func (s *Store) String() (str string) {
+ str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes)
+ return
+}
+
func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int) (s *Store) {
s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl}
s.Locations = make([]*DiskLocation, 0)
diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go
index 3527e4f59..209e3b4b3 100644
--- a/go/storage/store_vacuum.go
+++ b/go/storage/store_vacuum.go
@@ -1,9 +1,10 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
"fmt"
"strconv"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
diff --git a/go/storage/volume.go b/go/storage/volume.go
index de79e9107..a1eccd62c 100644
--- a/go/storage/volume.go
+++ b/go/storage/volume.go
@@ -2,7 +2,6 @@ package storage
import (
"bytes"
- "github.com/chrislusf/weed-fs/go/glog"
"errors"
"fmt"
"io"
@@ -10,6 +9,8 @@ import (
"path"
"sync"
"time"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
type Volume struct {
@@ -32,6 +33,10 @@ func NewVolume(dirname string, collection string, id VolumeId, replicaPlacement
e = v.load(true, true)
return
}
+func (v *Volume) String() string {
+ return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly)
+}
+
func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{}
@@ -72,7 +77,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error {
if e != nil {
if !os.IsPermission(e) {
- return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error())
+ return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e)
}
}
@@ -92,12 +97,12 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error {
if v.readOnly {
glog.V(1).Infoln("open to read file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil {
- return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error())
+ return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e)
}
} else {
glog.V(1).Infoln("open to write file", fileName+".idx")
if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil {
- return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error())
+ return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e)
}
}
glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly)
@@ -115,7 +120,7 @@ func (v *Volume) Size() int64 {
if e == nil {
return stat.Size()
}
- glog.V(0).Infof("Failed to read file size %s %s", v.dataFile.Name(), e.Error())
+ glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e)
return -1
}
func (v *Volume) Close() {
@@ -134,7 +139,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
oldNeedle := new(Needle)
oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) {
- n.Size = oldNeedle.Size
+ n.DataSize = oldNeedle.DataSize
return true
}
}
@@ -164,12 +169,13 @@ func (v *Volume) write(n *Needle) (size uint32, err error) {
v.accessLock.Lock()
defer v.accessLock.Unlock()
if v.isFileUnchanged(n) {
- size = n.Size
+ size = n.DataSize
glog.V(4).Infof("needle is unchanged!")
return
}
var offset int64
if offset, err = v.dataFile.Seek(0, 2); err != nil {
+ glog.V(0).Infof("faile to seek the end of file: %v", err)
return
}
@@ -177,21 +183,21 @@ func (v *Volume) write(n *Needle) (size uint32, err error) {
if offset%NeedlePaddingSize != 0 {
offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
if offset, err = v.dataFile.Seek(offset, 0); err != nil {
- glog.V(4).Infof("failed to align in datafile %s: %s", v.dataFile.Name(), err.Error())
+ glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
return
}
}
if size, err = n.Append(v.dataFile, v.Version()); err != nil {
if e := v.dataFile.Truncate(offset); e != nil {
- err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile.Name(), e.Error())
+ err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e)
}
return
}
nv, ok := v.nm.Get(n.Id)
if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
if _, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil {
- glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error())
+ glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
}
}
if v.lastModifiedTime < n.LastModified {
@@ -292,13 +298,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
offset := int64(SuperBlockSize)
n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
if e != nil {
- err = fmt.Errorf("cannot read needle header: %s", e)
+ err = fmt.Errorf("cannot read needle header: %v", e)
return
}
for n != nil {
if readNeedleBody {
if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil {
- err = fmt.Errorf("cannot read needle body: %s", err)
+ err = fmt.Errorf("cannot read needle body: %v", err)
return
}
}
@@ -310,7 +316,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
if err == io.EOF {
return nil
}
- return fmt.Errorf("cannot read needle header: %s", err)
+ return fmt.Errorf("cannot read needle header: %v", err)
}
}
@@ -360,7 +366,7 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) {
defer indexFile.Close()
glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName)
if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil {
- glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error())
+ glog.V(0).Infof("error converting %s.idx to %s.cdb: %v", fileName, fileName, e)
return false
}
return true
diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go
index 6410c1784..bc8049ea4 100644
--- a/go/storage/volume_info.go
+++ b/go/storage/volume_info.go
@@ -1,6 +1,7 @@
package storage
import (
+ "fmt"
"github.com/chrislusf/weed-fs/go/operation"
)
@@ -36,3 +37,7 @@ func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err er
vi.Ttl = LoadTTLFromUint32(*m.Ttl)
return vi, nil
}
+
+func (vi VolumeInfo) String() string {
+ return fmt.Sprintf("Id:%s, Size:%d, ReplicaPlacement:%s, Collection:%s, Version:%v, FileCount:%d, DeleteCount:%d, DeletedByteCount:%d, ReadOnly:%v", vi.Id, vi.Size, vi.ReplicaPlacement, vi.Collection, vi.Version, vi.FileCount, vi.DeleteCount, vi.DeletedByteCount, vi.ReadOnly)
+}
diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go
index a7e86b1c3..57e0deea9 100644
--- a/go/storage/volume_super_block.go
+++ b/go/storage/volume_super_block.go
@@ -1,9 +1,10 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
"fmt"
"os"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
const (
diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go
index b348434d2..7e026a61d 100644
--- a/go/storage/volume_vacuum.go
+++ b/go/storage/volume_vacuum.go
@@ -1,10 +1,11 @@
package storage
import (
- "github.com/chrislusf/weed-fs/go/glog"
"fmt"
"os"
"time"
+
+ "github.com/chrislusf/weed-fs/go/glog"
)
func (v *Volume) garbageLevel() float64 {
diff --git a/go/storage/volume_version.go b/go/storage/volume_version.go
index 9702ae904..2e9f58aa2 100644
--- a/go/storage/volume_version.go
+++ b/go/storage/volume_version.go
@@ -1,7 +1,5 @@
package storage
-import ()
-
type Version uint8
const (
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-23 14:19:37 +0000