aboutsummaryrefslogtreecommitdiff
path: root/weed/storage
diff options
context:
space:
mode:
Diffstat (limited to 'weed/storage')
-rw-r--r--weed/storage/disk_location.go2
-rw-r--r--weed/storage/needle_read_write.go9
-rw-r--r--weed/storage/store.go10
-rw-r--r--weed/storage/volume.go4
-rw-r--r--weed/storage/volume_create.go17
-rw-r--r--weed/storage/volume_create_linux.go19
-rw-r--r--weed/storage/volume_loading.go6
-rw-r--r--weed/storage/volume_vacuum.go8
8 files changed, 53 insertions, 22 deletions
diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go
index 039b4f3b9..496e0dd57 100644
--- a/weed/storage/disk_location.go
+++ b/weed/storage/disk_location.go
@@ -36,7 +36,7 @@ func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleM
_, found := l.volumes[vid]
mutex.RUnlock()
if !found {
- if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil); e == nil {
+ if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
mutex.Lock()
l.volumes[vid] = v
mutex.Unlock()
diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go
index ff43effb3..4f03ce396 100644
--- a/weed/storage/needle_read_write.go
+++ b/weed/storage/needle_read_write.go
@@ -146,18 +146,13 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
_, err = w.Write(header[0 : NeedleChecksumSize+padding])
- actualSize = NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize + int64(padding)
-
- return n.DataSize, actualSize, err
+ return n.DataSize, getActualSize(n.Size), err
}
return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
}
func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, block *Block, err error) {
- NeedleWithoutPaddingSize := NeedleHeaderSize + size + NeedleChecksumSize
- padding := NeedlePaddingSize - (NeedleWithoutPaddingSize % NeedlePaddingSize)
- readSize := NeedleWithoutPaddingSize + padding
- return getBytesForFileBlock(r, offset, int(readSize))
+ return getBytesForFileBlock(r, offset, int(getActualSize(size)))
}
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
diff --git a/weed/storage/store.go b/weed/storage/store.go
index 37a3904bd..be2044d64 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -95,7 +95,7 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
}
return
}
-func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string) error {
+func (s *Store) AddVolume(volumeListString string, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
rt, e := NewReplicaPlacementFromString(replicaPlacement)
if e != nil {
return e
@@ -111,7 +111,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, needleMapK
if err != nil {
return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
}
- e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl)
+ e = s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl, preallocate)
} else {
pair := strings.Split(range_string, "-")
start, start_err := strconv.ParseUint(pair[0], 10, 64)
@@ -123,7 +123,7 @@ func (s *Store) AddVolume(volumeListString string, collection string, needleMapK
return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
}
for id := start; id <= end; id++ {
- if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl); err != nil {
+ if err := s.addVolume(VolumeId(id), collection, needleMapKind, rt, ttl, preallocate); err != nil {
e = err
}
}
@@ -160,14 +160,14 @@ func (s *Store) findFreeLocation() (ret *DiskLocation) {
}
return ret
}
-func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) error {
+func (s *Store) addVolume(vid VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %d already exists!", vid)
}
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infof("In dir %s adds volume:%v collection:%s replicaPlacement:%v ttl:%v",
location.Directory, vid, collection, replicaPlacement, ttl)
- if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl); err == nil {
+ if volume, err := NewVolume(location.Directory, collection, vid, needleMapKind, replicaPlacement, ttl, preallocate); err == nil {
location.SetVolume(vid, volume)
return nil
} else {
diff --git a/weed/storage/volume.go b/weed/storage/volume.go
index 11ee600df..df9f0b7a7 100644
--- a/weed/storage/volume.go
+++ b/weed/storage/volume.go
@@ -29,11 +29,11 @@ type Volume struct {
lastCompactRevision uint16
}
-func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL) (v *Volume, e error) {
+func NewVolume(dirname string, collection string, id VolumeId, needleMapKind NeedleMapType, replicaPlacement *ReplicaPlacement, ttl *TTL, preallocate int64) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
v.needleMapKind = needleMapKind
- e = v.load(true, true, needleMapKind)
+ e = v.load(true, true, needleMapKind, preallocate)
return
}
func (v *Volume) String() string {
diff --git a/weed/storage/volume_create.go b/weed/storage/volume_create.go
new file mode 100644
index 000000000..6b3a17439
--- /dev/null
+++ b/weed/storage/volume_create.go
@@ -0,0 +1,17 @@
+// +build !linux
+
+package storage
+
+import (
+ "os"
+
+ "github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) {
+ file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
+ if preallocate > 0 {
+ glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
+ }
+ return file, e
+}
diff --git a/weed/storage/volume_create_linux.go b/weed/storage/volume_create_linux.go
new file mode 100644
index 000000000..8f6bab2fe
--- /dev/null
+++ b/weed/storage/volume_create_linux.go
@@ -0,0 +1,19 @@
+// +build linux
+
+package storage
+
+import (
+ "os"
+ "syscall"
+
+ "github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func createVolumeFile(fileName string, preallocate int64) (file *os.File, e error) {
+ file, e = os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
+ if preallocate != 0 {
+ syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)
+ glog.V(0).Infof("Preallocated %d bytes disk space for %s", preallocate, fileName)
+ }
+ return file, e
+}
diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go
index 7bc65a4a3..c4f1aae9b 100644
--- a/weed/storage/volume_loading.go
+++ b/weed/storage/volume_loading.go
@@ -12,11 +12,11 @@ func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId, need
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{}
v.needleMapKind = needleMapKind
- e = v.load(false, false, needleMapKind)
+ e = v.load(false, false, needleMapKind, 0)
return
}
-func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
+func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType, preallocate int64) error {
var e error
fileName := v.FileName()
@@ -34,7 +34,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
}
} else {
if createDatIfMissing {
- v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
+ v.dataFile, e = createVolumeFile(fileName+".dat", preallocate)
} else {
return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index f3ded5ff2..13072d1fb 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -24,7 +24,7 @@ func (v *Volume) Compact() error {
v.lastCompactIndexOffset = v.nm.IndexFileSize()
v.lastCompactRevision = v.SuperBlock.CompactRevision
glog.V(3).Infof("creating copies for volume %d ,last offset %d...", v.Id, v.lastCompactIndexOffset)
- return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
+ return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx", v.dataFileSize)
}
func (v *Volume) Compact2() error {
@@ -66,7 +66,7 @@ func (v *Volume) commitCompact() error {
//glog.V(3).Infof("Pretending to be vacuuming...")
//time.Sleep(20 * time.Second)
glog.V(3).Infof("Loading Commit file...")
- if e = v.load(true, false, v.needleMapKind); e != nil {
+ if e = v.load(true, false, v.needleMapKind, 0); e != nil {
return e
}
return nil
@@ -207,11 +207,11 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
return nil
}
-func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
+func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64) (err error) {
var (
dst, idx *os.File
)
- if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil {
+ if dst, err = createVolumeFile(dstName, preallocate); err != nil {
return
}
defer dst.Close()
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 11:52:40 +0000