aboutsummaryrefslogtreecommitdiff
path: root/weed/filesys/dirty_pages_temp_file.go
diff options
context:
space:
mode:
Diffstat (limited to 'weed/filesys/dirty_pages_temp_file.go')
-rw-r--r--weed/filesys/dirty_pages_temp_file.go27
1 files changed, 19 insertions, 8 deletions
diff --git a/weed/filesys/dirty_pages_temp_file.go b/weed/filesys/dirty_pages_temp_file.go
index a04efb6aa..58c150acf 100644
--- a/weed/filesys/dirty_pages_temp_file.go
+++ b/weed/filesys/dirty_pages_temp_file.go
@@ -56,18 +56,23 @@ func (pages *TempFileDirtyPages) AddPage(offset int64, data []byte) {
}
pages.tf = tf
pages.writtenIntervals.tempFile = tf
+ pages.writtenIntervals.lastOffset = 0
}
- writtenOffset := pages.writtenIntervals.TotalSize()
+ writtenOffset := pages.writtenIntervals.lastOffset
+ dataSize := int64(len(data))
- glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+int64(len(data)))
+ // glog.V(4).Infof("%s AddPage %v at %d [%d,%d)", pages.f.fullpath(), pages.tf.Name(), writtenOffset, offset, offset+dataSize)
if _, err := pages.tf.WriteAt(data, writtenOffset); err != nil {
pages.lastErr = err
} else {
pages.writtenIntervals.AddInterval(writtenOffset, len(data), offset)
+ pages.writtenIntervals.lastOffset += dataSize
}
+ // pages.writtenIntervals.debug()
+
return
}
@@ -81,6 +86,11 @@ func (pages *TempFileDirtyPages) FlushData() error {
pages.pageAddLock.Lock()
defer pages.pageAddLock.Unlock()
if pages.tf != nil {
+
+ pages.writtenIntervals.tempFile = nil
+ pages.writtenIntervals.lists = nil
+
+ pages.tf.Close()
os.Remove(pages.tf.Name())
pages.tf = nil
}
@@ -91,15 +101,16 @@ func (pages *TempFileDirtyPages) saveExistingPagesToStorage() {
pageSize := pages.f.wfs.option.ChunkSizeLimit
- uploadedSize := int64(0)
+ // glog.V(4).Infof("%v saveExistingPagesToStorage %d lists", pages.f.Name, len(pages.writtenIntervals.lists))
+
for _, list := range pages.writtenIntervals.lists {
- for {
- start, stop := max(list.Offset(), uploadedSize), min(list.Offset()+list.Size(), uploadedSize+pageSize)
+ listStopOffset := list.Offset() + list.Size()
+ for uploadedOffset:=int64(0); uploadedOffset < listStopOffset; uploadedOffset += pageSize {
+ start, stop := max(list.Offset(), uploadedOffset), min(listStopOffset, uploadedOffset+pageSize)
if start >= stop {
- break
+ continue
}
- uploadedSize = stop
- glog.V(4).Infof("uploading %v [%d,%d)", pages.f.Name, start, stop)
+ // glog.V(4).Infof("uploading %v [%d,%d) %d/%d", pages.f.Name, start, stop, i, len(pages.writtenIntervals.lists))
pages.saveToStorage(list.ToReader(start, stop), start, stop-start)
}
}