diff options
| -rw-r--r-- | weed/storage/memory_map/memory_map.go | 12 | ||||
| -rw-r--r-- | weed/storage/memory_map/memory_map_windows.go | 22 | ||||
| -rw-r--r-- | weed/storage/needle/needle_read_write.go | 11 | ||||
| -rw-r--r-- | weed/storage/volume_super_block.go | 6 |
4 files changed, 24 insertions, 27 deletions
diff --git a/weed/storage/memory_map/memory_map.go b/weed/storage/memory_map/memory_map.go index 4baf6fc1f..5f0327ea7 100644 --- a/weed/storage/memory_map/memory_map.go +++ b/weed/storage/memory_map/memory_map.go @@ -25,16 +25,16 @@ type MemoryMap struct { var FileMemoryMap = make(map[string]*MemoryMap) -func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { - +func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) { } -func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) { - return MemoryBuffer{}, fmt.Errorf("Memory Map not implemented for this platform") -} +func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { -func (mBuffer *MemoryBuffer) ReleaseMemory() { +} +func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) ([]byte, error) { + dataSlice := []byte{} + return dataSlice, fmt.Errorf("Memory Map not implemented for this platform") } func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() { diff --git a/weed/storage/memory_map/memory_map_windows.go b/weed/storage/memory_map/memory_map_windows.go index a06b7947c..ac4493188 100644 --- a/weed/storage/memory_map/memory_map_windows.go +++ b/weed/storage/memory_map/memory_map_windows.go @@ -48,7 +48,7 @@ var currentMaxWorkingSet uint64 = 0 var _ = getProcessWorkingSetSize(uintptr(currentProcess), ¤tMinWorkingSet, ¤tMaxWorkingSet) var systemInfo, _ = getSystemInfo() -var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256 +var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 128 var memoryStatusEx, _ = globalMemoryStatusEx() var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8) @@ -82,7 +82,7 @@ func (mMap *MemoryMap) DeleteFileAndMemoryMap() { windows.CloseHandle(windows.Handle(mMap.File.Fd())) for _, view := range mMap.write_map_views { - view.ReleaseMemory() + view.releaseMemory() } mMap.write_map_views = nil @@ -130,17 +130,21 @@ func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) { } } -func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) { - return allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false) +func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (dataSlice []byte, err error) { + dataSlice = make([]byte, length) + mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false) + copy(dataSlice, mBuffer.Buffer) + mBuffer.releaseMemory() + return dataSlice, err } -func (mBuffer *MemoryBuffer) ReleaseMemory() { +func (mBuffer *MemoryBuffer) releaseMemory() { windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length)) windows.UnmapViewOfFile(mBuffer.aligned_ptr) - currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length - currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length + currentMinWorkingSet -= mBuffer.aligned_length + currentMaxWorkingSet -= mBuffer.aligned_length if currentMinWorkingSet < maxMemoryLimitBytes { var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet) @@ -182,8 +186,8 @@ func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) access = windows.FILE_MAP_WRITE } - currentMinWorkingSet = currentMinWorkingSet + aligned_length - currentMaxWorkingSet = currentMaxWorkingSet + aligned_length + currentMinWorkingSet += aligned_length + currentMaxWorkingSet += aligned_length if currentMinWorkingSet < maxMemoryLimitBytes { // increase the process working set size to hint to windows memory manager to diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go index 3fe6a246c..5f8bfee8b 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read_write.go @@ -164,13 +164,11 @@ func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32 func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dataSlice []byte, err error) { dataSize := GetActualSize(size, version) - dataSlice = make([]byte, dataSize) + dataSlice = make([]byte, int(dataSize)) mMap, exists := memory_map.FileMemoryMap[r.Name()] if exists { - mBuffer, err := mMap.ReadMemory(uint64(offset), uint64(dataSize)) - copy(dataSlice, mBuffer.Buffer) - mBuffer.ReleaseMemory() + dataSlice, err := mMap.ReadMemory(uint64(offset), uint64(dataSize)) return dataSlice, err } else { _, err = r.ReadAt(dataSlice, offset) @@ -291,10 +289,7 @@ func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, byt mMap, exists := memory_map.FileMemoryMap[r.Name()] if exists { - mBuffer, err := mMap.ReadMemory(uint64(offset), NeedleHeaderSize) - copy(bytes, mBuffer.Buffer) - mBuffer.ReleaseMemory() - + bytes, err = mMap.ReadMemory(uint64(offset), NeedleHeaderSize) if err != nil { return nil, bytes, 0, err } diff --git a/weed/storage/volume_super_block.go b/weed/storage/volume_super_block.go index 7df733b57..414d55f68 100644 --- a/weed/storage/volume_super_block.go +++ b/weed/storage/volume_super_block.go @@ -113,13 +113,11 @@ func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) { header := make([]byte, _SuperBlockSize) mMap, exists := memory_map.FileMemoryMap[dataFile.Name()] if exists { - mBuffer, e := mMap.ReadMemory(0, _SuperBlockSize) + header, err = mMap.ReadMemory(0, _SuperBlockSize) if err != nil { - err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e) + err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), err) return } - copy(header, mBuffer.Buffer) - mBuffer.ReleaseMemory() } else { if _, err = dataFile.Seek(0, 0); err != nil { err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err) |