diff options
| author | Eric Yang <lanqingy@usc.edu> | 2022-09-29 12:48:36 -0700 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-09-29 12:48:36 -0700 |
| commit | 56c94cc08ec8d5dfbc9aace0c1931334ef79f4b4 (patch) | |
| tree | 29de0b3aa4f0c1caa0dd03dbd381aa364d441690 | |
| parent | 3de1e1978091e9675c9d54655dcde0c7dea9d222 (diff) | |
| download | seaweedfs-56c94cc08ec8d5dfbc9aace0c1931334ef79f4b4.tar.xz seaweedfs-56c94cc08ec8d5dfbc9aace0c1931334ef79f4b4.zip | |
ADHOC: filter deleted files from idx file binary search (#3763)
* ADHOC: filter deleted files from idx file binary search
* remove unwanted check
Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>
| -rw-r--r-- | weed/shell/command_volume_fsck.go | 18 | ||||
| -rw-r--r-- | weed/storage/volume_read.go | 4 |
2 files changed, 15 insertions, 7 deletions
diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 04d8bc2c4..1ee5e351c 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -393,8 +393,9 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI } buf.Write(resp.FileContent) } + fileredBuf := filterDeletedNeedleFromIdx(buf.Bytes()) if vinfo.isReadOnly == false { - index, err := idx.FirstInvalidIndex(buf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { + index, err := idx.FirstInvalidIndex(fileredBuf.Bytes(), func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ VolumeId: volumeId, NeedleId: uint64(key), @@ -409,11 +410,11 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(tempFolder string, dataNodeI if err != nil { fmt.Fprintf(writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err) } else { - buf.Truncate(index * types.NeedleMapEntrySize) + fileredBuf.Truncate(index * types.NeedleMapEntrySize) } } idxFilename := getVolumeFileIdFile(tempFolder, dataNodeId, volumeId) - err = writeToFile(buf.Bytes(), idxFilename) + err = writeToFile(fileredBuf.Bytes(), idxFilename) if err != nil { return fmt.Errorf("failed to copy %d%s from %s: %v", volumeId, ext, vinfo.server, err) } @@ -719,3 +720,14 @@ func writeToFile(bytes []byte, fileName string) error { dst.Write(bytes) return nil } + +func filterDeletedNeedleFromIdx(arr []byte) bytes.Buffer { + var filteredBuf bytes.Buffer + for i := 0; i < len(arr); i += types.NeedleMapEntrySize { + size := types.BytesToSize(arr[i+types.NeedleIdSize+types.OffsetSize : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize]) + if size > 0 { + filteredBuf.Write(arr[i : i+types.NeedleIdSize+types.OffsetSize+types.SizeSize]) + } + } + return filteredBuf +} diff --git a/weed/storage/volume_read.go b/weed/storage/volume_read.go index 8b30bd5d4..b0e542284 100644 --- a/weed/storage/volume_read.go +++ b/weed/storage/volume_read.go @@ -84,10 +84,6 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) { v.dataFileAccessLock.RLock() defer v.dataFileAccessLock.RUnlock() - // read deleted meta data - if size < 0 { - size = -size - } err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version()) if err == needle.ErrorSizeMismatch && OffsetSize == 4 { err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version()) |