diff options
| -rw-r--r-- | .github/workflows/binaries_dev.yml | 3 | ||||
| -rw-r--r-- | go.mod | 2 | ||||
| -rw-r--r-- | go.sum | 4 | ||||
| -rw-r--r-- | weed/server/volume_grpc_copy.go | 1 | ||||
| -rw-r--r-- | weed/shell/command_volume_fsck.go | 8 | ||||
| -rw-r--r-- | weed/storage/volume.go | 15 |
6 files changed, 26 insertions, 7 deletions
diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml index 29b1cf6ab..b897732f6 100644 --- a/.github/workflows/binaries_dev.yml +++ b/.github/workflows/binaries_dev.yml @@ -82,7 +82,8 @@ jobs: strategy: matrix: goos: [darwin] - goarch: [amd64, arm64] + # goarch: [amd64, arm64] + goarch: [amd64] steps: @@ -153,7 +153,7 @@ require ( github.com/arangodb/go-driver v1.3.1 github.com/fluent/fluent-logger-golang v1.9.0 github.com/hanwen/go-fuse/v2 v2.1.0 - github.com/hashicorp/raft v1.3.7 + github.com/hashicorp/raft v1.3.8 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 ) @@ -550,8 +550,8 @@ github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.3.7 h1:mnuQAuSMNRjAvFc9IYxnGxZJ9LBBZyheMR8Kv8Ve71M= -github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft v1.3.8 h1:lrhx4wesQLOSv3ERX/pK4cwfzQ0J2RgzsvAkBxHe1bA= +github.com/hashicorp/raft v1.3.8/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc= github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go index 6ed2724f1..e3ec5b724 100644 --- a/weed/server/volume_grpc_copy.go +++ b/weed/server/volume_grpc_copy.go @@ -279,6 +279,7 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v if uint32(v.CompactionRevision) != req.CompactionRevision && req.CompactionRevision != math.MaxUint32 { return fmt.Errorf("volume %d is compacted", req.VolumeId) } + v.SyncToDisk() fileName = v.FileName(req.Ext) } else { baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) + req.Ext diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 1aa33e054..584ce722b 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -33,7 +33,8 @@ func init() { } type commandVolumeFsck struct { - env *CommandEnv + env *CommandEnv + forcePurging *bool } func (c *commandVolumeFsck) Name() string { @@ -68,6 +69,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. findMissingChunksInFilerPath := fsckCommand.String("findMissingChunksInFilerPath", "/", "used together with findMissingChunksInFiler") findMissingChunksInVolumeId := fsckCommand.Int("findMissingChunksInVolumeId", 0, "used together with findMissingChunksInFiler") applyPurging := fsckCommand.Bool("reallyDeleteFromVolume", false, "<expert only!> after detection, delete missing data from volumes / delete missing file entries from filer") + c.forcePurging = fsckCommand.Bool("forcePurging", false, "delete missing data from volumes in one replica used together with applyPurging") purgeAbsent := fsckCommand.Bool("reallyDeleteFilerEntries", false, "<expert only!> delete missing file entries from filer if the corresponding volume is missing for any reason, please ensure all still existing/expected volumes are connected! used together with findMissingChunksInFiler") tempPath := fsckCommand.String("tempPath", path.Join(os.TempDir()), "path for temporary idx files") @@ -293,7 +295,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn } orphanFileIds := []string{} for fid, foundInAllReplicas := range orphanReplicaFileIds { - if !isSeveralReplicas[volumeId] || (isSeveralReplicas[volumeId] && foundInAllReplicas) { + if !isSeveralReplicas[volumeId] || *c.forcePurging || (isSeveralReplicas[volumeId] && foundInAllReplicas) { orphanFileIds = append(orphanFileIds, fid) } } @@ -301,7 +303,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn continue } if verbose { - fmt.Fprintf(writer, "purging process for volume %d", volumeId) + fmt.Fprintf(writer, "purging process for volume %d.\n", volumeId) } if isEcVolumeReplicas[volumeId] { diff --git a/weed/storage/volume.go b/weed/storage/volume.go index 6d48fbc83..f388c7d99 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -190,6 +190,21 @@ func (v *Volume) SetStopping() { } } +func (v *Volume) SyncToDisk() { + v.dataFileAccessLock.Lock() + defer v.dataFileAccessLock.Unlock() + if v.nm != nil { + if err := v.nm.Sync(); err != nil { + glog.Warningf("Volume Close fail to sync volume idx %d", v.Id) + } + } + if v.DataBackend != nil { + if err := v.DataBackend.Sync(); err != nil { + glog.Warningf("Volume Close fail to sync volume %d", v.Id) + } + } +} + // Close cleanly shuts down this volume func (v *Volume) Close() { v.dataFileAccessLock.Lock() |