diff options
| author | Chris Lu <chris.lu@gmail.com> | 2020-12-12 03:38:34 -0800 |
|---|---|---|
| committer | Chris Lu <chris.lu@gmail.com> | 2020-12-12 03:38:34 -0800 |
| commit | 03637d6f57e34be281e778c5e9fa3e3096331c2d (patch) | |
| tree | b0886d138559f24f182240001925d4205e240289 /weed/s3api | |
| parent | 37075a414dcff0769f06ee0f3268d7f338ffba63 (diff) | |
| download | seaweedfs-03637d6f57e34be281e778c5e9fa3e3096331c2d.tar.xz seaweedfs-03637d6f57e34be281e778c5e9fa3e3096331c2d.zip | |
s3: move "delete-directory-if-empty" to read time
move "delete-directory-if-empty" to read time instead of entry deletion time
the listing speed for a s3 bucket folder will slow down if it has many sub folders
related to https://github.com/chrislusf/seaweedfs/commit/0d345ac97d941bef0991c0cb08632ccf08c0ee83
fix https://github.com/chrislusf/seaweedfs/issues/1647
fix https://github.com/chrislusf/seaweedfs/issues/1670
Diffstat (limited to 'weed/s3api')
| -rw-r--r-- | weed/s3api/s3api_objects_list_handlers.go | 27 |
1 files changed, 25 insertions, 2 deletions
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 40583f478..5804f0d44 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -246,8 +246,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d if entry.IsDirectory { // println("ListEntries", dir, "dir:", entry.Name) if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys - eachEntryFn(dir, entry) if delimiter != "/" { + eachEntryFn(dir, entry) // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn) if subErr != nil { @@ -262,7 +262,18 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } } else { - counter++ + var isEmpty bool + if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir+"/"+entry.Name); err != nil { + return + } + if isEmpty { + if err = doDeleteEntry(client, dir, entry.Name, true, true); err != nil { + return + } + } else { + eachEntryFn(dir, entry) + counter++ + } } } } else { @@ -299,3 +310,15 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, } return } + +func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, dir string) (isEmpty bool, err error) { + // println("+ isDirectoryAllEmpty", dir) + subCounter, _, _, subErr := s3a.doListFilerEntries(filerClient, dir, "", 32, "", "/", func(dir string, entry *filer_pb.Entry) { + }) + if subErr != nil { + err = fmt.Errorf("isDirectoryAllEmpty: %v", subErr) + } + isEmpty = subCounter <= 0 + // println("- isDirectoryAllEmpty", dir, isEmpty) + return +} |