S3: add object versioning (#6945)

* add object versioning

* add missing file

* Update weed/s3api/s3api_object_versioning.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_versioning.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_versioning.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* ListObjectVersionsResult is better to show multiple version entries

* fix test

* Update weed/s3api/s3api_object_handlers_put.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update weed/s3api/s3api_object_versioning.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* multiple improvements

* move PutBucketVersioningHandler into weed/s3api/s3api_bucket_handlers.go file
* duplicated code for reading bucket config, versioningEnabled, etc. try to use functions
* opportunity to cache bucket config

* error handling if bucket is not found

* in case bucket is not found

* fix build

* add object versioning tests

* remove non-existent tests

* add tests

* add versioning tests

* skip a new test

* ensure .versions directory exists before saving info into it

* fix creating version entry

* logging on creating version directory

* Update s3api_object_versioning_test.go

* retry and wait for directory creation

* revert add more logging

* Update s3api_object_versioning.go

* more debug messages

* clean up logs, and touch directory correctly

* log the .versions creation and then parent directory listing

* use mkFile instead of touch

touch is for update

* clean up data

* add versioning test in go

* change location

* if modified, latest version is moved to .versions directory, and create a new latest version

 Core versioning functionality: WORKING
TestVersioningBasicWorkflow - PASS
TestVersioningDeleteMarkers - PASS
TestVersioningMultipleVersionsSameObject - PASS
TestVersioningDeleteAndRecreate - PASS
TestVersioningListWithPagination - PASS
❌ Some advanced features still failing:
ETag calculation issues (using mtime instead of proper MD5)
Specific version retrieval (EOF error)
Version deletion (internal errors)
Concurrent operations (race conditions)

* calculate multi chunk md5

Test Results - All Passing:
✅ TestBucketListReturnDataVersioning - PASS
✅ TestVersioningCreateObjectsInOrder - PASS
✅ TestVersioningBasicWorkflow - PASS
✅ TestVersioningMultipleVersionsSameObject - PASS
✅ TestVersioningDeleteMarkers - PASS

* dedupe

* fix TestVersioningErrorCases

* fix eof error of reading old versions

* get specific version also check current version

* enable integration tests for versioning

* trigger action to work for now

* Fix GitHub Actions S3 versioning tests workflow

- Fix syntax error (incorrect indentation)
- Update directory paths from weed/s3api/versioning_tests/ to test/s3/versioning/
- Add push trigger for add-object-versioning branch to enable CI during development
- Update artifact paths to match correct directory structure

* Improve CI robustness for S3 versioning tests

Makefile improvements:
- Increase server startup timeout from 30s to 90s for CI environments
- Add progressive timeout reporting (logs at 30s, full logs at 90s)
- Better error handling with server logs on failure
- Add server PID tracking for debugging
- Improved test failure reporting

GitHub Actions workflow improvements:
- Increase job timeouts to account for CI environment delays
- Add system information logging (memory, disk space)
- Add detailed failure reporting with server logs
- Add process and network diagnostics on failure
- Better error messaging and log collection

These changes should resolve the 'Server failed to start within 30 seconds' issue
that was causing the CI tests to fail.

* adjust testing volume size

* Update Makefile

* Update Makefile

* Update Makefile

* Update Makefile

* Update s3-versioning-tests.yml

* Update s3api_object_versioning.go

* Update Makefile

* do not clean up

* log received version id

* more logs

* printout response

* print out list version response

* use tmp files when put versioned object

* change to versions folder layout

* Delete weed-test.log

* test with mixed versioned and unversioned objects

* remove versionDirCache

* remove unused functions

* remove unused function

* remove fallback checking

* minor

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

1 files changed, 67 insertions, 24 deletions

diff --git a/weed/s3api/s3api_object_handlers_delete.go b/weed/s3api/s3api_object_handlers_delete.go
index 802e82b5f..d7457fabe 100644
--- a/weed/s3api/s3api_object_handlers_delete.go
+++ b/weed/s3api/s3api_object_handlers_delete.go
@@ -29,44 +29,87 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 	bucket, object := s3_constants.GetBucketAndObject(r)
 	glog.V(3).Infof("DeleteObjectHandler %s %s", bucket, object)
 
-	target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
-	dir, name := target.DirAndName()
+	// Check for specific version ID in query parameters
+	versionId := r.URL.Query().Get("versionId")
 
-	var auditLog *s3err.AccessLog
+	// Check if versioning is enabled for the bucket
+	versioningEnabled, err := s3a.isVersioningEnabled(bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
 
+	var auditLog *s3err.AccessLog
 	if s3err.Logger != nil {
 		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
 	}
 
-	err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+	if versioningEnabled {
+		// Handle versioned delete
+		if versionId != "" {
+			// Delete specific version
+			err := s3a.deleteSpecificObjectVersion(bucket, object, versionId)
+			if err != nil {
+				glog.Errorf("Failed to delete specific version %s: %v", versionId, err)
+				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+				return
+			}
 
-		if err := doDeleteEntry(client, dir, name, true, false); err != nil {
-			return err
-		}
+			// Set version ID in response header
+			w.Header().Set("x-amz-version-id", versionId)
+		} else {
+			// Create delete marker (logical delete)
+			deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
+			if err != nil {
+				glog.Errorf("Failed to create delete marker: %v", err)
+				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+				return
+			}
 
-		if auditLog != nil {
-			auditLog.Key = name
-			s3err.PostAccessLog(*auditLog)
+			// Set delete marker version ID in response header
+			w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
+			w.Header().Set("x-amz-delete-marker", "true")
 		}
+	} else {
+		// Handle regular delete (non-versioned)
+		target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object))
+		dir, name := target.DirAndName()
 
-		if s3a.option.AllowEmptyFolder {
-			return nil
-		}
+		err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+			if err := doDeleteEntry(client, dir, name, true, false); err != nil {
+				return err
+			}
 
-		directoriesWithDeletion := make(map[string]int)
-		if strings.LastIndex(object, "/") > 0 {
-			directoriesWithDeletion[dir]++
-			// purge empty folders, only checking folders with deletions
-			for len(directoriesWithDeletion) > 0 {
-				directoriesWithDeletion = s3a.doDeleteEmptyDirectories(client, directoriesWithDeletion)
+			if s3a.option.AllowEmptyFolder {
+				return nil
 			}
+
+			directoriesWithDeletion := make(map[string]int)
+			if strings.LastIndex(object, "/") > 0 {
+				directoriesWithDeletion[dir]++
+				// purge empty folders, only checking folders with deletions
+				for len(directoriesWithDeletion) > 0 {
+					directoriesWithDeletion = s3a.doDeleteEmptyDirectories(client, directoriesWithDeletion)
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return
 		}
+	}
 
-		return nil
-	})
-	if err != nil {
-		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
-		return
+	if auditLog != nil {
+		auditLog.Key = strings.TrimPrefix(object, "/")
+		s3err.PostAccessLog(*auditLog)
 	}
 
 	stats_collect.RecordBucketActiveTime(bucket)