S3: pass HTTP 429 from volume servers to S3 clients (#7556)

With the recent changes (commit c1b8d4bf0) that made S3 directly access
volume servers instead of proxying through filer, we need to properly
handle HTTP 429 (Too Many Requests) errors from volume servers.

This change ensures that when volume servers rate limit requests with
HTTP 429, the S3 API properly translates this to an S3-compatible error
response (ErrRequestBytesExceed with HTTP 503) instead of returning a
generic InternalError.

Changes:
- Add ErrTooManyRequests sentinel error in weed/util/http
- Detect HTTP 429 in ReadUrlAsStream and wrap with ErrTooManyRequests
- Check for ErrTooManyRequests in GetObjectHandler and map to S3 error
- Return ErrRequestBytesExceed (HTTP 503) for rate limiting scenarios

This addresses the same issue as PR #7482 but for the new direct
volume server access path instead of the filer proxy path.

Fixes: Rate limiting errors from volume servers being masked as 500

2 files changed, 11 insertions, 1 deletions

diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go
index b1446c3e7..c403698e5 100644
--- a/weed/s3api/s3api_object_handlers.go
+++ b/weed/s3api/s3api_object_handlers.go
@@ -25,6 +25,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 )
@@ -740,7 +741,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 			return
 		}
 		// Response not yet written - safe to write S3 error response
-		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		// Check if error is due to volume server rate limiting (HTTP 429)
+		if errors.Is(err, util_http.ErrTooManyRequests) {
+			s3err.WriteErrorResponse(w, r, s3err.ErrRequestBytesExceed)
+		} else {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		}
 		return
 	}
 }
diff --git a/weed/util/http/http_global_client_util.go b/weed/util/http/http_global_client_util.go
index 38f129365..3a969fdc8 100644
--- a/weed/util/http/http_global_client_util.go
+++ b/weed/util/http/http_global_client_util.go
@@ -24,6 +24,7 @@ import (
 )
 
 var ErrNotFound = fmt.Errorf("not found")
+var ErrTooManyRequests = fmt.Errorf("too many requests")
 
 var (
 	jwtSigningReadKey        security.SigningKey
@@ -332,6 +333,9 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte,
 		if r.StatusCode == http.StatusNotFound {
 			return true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound)
 		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			return false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests)
+		}
 		retryable = r.StatusCode >= 499
 		return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
 	}