fix: handle HTTP 404/410 from remote CAR gateways

wrap HTTP error responses from remote CAR fetcher with ErrorStatusCode
to ensure proper error classification in gateway handlers

- 404 responses are now recognized by isErrNotFound()
- 410 responses are now recognized by isErrContentBlocked()
- limit error message reading to 512 bytes to prevent memory exhaustion
- properly close/drain response body to avoid connection leaks
- fixes directory listing failures with remote-car-backend

Author: lidel

gateway/backend_car_fetcher.go

@@ -66,15 +66,33 @@ func (ps *remoteCarFetcher) Fetch(ctx context.Context, path path.ImmutablePath,
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		errData, err := io.ReadAll(resp.Body)
+		defer resp.Body.Close() // Ensure body is closed and drained
+
+		// Limit error message reading to prevent memory exhaustion
+		// 512 bytes is enough for one sentence of error description
+		const maxErrorSize = 512
+		limitedReader := io.LimitReader(resp.Body, maxErrorSize)
+		errData, err := io.ReadAll(limitedReader)
+		var errMsg string
 		if err != nil {
-			err = fmt.Errorf("could not read error message: %w", err)
+			errMsg = fmt.Sprintf("could not read error message: %v", err)
 		} else {
-			err = fmt.Errorf("%q", string(errData))
+			errMsg = string(errData)
+			// Add ellipsis if we hit the limit
+			if len(errData) == maxErrorSize {
+				errMsg = errMsg + "..."
+			}
 		}
-		return fmt.Errorf("http error from car gateway: %s: %w", resp.Status, err)
+
+		// Wrap with appropriate status code for proper handling downstream
+		return NewErrorStatusCode(
+			fmt.Errorf("car gateway responded with %s: %q", resp.Status, errMsg),
+			resp.StatusCode,
+		)
 	}
 
+	// For successful responses, callback is responsible for reading
+	// and closing the body
 	err = cb(path, resp.Body)
 	if err != nil {
 		resp.Body.Close()

gateway/errors.go

@@ -253,6 +253,12 @@ func webError(w http.ResponseWriter, r *http.Request, c *Config, err error, defa
 // isErrNotFound returns true for IPLD errors that should return 4xx errors (e.g. the path doesn't exist, the data is
 // the wrong type, etc.), rather than issues with just finding and retrieving the data.
 func isErrNotFound(err error) bool {
+	// Check for ErrorStatusCode with 404
+	var statusErr *ErrorStatusCode
+	if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusNotFound {
+		return true
+	}
+
 	if errors.Is(err, &resolver.ErrNoLink{}) || errors.Is(err, schema.ErrNoSuchField{}) {
 		return true
 	}
@@ -295,6 +301,12 @@ func isErrNotFound(err error) bool {
 // TODO: When nopfs becomes a direct dependency, replace this string matching with proper
 // type assertion or errors.Is() for more robust error detection.
 func isErrContentBlocked(err error) bool {
+	// Check for ErrorStatusCode with 410
+	var statusErr *ErrorStatusCode
+	if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusGone {
+		return true
+	}
+
 	// The nopfs StatusError.Error() returns messages in the format:
 	// - "{cid} is blocked and cannot be provided" for blocked CIDs
 	// - "{path} is blocked and cannot be provided" for blocked paths