diff --git a/storage/invoke.go b/storage/invoke.go index 6fde482404e..d8f5a6d4a6f 100644 --- a/storage/invoke.go +++ b/storage/invoke.go @@ -136,12 +136,11 @@ func ShouldRetry(err error) bool { return true } } - // HTTP 429, 502, 503, and 504 all map to gRPC UNAVAILABLE per - // https://grpc.github.io/grpc/core/md_doc_http-grpc-status-mapping.html. - // - // This is only necessary for the experimental gRPC-based media operations. - if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable { - return true + // UNAVAILABLE, RESOURCE_EXHAUSTED, and INTERNAL codes are all retryable for gRPC. + if st, ok := status.FromError(err); ok { + if code := st.Code(); code == codes.Unavailable || code == codes.ResourceExhausted || code == codes.Internal { + return true + } } // Unwrap is only supported in go1.13.x+ if e, ok := err.(interface{ Unwrap() error }); ok { diff --git a/storage/invoke_test.go b/storage/invoke_test.go index 48f19bc0f33..58f608cb7cd 100644 --- a/storage/invoke_test.go +++ b/storage/invoke_test.go @@ -72,6 +72,14 @@ func TestInvoke(t *testing.T) { isIdempotentValue: true, expectFinalErr: true, }, + { + desc: "retryable gRPC error is retried", + count: 1, + initialErr: status.Error(codes.ResourceExhausted, "rate limit"), + finalErr: nil, + isIdempotentValue: true, + expectFinalErr: true, + }, { desc: "returns non-retryable error after retryable error", count: 1,