[receiver/otlp] Switch from InvalidArgument to Internal (#9415)

The otlp receiver was recently updated via
#8080 to
properly propagate consumer errors back to clients as either permanent
or retriable. The code we're using to indicate a non-retriable error is
`codes.InvalidArgument`, which is the equivalent of `400` in HTTP.

While 100% correct according to the [OTLP
specification](https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specification.md#failures)
to indicate a non-retriable error, I think `codes.Internal` (which is
equivalent to HTTP `500`), better conveys the actual state of the
collector in these situations.

Related to
#9357 (comment)


---------

Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com>

.chloggen/update-GetStatusFromError-code.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: receiver/otlp
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Update gRPC code from `codes.InvalidArgument` to `codes.Internal` when a permanent error doesn't contain a gRPC status
+
+# One or more tracking issues or pull requests related to the change
+issues: [9415]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/otlpreceiver/internal/errors/errors.go

@@ -17,7 +17,8 @@ func GetStatusFromError(err error) error {
 		// https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specification.md#failures
 		code := codes.Unavailable
 		if consumererror.IsPermanent(err) {
-			code = codes.InvalidArgument
+			// If an error is permanent but doesn't have an attached gRPC status, assume it is server-side.
+			code = codes.Internal
 		}
 		s = status.New(code, err.Error())
 	}

receiver/otlpreceiver/internal/errors/errors_test.go

@@ -28,7 +28,7 @@ func Test_GetStatusFromError(t *testing.T) {
 		{
 			name:     "Permanent Error",
 			input:    consumererror.NewPermanent(fmt.Errorf("test")),
-			expected: status.New(codes.InvalidArgument, "Permanent error: test"),
+			expected: status.New(codes.Internal, "Permanent error: test"),
 		},
 		{
 			name:     "Non-Permanent Error",

receiver/otlpreceiver/internal/logs/otlp_test.go

@@ -67,7 +67,7 @@ func TestExport_PermanentErrorConsumer(t *testing.T) {
 
 	logClient := makeLogsServiceClient(t, consumertest.NewErr(consumererror.NewPermanent(errors.New("my error"))))
 	resp, err := logClient.Export(context.Background(), req)
-	assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = Permanent error: my error")
+	assert.EqualError(t, err, "rpc error: code = Internal desc = Permanent error: my error")
 	assert.IsType(t, status.Error(codes.Unknown, ""), err)
 	assert.Equal(t, plogotlp.ExportResponse{}, resp)
 }

receiver/otlpreceiver/internal/metrics/otlp_test.go

@@ -67,7 +67,7 @@ func TestExport_PermanentErrorConsumer(t *testing.T) {
 
 	metricsClient := makeMetricsServiceClient(t, consumertest.NewErr(consumererror.NewPermanent(errors.New("my error"))))
 	resp, err := metricsClient.Export(context.Background(), req)
-	assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = Permanent error: my error")
+	assert.EqualError(t, err, "rpc error: code = Internal desc = Permanent error: my error")
 	assert.IsType(t, status.Error(codes.Unknown, ""), err)
 	assert.Equal(t, pmetricotlp.ExportResponse{}, resp)
 }

receiver/otlpreceiver/internal/trace/otlp_test.go

@@ -64,7 +64,7 @@ func TestExport_PermanentErrorConsumer(t *testing.T) {
 
 	traceClient := makeTraceServiceClient(t, consumertest.NewErr(consumererror.NewPermanent(errors.New("my error"))))
 	resp, err := traceClient.Export(context.Background(), req)
-	assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = Permanent error: my error")
+	assert.EqualError(t, err, "rpc error: code = Internal desc = Permanent error: my error")
 	assert.IsType(t, status.Error(codes.Unknown, ""), err)
 	assert.Equal(t, ptraceotlp.ExportResponse{}, resp)
 }

receiver/otlpreceiver/otlp_test.go

@@ -501,7 +501,7 @@ func TestOTLPReceiverGRPCTracesIngestTest(t *testing.T) {
 		},
 		{
 			okToIngest:   false,
-			expectedCode: codes.InvalidArgument,
+			expectedCode: codes.Internal,
 			permanent:    true,
 		},
 		{