parentSpanId is empty in nodejs grpc server

[mostafafarzaneh]

What version of OpenTelemetry are you using?

"@opentelemetry/api": "^1.8.0",
"@opentelemetry/core": "^1.23.0",
"@opentelemetry/exporter-trace-otlp-grpc": "^0.50.0",
"@opentelemetry/id-generator-aws-xray": "^1.2.1",
"@opentelemetry/instrumentation-aws-sdk": "^0.40.0",
"@opentelemetry/instrumentation-express": "^0.37.0",
"@opentelemetry/instrumentation-grpc": "^0.50.0",
"@opentelemetry/instrumentation-http": "^0.50.0",
"@opentelemetry/propagator-aws-xray": "^1.3.1",
"@opentelemetry/resource-detector-aws": "^1.4.1",
"@opentelemetry/sdk-node": "^0.50.0",
"@opentelemetry/sdk-trace-base": "^1.23.0",
"@opentelemetry/sdk-trace-node": "^1.23.0",

What version of Node are you using?

v20.12.1

What did you do?

I have a NodeJs gRPC server that serves a Golang gRPC client. The client and server are set up to generate open telemetry traces and it is configured to be compatible with Xray.
The problem is that the NodeJs server generates traces that are not connected to client traces. I can see in the Xray console that the client and server traces are not connected.

Here is the server code:

(async () => {
  if (
    process.env.DISABLE_MONITORING == "0" ||
    process.env.DISABLE_MONITORING == undefined
  ) {
    const { setupTracing } = require("./otel");
    await setupTracing("cisco.WebexIntegration");
  }

  const grpc = require("@grpc/grpc-js");
  const wiServices = require("@jibb/jibbapis-nodejs/pb/cisco/webexintegration_service_grpc_pb");
  const macroServices = require("@jibb/jibbapis-nodejs/pb/cisco/macro_service_grpc_pb");
  const webexImp = require("../../src/services/webexintegration/src/webex_integration_imp");
  const macroImp = require("../../src/services/macro/macro_imp");
  const { HealthImplementation } = require("grpc-health-check");

  const server = new grpc.Server();

  const statusMap = {
    "": "SERVING",
  };
  const healthImpl = new HealthImplementation(statusMap);
  healthImpl.addToServer(server);
  healthImpl.setStatus("", "SERVING"); //The control plane has been configured with empty grpc health check

  server.addService(wiServices.WebexIntegrationService, webexImp);
  server.addService(macroServices.MacroService, macroImp);

  const port = process.env.WEBEX_INTEGRATION_PORT || "81";
  server.bindAsync(
    `0.0.0.0:${port}`,
    grpc.ServerCredentials.createInsecure(),
    () => {
      //server.start();
      console.log("WebexIntegration started on port ", port);
    }
  );
})();

and here is the otel code:

const { NodeSDK } = require("@opentelemetry/sdk-node");
const { AWSXRayPropagator } = require("@opentelemetry/propagator-aws-xray");
const { AWSXRayIdGenerator } = require("@opentelemetry/id-generator-aws-xray");
const { W3CTraceContextPropagator } = require("@opentelemetry/core");
const { BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base");
const { CompositePropagator } = require("@opentelemetry/core");
const {
  OTLPTraceExporter,
} = require("@opentelemetry/exporter-trace-otlp-grpc");
const { Resource } = require("@opentelemetry/resources");
const {
  SemanticResourceAttributes,
} = require("@opentelemetry/semantic-conventions");
const grpc = require("@grpc/grpc-js");
const { detectResourcesSync } = require("@opentelemetry/resources");
const { awsEcsDetector } = require("@opentelemetry/resource-detector-aws");

// Instrumentations
const { HttpInstrumentation } = require("@opentelemetry/instrumentation-http");
const { GrpcInstrumentation } = require("@opentelemetry/instrumentation-grpc");
const {
  ExpressInstrumentation,
} = require("@opentelemetry/instrumentation-express");
const { AwsInstrumentation } = require("@opentelemetry/instrumentation-aws-sdk");

const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api');
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);

async function setupTracing(serviceName) {
  const ecsResource = detectResourcesSync({ detectors: [awsEcsDetector] });

  const resource = Resource.default().merge(
    new Resource({
      [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
    }),
    ecsResource
  );

  const traceExporter = new OTLPTraceExporter({
    url: "0.0.0.0:4317", // Adjust this URL/port as necessary
    credentials: grpc.credentials.createInsecure(),
  });

  const spanProcessor = new BatchSpanProcessor(traceExporter);

  const sdk = new NodeSDK({
    resource: resource,
    spanProcessor: spanProcessor,
    instrumentations: [
      new HttpInstrumentation({
        ignoreIncomingPaths: [/^\/health/, /^\/metrics/],
      }),
      new ExpressInstrumentation({
        ignoreLayers: [/^\/health/, /^\/metrics/],
      }),
      new GrpcInstrumentation({
        ignoreGrpcMethods: ["/grpc.health.v1.Health/Check", "Check"],
      }),
      new AwsInstrumentation({
          suppressInternalInstrumentation: true
      }),
    ],
    idGenerator: new AWSXRayIdGenerator(),
    textMapPropagator: new AWSXRayPropagator(),

  });
  await sdk.start();
}

module.exports = { setupTracing };

I generated a debug in the client application and here is the trace:

x-amzn-trace-id:[Root=1-661c9fab-031aa61a7481ac4925ee28d0;Parent=568846939e558d55;Sampled=1]

and here is the debug in the server

items to be sent [
  Span {
    attributes: {
      'rpc.system': 'grpc',
      'rpc.method': 'GetWebexDevicesList',
      'rpc.service': 'cisco.WebexIntegration',
      'rpc.grpc.status_code': 0
    },
    links: [],
    events: [],
    _droppedAttributesCount: 0,
    _droppedEventsCount: 0,
    _droppedLinksCount: 0,
    status: { code: 0 },
    endTime: [ 1713151915, 637065054 ],
    _ended: true,
    _duration: [ 0, 69065054 ],
    name: 'grpc.cisco.WebexIntegration/GetWebexDevicesList',
    _spanContext: {
      traceId: '661c9fabbd0f2be904697ef045391bf8',
      spanId: '88bf7356c17d6e77',
      traceFlags: 1,
      traceState: undefined
    },
    parentSpanId: undefined,
    kind: 1,
    _performanceStartTime: 1663473.718939,
    _performanceOffset: -0.30810546875,
    _startTimeProvided: false,
    startTime: [ 1713151915, 568000000 ],
    resource: Resource {
      _attributes: [Object],
      asyncAttributesPending: false,
      _syncAttributes: [Object],
      _asyncAttributesPromise: [Promise]
    },
    instrumentationLibrary: {
      name: '@opentelemetry/instrumentation-grpc',
      version: '0.50.0',
      schemaUrl: undefined
    },
    _spanLimits: {
      attributeValueLengthLimit: Infinity,
      attributeCountLimit: 128,
      linkCountLimit: 128,
      eventCountLimit: 128,
      attributePerEventCountLimit: 128,
      attributePerLinkCountLimit: 128
    },
    _attributeValueLengthLimit: Infinity,
    _spanProcessor: MultiSpanProcessor { _spanProcessors: [Array] }
  }
]

What did you expect to see?

I was expecting that the traces from the client to server are connected.

What did you see instead?

I can see in the XRay console that the client sends traces to a remote service instead of a NodeJs service, and can see a separate traces fro NodeJs server

Additional context

[pichlermarc]

I think this may be related to import-ordering (@grpc/grpc-js) is required before the instrumentation is enabled, which means it's never instrumented. It's a limitation of the way we can currently configure the exporters as there's no way to programmatically pass metadata without requiring @grpc/grpc-js.

Temporary Fix: setting the env var OTEL_EXPORTER_OTLP_INSECURE=true, don't set credentials in the exporter, and only require @grpc/grpc-js in the server code after the instrumentation is registered.

Permanent Fix: I'll work on a way to allow us setting the metadata/credentials so that users don't have to to require @grpc/grpc-js for it - I've prepared something like this in the GrpcExporterTransport a while ago, we just have to expose it to users via the exporter itself.