How to send application trace data to OpenTelemetry in Kubernetes?

[seikyo-cho-lvgs]

I added open-telemetry helm chart

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts

Install opentelemery operator

helm install opentelemetry-operator open-telemetry/opentelemetry-operator \
--set "manager.collectorImage.repository=otel/opentelemetry-collector-k8s"

Install opentelemery collector

helm install opentelemetry-collector open-telemetry/opentelemetry-collector --set mode=deployment --set image.repository="otel/opentelemetry-collector-k8s" --set command.name="otelcol-k8s" -f values.yaml

values.yaml

mode: deployment

config:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318
  exporters:
    otlp:
      endpoint: jaeger-collector.default.svc.cluster.local:14250
      tls:
        insecure: true
    debug:
      verbosity: detailed
  service:
    pipelines:
      traces:
        receivers: [otlp]
        exporters: [otlp]
      metrics:
        receivers: [otlp]
        exporters: [debug]
      logs:
        receivers: [otlp]
        exporters: [debug]

Install Jaeger

helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
helm install jaeger jaegertracing/jaeger

kubectl get pods

NAME                                      READY   STATUS      RESTARTS         AGE
jaeger-agent-gsrxq                        1/1     Running     0                3h40m
jaeger-cassandra-0                        1/1     Running     0                3h40m
jaeger-cassandra-1                        1/1     Running     0                3h38m
jaeger-cassandra-2                        1/1     Running     0                3h37m
jaeger-cassandra-schema-dqbvd             0/1     Completed   1                3h40m
jaeger-collector-68dbd7f84b-tznkk         1/1     Running     5 (3h38m ago)    3h40m
jaeger-query-684f47b76b-pgjc6             2/2     Running     5 (3h38m ago)    3h40m
opentelemetry-collector-56bc5b788b-5sx5z  1/1     Running     0                50m
opentelemetry-operator-54dd75d64f-t5l9v   2/2     Running     4 (3h54m ago)    6h12m

kubectl get services

NAME                             TYPE           CLUSTER-IP      EXTERNAL-IP    PORT(S)                                         AGE
jaeger-agent                     ClusterIP      10.43.47.36     <none>         5775/UDP,6831/UDP,6832/UDP,5778/TCP,14271/TCP   3h41m
jaeger-cassandra                 ClusterIP      None            <none>         7000/TCP,7001/TCP,7199/TCP,9042/TCP,9160/TCP    3h41m
jaeger-collector                 ClusterIP      10.43.209.8     <none>         14250/TCP,14268/TCP,14269/TCP                   3h41m
jaeger-query                     ClusterIP      10.43.36.237    <none>         80/TCP,16685/TCP,16687/TCP                      3h41m
opentelemetry-collector          ClusterIP      10.43.8.16      <none>         6831/UDP,14250/TCP,14268/TCP,4317/TCP,4318/TCP,9411/TCP   25m
opentelemetry-operator           ClusterIP      10.43.237.13    <none>         8443/TCP,8080/TCP                               6h13m
opentelemetry-operator-webhook   ClusterIP      10.43.51.200    <none>         443/TCP                                         6h13m

Port forward

kubectl port-forward svc/opentelemetry-collector 4317:4317
kubectl port-forward svc/opentelemetry-collector 4318:4318

Run an application

/*app.ts*/
import express, { Express } from 'express';

const PORT: number = parseInt(process.env.PORT || '8080');
const app: Express = express();

function getRandomNumber(min: number, max: number) {
  return Math.floor(Math.random() * (max - min) + min);
}

app.get('/rolldice', (req, res) => {
  res.send(getRandomNumber(1, 6).toString());
});

app.listen(PORT, () => {
  console.log(`Listening for requests on http://localhost:${PORT}`);
});

/*instrumentation.ts*/
import * as opentelemetry from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';

const resource = new Resource({
  [SemanticResourceAttributes.SERVICE_NAME]: 'test-service',
});

const sdk = new opentelemetry.NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: 'http://localhost:4317',
    // optional - collection of custom headers to be sent with each request, empty by default
    headers: {},
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://localhost:4317',
      headers: {}, // an optional object containing custom headers to be sent with each request
    }),
  }),
  resource: resource,
  instrumentations: [getNodeAutoInstrumentations()],
});
sdk.start();

// Run
npx ts-node --require ./instrumentation.ts app.ts

Send access

curl http://localhost:8080/rolldice
curl http://localhost:8080/rolldice

The opentelemetry-collector pod got error log:

2024-07-19T08:50:13.705Z	error	exporterhelper/queue_sender.go:90	Exporting failed. Dropping data.	{"kind": "exporter", "data_type": "traces", "name": "otlp", "error": "not retryable error: Permanent error: rpc error: code = Unimplemented desc = unknown service opentelemetry.proto.collector.trace.v1.TraceService", "dropped_items": 2}
go.opentelemetry.io/collector/exporter/exporterhelper.newQueueSender.func1
	go.opentelemetry.io/collector/[email protected]/exporterhelper/queue_sender.go:90
go.opentelemetry.io/collector/exporter/internal/queue.(*boundedMemoryQueue[...]).Consume
	go.opentelemetry.io/collector/[email protected]/internal/queue/bounded_memory_queue.go:52
go.opentelemetry.io/collector/exporter/internal/queue.(*Consumers[...]).Start.func1
	go.opentelemetry.io/collector/[email protected]/internal/queue/consumers.go:43

How to setup correctly?