PS D:\Code\observability> kubectl logs -n jaeger jaeger-794bffdd68-d4k8x -f
2025/06/05 21:36:41 application version: git-commit=02128431cc2566b5cc3f38b8aea88a30f182ce3d, git-version=v2.6.0, build-date=2025-05-09T19:30:14Z
2025-06-05T21:36:41.486+0800    info    service@v0.125.0/service.go:199 Setting up own telemetry...
2025-06-05T21:36:41.486+0800    info    builders/builders.go:26 Development component. May change in the future.        {"otelcol.component.id": "jaeger_storage_exporter", "otelcol.component.kind": "exporter", "otelcol.signal": "traces"}
2025-06-05T21:36:41.487+0800    info    builders/extension.go:50        Development component. May change in the future.        {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.505+0800    info    service@v0.125.0/service.go:266 Starting jaeger...      {"Version": "v2.6.0", "NumCPU": 16}
2025-06-05T21:36:41.505+0800    info    extensions/extensions.go:41     Starting extensions...
2025-06-05T21:36:41.505+0800    info    extensions/extensions.go:45     Extension is starting...        {"otelcol.component.id": "pprof", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.505+0800    info    pprofextension@v0.125.0/pprofextension.go:61    Starting net/http/pprof server  {"otelcol.component.id": "pprof", "otelcol.component.kind": "extension", "config": {"TCPAddr":{"Endpoint":"0.0.0.0:1777","DialerConfig":{"Timeout":0}},"BlockProfileFraction":0,"MutexProfileFraction":0,"SaveToFile":""}}
2025-06-05T21:36:41.505+0800    info    extensions/extensions.go:62     Extension started.      {"otelcol.component.id": "pprof", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.505+0800    info    extensions/extensions.go:45     Extension is starting...        {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.506+0800    info    extensions/extensions.go:62     Extension started.      {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.506+0800    info    extensions/extensions.go:45     Extension is starting...        {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.506+0800    info    jaegerstorage/extension.go:159  Initializing storage 'main'     {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.520+0800    info    config/config.go:302    Elasticsearch detected  {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "version": 9}
2025-06-05T21:36:41.520+0800    info    jaegerstorage/extension.go:159  Initializing storage 'archive'  {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.603+0800    info    config/config.go:302    Elasticsearch detected  {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "version": 9}
2025-06-05T21:36:41.603+0800    info    extensions/extensions.go:62     Extension started.      {"otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.603+0800    info    extensions/extensions.go:45     Extension is starting...        {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.603+0800    info    jaegerquery/server.go:144       Archive storage not configured  {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.603+0800    info    jaegerquery/server.go:186       Metric storage not configured   {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.603+0800    info    app/static_handler.go:92        Using UI configuration  {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "path": "/mnt/config-ui.json"}
2025-06-05T21:36:41.604+0800    info    app/server.go:256       Query server started    {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "http_addr": "[::]:16686", "grpc_addr": "[::]:16685"}
2025-06-05T21:36:41.604+0800    info    extensions/extensions.go:62     Extension started.      {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.604+0800    info    app/server.go:284       Starting HTTP server    {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16686, "addr": "0.0.0.0:16686"}
2025-06-05T21:36:41.604+0800    info    app/server.go:298       Starting GRPC server    {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16685, "addr": "0.0.0.0:16685"}
2025-06-05T21:36:41.620+0800    error   graph/graph.go:438      Failed to start component       {"error": "cannot create trace writer: failed to create template \"jaeger-main-jaeger-span\": error creating index template jaeger-main-jaeger-span: [400 Bad Request] {\"error\":{\"root_cause\":[{\"type\":\"parse_exception\",\"reason\":\"request body is required\"}],\"type\":\"parse_exception\",\"reason\":\"request body is required\"},\"status\":400}", "type": "Exporter", "id": "jaeger_storage_exporter"}
2025-06-05T21:36:41.620+0800    info    service@v0.125.0/service.go:331 Starting shutdown...
2025-06-05T21:36:41.620+0800    info    extensions/extensions.go:69     Stopping extensions...
2025-06-05T21:36:41.620+0800    info    app/server.go:323       Closing HTTP server     {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.620+0800    info    app/server.go:291       HTTP server stopped     {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16686, "addr": "0.0.0.0:16686"}
2025-06-05T21:36:41.624+0800    info    app/server.go:328       Stopping gRPC server    {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.624+0800    info    app/server.go:306       GRPC server stopped     {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16685, "addr": "0.0.0.0:16685"}
2025-06-05T21:36:41.624+0800    info    app/server.go:333       Server stopped  {"otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2025-06-05T21:36:41.625+0800    info    healthcheckv2extension@v0.125.0/extension.go:138        discarding event received after shutdown        {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension", "source": {}, "event": {}}
2025-06-05T21:36:41.625+0800    info    healthcheckv2extension@v0.125.0/extension.go:138        discarding event received after shutdown        {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension", "source": {}, "event": {}}
2025-06-05T21:36:41.625+0800    info    healthcheckv2extension@v0.125.0/extension.go:138        discarding event received after shutdown        {"otelcol.component.id": "healthcheckv2", "otelcol.component.kind": "extension", "source": {}, "event": {}}
2025-06-05T21:36:41.625+0800    info    service@v0.125.0/service.go:345 Shutdown complete.
Error: cannot start pipelines: failed to start "jaeger_storage_exporter" exporter: cannot create trace writer: failed to create template "jaeger-main-jaeger-span": error creating index template jaeger-main-jaeger-span: [400 Bad Request] {"error":{"root_cause":[{"type":"parse_exception","reason":"request body is required"}],"type":"parse_exception","reason":"request body is required"},"status":400}
2025/06/05 21:36:41 cannot start pipelines: failed to start "jaeger_storage_exporter" exporter: cannot create trace writer: failed to create template "jaeger-main-jaeger-span": error creating index template jaeger-main-jaeger-span: [400 Bad Request] {"error":{"root_cause":[{"type":"parse_exception","reason":"request body is required"}],"type":"parse_exception","reason":"request body is required"},"status":400}

apiVersion: v1
kind: Namespace
metadata:
  name: jaeger
  labels:
    kubernetes.io/metadata.name: jaeger
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: jaeger-config
  namespace: jaeger
data:
  config-ui.json: |
    {
      "archiveEnabled": true,
      "dependencies": {
        "dagMaxNumServices": 500,
        "menuEnabled": true
      },
      "monitor": {
        "menuEnabled": true
      },
      "tracking": {
        "gaID": null,
        "customWebAnalytics": null,
        "trackErrors": true
      },
      "menu": [
        {
          "label": "关于",
          "items": [
            {
              "label": "GitHub",
              "url": "https://github.com/jaegertracing/jaeger"
            },
            {
              "label": "文档",
              "url": "https://www.jaegertracing.io/docs/latest/"
            },
            {
              "label": "部署",
              "url": "https://gitlab.glm.ai/devops/observability.git"
            },
            {
              "label": "镜像",
              "url": "https://gitlab.glm.ai/devops/jaeger.git"
            }
          ]
        }
      ],
      "search": {
        "maxLimit": 2500,
        "maxLookback": {
          "label": "2天",
          "value": "2d"
        }
      },
      "linkPatterns": [{
        "type": "process",
        "key": "jaeger.version",
        "url": "https://github.com/jaegertracing/jaeger-client-java/releases/tag/#{jaeger.version}",
        "text": "Information about Jaeger release #{jaeger.version}"
      },
      {
        "type": "tags",
        "key": "uniqueId",
        "url": "https://mykibana.com/uniqueId=#{uniqueId}&traceId=#{trace.traceID}",
        "text": "Redirect to kibana to view log"
      },
      {
        "type": "traces",
        "url": "https://my-logs.server?from=#{startTime | add -60000000 | epoch_micros_to_date_iso}&to=#{endTime | add 60000000 | epoch_micros_to_date_iso}'",
        "text": "Redirect to kibana to view log with formatted dates"
      }]
    }
  sampling-strategies.json: |
    {
      "default_strategy": {
        "type": "probabilistic",
        "param": 1
      }
    }
  config.yaml: |
    service:
      extensions: [jaeger_storage, jaeger_query, healthcheckv2, pprof]
      pipelines:
        traces:
          receivers: [otlp, jaeger, zipkin]
          processors: [batch]
          exporters: [jaeger_storage_exporter]
      telemetry:
        resource:
          service.name: jaeger
        metrics:
          level: detailed
          readers:
            - pull:
                exporter:
                  prometheus:
                    host: 0.0.0.0
                    port: 8888
        logs:
          level: info
    extensions:
      healthcheckv2:
        use_v2: true
        http:
      pprof:
        endpoint: 0.0.0.0:1777
      jaeger_storage:
        backends:
          main:
            elasticsearch:
              server_urls: 
              - http://elastic:WyE3qnAm277ew229Z83IGdD3@infra-es-http.elastic.svc:9200
              indices:
                index_prefix: "jaeger-main"
                spans:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                services:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                dependencies:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                sampling:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
          archive:
            elasticsearch:
              server_urls:
                - http://elastic:WyE3qnAm277ew229Z83IGdD3@infra-es-http.elastic.svc:9200
              indices:
                index_prefix: "jaeger-archive"
                spans:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                services:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                dependencies:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
                sampling:
                  date_layout: "2006-01-02"
                  rollover_frequency: "day"
                  shards: 5
                  replicas: 1
      jaeger_query:
        max_clock_skew_adjust: 30s
        base_path: /
        storage:
          traces: main
        ui:
          config_file: /mnt/config-ui.json
          log_access: true
        grpc:
            endpoint: 0.0.0.0:16685
        http:
          endpoint: 0.0.0.0:16686
    receivers:
      otlp:
        protocols:
          grpc: 
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
      jaeger:
        protocols:
          grpc:
             endpoint: 0.0.0.0:14250
          thrift_binary:
          thrift_compact:
          thrift_http:
             endpoint: 0.0.0.0:14268
      zipkin:
    processors:
      batch:
        timeout: 5s
        send_batch_size: 1000
        send_batch_max_size: 2000
    exporters:
      jaeger_storage_exporter:
        trace_storage: main
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jaeger
  namespace: jaeger
  labels:
    app: jaeger
spec:
  selector:
    matchLabels:
      app: jaeger
  template:
    metadata:
      labels:
        app: jaeger
    spec:
      hostNetwork: false
      hostPID: false
      hostIPC: false
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
      - key: node-role.kubernetes.io/master
        operator: Exists
        effect: NoSchedule
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
      imagePullSecrets:
      - name: jaeger-pull-token
      containers:
        - name: jaeger
          image: "registry.glm.ai/devops/jaeger/jaeger:1.0.2"
          imagePullPolicy: IfNotPresent
          command: 
          - /usr/local/bin/jaeger
          args:
          - --config
          - /mnt/config.yaml
          resources:
            limits:
              memory: "4Gi"
              cpu: "2"
            requests:
              memory: "128Mi"
              cpu: "100m"
          ports:
            - containerPort: 8888
              name: metrics
              protocol: TCP
            - containerPort: 1777
              name: pprof
              protocol: TCP
            - containerPort: 16685
              name: query-grpc
              protocol: TCP
            - containerPort: 16686
              name: query-http
              protocol: TCP
            - containerPort: 4317
              name: recv-otlp-grpc
              protocol: TCP
            - containerPort: 4318
              name: recv-otlp-http
              protocol: TCP
            - containerPort: 14250
              name: recv-jv2-grpc
              protocol: TCP
            - containerPort: 14268
              name: recv-jv2-http
              protocol: TCP
          volumeMounts:
          - name: jaeger-config
            mountPath: /mnt
            readOnly: true
      volumes:
      - name: jaeger-config
        configMap:
          name: jaeger-config
          items: 
          - key: config.yaml
            path: config.yaml
          - key: config-ui.json
            path: config-ui.json
          - key: sampling-strategies.json
            path: sampling-strategies.json