[Bug]: podTemplate securityContext doesn't honor sysctl values about strimzi-kafka-operator HOT 12 CLOSED

$ kubectl get pod my-bridge-bridge-5cdd484c55-sp98n -o jsonpath='{.spec.securityContext}'
{"sysctls":[{"name":"net.ipv4.tcp_keepalive_time","value":"300"}]}

apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
  name: lc-standalone
  namespace: kafka
  annotations:
    strimzi.io/node-pools: enabled
spec:
  kafka:
    version: 3.7.0
    replicas: 1
    metricsConfig:
      type: jmxPrometheusExporter
      valueFrom:
        configMapKeyRef:
          name: kafka-metrics
          key: kafka-metrics-config.yml
    template:
      pod:
        securityContext:
          sysctls:
          - name: net.ipv4.tcp_keepalive_time
            value: "300"  # Default 7200
    authorization:
      type: simple
      superUsers:
      - ***
      - ***
      - ***
      - ***
      - ***
      - ***
    listeners:
    - name: plain
      port: 9092
      type: internal
      tls: false
    - name: tls
      port: 9093
      type: internal
      tls: true
    - name: external
      port: 9094
      type: loadbalancer
      tls: false
      authentication:
        type: scram-sha-512
      configuration:
        createBootstrapService: false
        externalTrafficPolicy: Local
        brokers:
        - broker: 0
          advertisedHost: kafka.***
          loadBalancerIP: ***
    - name: tlsauth
      port: 9095
      type: internal
      tls: true
      authentication:
        type: tls
    config:
      auto.create.topics.enable: false
      # inter.broker.protocol.version: "3.2"
      default.replication.factor: 1
      offsets.topic.replication.factor: 1
      transaction.state.log.replication.factor: 1
      transaction.state.log.min.isr: 1
      # message.max.bytes: 1048588
      # No replication so make sure we flush to disk more often
      # flush every 100k messages
      # log.flush.interval.messages: 100000
      # The maximum amount of time a message can sit in a log before we force a flush
      # Flush after 1 minute
      # log.flush.interval.ms: 60000
      log.flush.interval.messages: 1   # Test flushing every message
      # Setting log flush interval won't work unless you check for flush
      # log.flush.scheduler.interval.ms: 60000
      # Define whether the timestamp in the message is message create time or log append time.
      # The value should be either `CreateTime` or `LogAppendTime`
      # we set this to log append time to enable ingest to process by receive date
      log.message.timestamp.type: LogAppendTime
      # Retention settings
      # Set 60 day minimum for NEON on LC
      log.retention.hours: 1440
      # The maximum size of a single log file
      # Default 1.1GiB
      # log.segment.bytes: 1073741824
      # Try 200 MiB
      log.segment.bytes: 214748365
      # Reduce index segment size too
      # log.segment.index.bytes: 10485760
      log.segment.index.bytes: 2621440
      # Default roll every 7 days
      # log.roll.hours: 168
      # Default log segment sizes is 1GiB
      # log.segment.bytes: 1073741824
      # The amount of time to wait before deleting a file from the filesystem (default 1 minute)
      # log.segment.delete.delay.ms: 60000
      # After a consumer group loses all its consumers (i.e. becomes empty) its offsets will be kept for this retention
      # period before getting discarded. For standalone consumers (using manual assignment), offsets will be expired after
      # the time of last commit plus this retention period. (Default 7 days)
      # offsets.retention.minutes: 10080
      # Up this to 30 days as well (mm2 offset)
      # Note that this does not affect mirrormaker2 as it tracks it's own offsets
      offsets.retention.minutes: 43200
      # The configuration controls the maximum amount of time the client will wait for the response of a request. If the response
      # is not received before the timeout elapses the client will resend the request if necessary or fail the request if retries
      # are exhausted. (default 30000)
      # request.timeout.ms: 30000
      # Let the OS set tcp send/receive buffers
      socket.send.buffer.bytes: -1
      socket.receive.buffer.bytes: -1
  zookeeper:
    metricsConfig:
      type: jmxPrometheusExporter
      valueFrom:
        configMapKeyRef:
          name: kafka-metrics
          key: zookeeper-metrics-config.yml
    replicas: 1
    jvmOptions:
      -XX:
        ExplicitGCInvokesConcurrent: "true"
        InitiatingHeapOccupancyPercent: "35"
        MaxGCPauseMillis: "20"
        MaxInlineLevel: "15"
        UseG1GC: "true"
      -Xms: 512m
      -Xmx: 512m
      javaSystemProperties:
      - name: java.awt.headless
        value: "true"
    resources:
      requests:
        memory: 2.0Gi
        cpu: "0.01"
      limits:
        memory: 2.0Gi
        cpu: "0.5"
    storage:
      type: persistent-claim
      size: 5110Mi
      deleteClaim: false
      class: local-storage-zk
  entityOperator:
    # topicOperator:
    #   resources:
    #     requests:
    #       memory: 512Mi
    #       cpu: "0.01"
    #     limits:
    #       memory: 512Mi
    #       cpu: "0.5"
    userOperator:
      resources:
        requests:
          memory: 768Mi
          cpu: "0.01"
        limits:
          memory: 768Mi
          cpu: "0.5"
  kafkaExporter:
    topicRegex: ".*"
    groupRegex: ".*"

apiVersion: core.strimzi.io/v1beta2
kind: StrimziPodSet
metadata:
  annotations:
    strimzi.io/kafka-version: 3.7.0
    strimzi.io/storage: '{"volumes":[{"type":"persistent-claim","size":"99Gi","class":"local-storage-kafka","deleteClaim":true,"id":0}],"type":"jbod"}'
  creationTimestamp: "2024-02-05T17:21:37Z"
  generation: 6
  labels:
    app.kubernetes.io/instance: lc-standalone
    app.kubernetes.io/managed-by: strimzi-cluster-operator
    app.kubernetes.io/name: kafka
    app.kubernetes.io/part-of: strimzi-lc-standalone
    strimzi.io/cluster: lc-standalone
    strimzi.io/component-type: kafka
    strimzi.io/kind: Kafka
    strimzi.io/name: lc-standalone-kafka
    strimzi.io/pool-name: kafka
  name: lc-standalone-kafka
  namespace: kafka
  ownerReferences:
  - apiVersion: kafka.strimzi.io/v1beta2
    blockOwnerDeletion: false
    controller: false
    kind: KafkaNodePool
    name: kafka
    uid: b5b16d58-ee30-473f-96ff-2a472355d8c3
  resourceVersion: "34214496"
  uid: 0ca5737c-5105-4d24-b3dd-230410df20b3
spec:
  pods:
  - apiVersion: v1
    kind: Pod
    metadata:
      annotations:
        strimzi.io/broker-configuration-hash: 2ecbbbe0
        strimzi.io/clients-ca-cert-generation: "0"
        strimzi.io/cluster-ca-cert-generation: "0"
        strimzi.io/cluster-ca-key-generation: "0"
        strimzi.io/inter-broker-protocol-version: "3.7"
        strimzi.io/kafka-version: 3.7.0
        strimzi.io/log-message-format-version: "3.7"
        strimzi.io/logging-appenders-hash: e893ac9f
        strimzi.io/revision: d7a84336
        strimzi.io/server-cert-hash: c0a2e7dca192da31672121098141b12ed037beb3
      labels:
        app.kubernetes.io/instance: lc-standalone
        app.kubernetes.io/managed-by: strimzi-cluster-operator
        app.kubernetes.io/name: kafka
        app.kubernetes.io/part-of: strimzi-lc-standalone
        statefulset.kubernetes.io/pod-name: lc-standalone-kafka-0
        strimzi.io/broker-role: "true"
        strimzi.io/cluster: lc-standalone
        strimzi.io/component-type: kafka
        strimzi.io/controller: strimzipodset
        strimzi.io/controller-name: lc-standalone-kafka
        strimzi.io/controller-role: "false"
        strimzi.io/kind: Kafka
        strimzi.io/name: lc-standalone-kafka
        strimzi.io/pod-name: lc-standalone-kafka-0
        strimzi.io/pool-name: kafka
      name: lc-standalone-kafka-0
      namespace: kafka
    spec:
      affinity: {}
      containers:
      - args:
        - /opt/kafka/kafka_run.sh
        env:
        - name: KAFKA_METRICS_ENABLED
          value: "true"
        - name: STRIMZI_KAFKA_GC_LOG_ENABLED
          value: "false"
        - name: KAFKA_HEAP_OPTS
          value: -Xms1g -Xmx1g
        - name: KAFKA_JVM_PERFORMANCE_OPTS
          value: -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=35
            -XX:MaxDirectMemorySize=128m -XX:MaxGCPauseMillis=20 -XX:MetaspaceSize=20m
            -XX:+UseZGC
        - name: STRIMZI_JAVA_SYSTEM_PROPERTIES
          value: -Djava.awt.headless=true
        image: quay.io/strimzi/kafka:0.40.0-kafka-3.7.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - /opt/kafka/kafka_liveness.sh
          initialDelaySeconds: 15
          timeoutSeconds: 5
        name: kafka
        ports:
        - containerPort: 9090
          name: tcp-ctrlplane
          protocol: TCP
        - containerPort: 9091
          name: tcp-replication
          protocol: TCP
        - containerPort: 9092
          name: tcp-clients
          protocol: TCP
        - containerPort: 9093
          name: tcp-clientstls
          protocol: TCP
        - containerPort: 9094
          name: tcp-external
          protocol: TCP
        - containerPort: 9095
          name: tcp-tlsauth
          protocol: TCP
        - containerPort: 9404
          name: tcp-prometheus
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - /opt/kafka/kafka_readiness.sh
          initialDelaySeconds: 15
          timeoutSeconds: 5
        resources:
          limits:
            cpu: "1.0"
            memory: 6Gi
          requests:
            cpu: "0.2"
            memory: 6Gi
        volumeMounts:
        - mountPath: /var/lib/kafka/data-0
          name: data-0
        - mountPath: /tmp
          name: strimzi-tmp
        - mountPath: /opt/kafka/cluster-ca-certs
          name: cluster-ca
        - mountPath: /opt/kafka/broker-certs
          name: broker-certs
        - mountPath: /opt/kafka/client-ca-certs
          name: client-ca-cert
        - mountPath: /opt/kafka/custom-config/
          name: kafka-metrics-and-logging
        - mountPath: /var/opt/kafka
          name: ready-files
      hostname: lc-standalone-kafka-0
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 0
      serviceAccountName: lc-standalone-kafka
      subdomain: lc-standalone-kafka-brokers
      terminationGracePeriodSeconds: 30
      volumes:
      - name: data-0
        persistentVolumeClaim:
          claimName: data-0-lc-standalone-kafka-0
      - emptyDir:
          medium: Memory
          sizeLimit: 10Mi
        name: strimzi-tmp
      - name: cluster-ca
        secret:
          defaultMode: 292
          secretName: lc-standalone-cluster-ca-cert
      - name: broker-certs
        secret:
          defaultMode: 292
          secretName: lc-standalone-kafka-brokers
      - name: client-ca-cert
        secret:
          defaultMode: 292
          secretName: lc-standalone-clients-ca-cert
      - configMap:
          name: lc-standalone-kafka-0
        name: kafka-metrics-and-logging
      - emptyDir:
          medium: Memory
          sizeLimit: 1Ki
        name: ready-files
  selector:
    matchLabels:
      strimzi.io/cluster: lc-standalone
      strimzi.io/kind: Kafka
      strimzi.io/name: lc-standalone-kafka
      strimzi.io/pool-name: kafka
status:
  currentPods: 1
  observedGeneration: 6
  pods: 1
  readyPods: 1

apiVersion: v1
kind: Pod
metadata:
  annotations:
    strimzi.io/broker-configuration-hash: 2ecbbbe0
    strimzi.io/clients-ca-cert-generation: "0"
    strimzi.io/cluster-ca-cert-generation: "0"
    strimzi.io/cluster-ca-key-generation: "0"
    strimzi.io/inter-broker-protocol-version: "3.7"
    strimzi.io/kafka-version: 3.7.0
    strimzi.io/log-message-format-version: "3.7"
    strimzi.io/logging-appenders-hash: e893ac9f
    strimzi.io/revision: d7a84336
    strimzi.io/server-cert-hash: c0a2e7dca192da31672121098141b12ed037beb3
  creationTimestamp: "2024-05-03T02:38:27Z"
  labels:
    app.kubernetes.io/instance: lc-standalone
    app.kubernetes.io/managed-by: strimzi-cluster-operator
    app.kubernetes.io/name: kafka
    app.kubernetes.io/part-of: strimzi-lc-standalone
    statefulset.kubernetes.io/pod-name: lc-standalone-kafka-0
    strimzi.io/broker-role: "true"
    strimzi.io/cluster: lc-standalone
    strimzi.io/component-type: kafka
    strimzi.io/controller: strimzipodset
    strimzi.io/controller-name: lc-standalone-kafka
    strimzi.io/controller-role: "false"
    strimzi.io/kind: Kafka
    strimzi.io/name: lc-standalone-kafka
    strimzi.io/pod-name: lc-standalone-kafka-0
    strimzi.io/pool-name: kafka
  name: lc-standalone-kafka-0
  namespace: kafka
  ownerReferences:
  - apiVersion: core.strimzi.io/v1beta2
    blockOwnerDeletion: false
    controller: true
    kind: StrimziPodSet
    name: lc-standalone-kafka
    uid: 0ca5737c-5105-4d24-b3dd-230410df20b3
  resourceVersion: "34214491"
  uid: 50a2c229-62bb-4de7-950d-cee2cfbdaf66
spec:
  affinity: {}
  containers:
  - args:
    - /opt/kafka/kafka_run.sh
    env:
    - name: KAFKA_METRICS_ENABLED
      value: "true"
    - name: STRIMZI_KAFKA_GC_LOG_ENABLED
      value: "false"
    - name: KAFKA_HEAP_OPTS
      value: -Xms1g -Xmx1g
    - name: KAFKA_JVM_PERFORMANCE_OPTS
      value: -XX:+ExplicitGCInvokesConcurrent -XX:InitiatingHeapOccupancyPercent=35
        -XX:MaxDirectMemorySize=128m -XX:MaxGCPauseMillis=20 -XX:MetaspaceSize=20m
        -XX:+UseZGC
    - name: STRIMZI_JAVA_SYSTEM_PROPERTIES
      value: -Djava.awt.headless=true
    image: quay.io/strimzi/kafka:0.40.0-kafka-3.7.0
    imagePullPolicy: IfNotPresent
    livenessProbe:
      exec:
        command:
        - /opt/kafka/kafka_liveness.sh
      failureThreshold: 3
      initialDelaySeconds: 15
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 5
    name: kafka
    ports:
    - containerPort: 9090
      name: tcp-ctrlplane
      protocol: TCP
    - containerPort: 9091
      name: tcp-replication
      protocol: TCP
    - containerPort: 9092
      name: tcp-clients
      protocol: TCP
    - containerPort: 9093
      name: tcp-clientstls
      protocol: TCP
    - containerPort: 9094
      name: tcp-external
      protocol: TCP
    - containerPort: 9095
      name: tcp-tlsauth
      protocol: TCP
    - containerPort: 9404
      name: tcp-prometheus
      protocol: TCP
    readinessProbe:
      exec:
        command:
        - /opt/kafka/kafka_readiness.sh
      failureThreshold: 3
      initialDelaySeconds: 15
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 5
    resources:
      limits:
        cpu: "1"
        memory: 6Gi
      requests:
        cpu: 200m
        memory: 6Gi
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/lib/kafka/data-0
      name: data-0
    - mountPath: /tmp
      name: strimzi-tmp
    - mountPath: /opt/kafka/cluster-ca-certs
      name: cluster-ca
    - mountPath: /opt/kafka/broker-certs
      name: broker-certs
    - mountPath: /opt/kafka/client-ca-certs
      name: client-ca-cert
    - mountPath: /opt/kafka/custom-config/
      name: kafka-metrics-and-logging
    - mountPath: /var/opt/kafka
      name: ready-files
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-zshz4
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  hostname: lc-standalone-kafka-0
  nodeName: lc-hqtw.hqtw.eng.neoninternal.org
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 0
  serviceAccount: lc-standalone-kafka
  serviceAccountName: lc-standalone-kafka
  subdomain: lc-standalone-kafka-brokers
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - name: data-0
    persistentVolumeClaim:
      claimName: data-0-lc-standalone-kafka-0
  - emptyDir:
      medium: Memory
      sizeLimit: 10Mi
    name: strimzi-tmp
  - name: cluster-ca
    secret:
      defaultMode: 292
      secretName: lc-standalone-cluster-ca-cert
  - name: broker-certs
    secret:
      defaultMode: 292
      secretName: lc-standalone-kafka-brokers
  - name: client-ca-cert
    secret:
      defaultMode: 292
      secretName: lc-standalone-clients-ca-cert
  - configMap:
      defaultMode: 420
      name: lc-standalone-kafka-0
    name: kafka-metrics-and-logging
  - emptyDir:
      medium: Memory
      sizeLimit: 1Ki
    name: ready-files
  - name: kube-api-access-zshz4
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2024-05-03T02:38:28Z"
    status: "True"
    type: PodReadyToStartContainers
  - lastProbeTime: null
    lastTransitionTime: "2024-05-03T02:38:27Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2024-05-03T02:39:27Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2024-05-03T02:39:27Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2024-05-03T02:38:27Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: containerd://1cfeba888e747df7a3516e8a244d4cb55de1038f3a6a2fc4c899e8e8f740439b
    image: quay.io/strimzi/kafka:0.40.0-kafka-3.7.0
    imageID: quay.io/strimzi/kafka@sha256:c491a95f9b9b58f406461faf222ef612ad8d2f4d7b53f7f3645b20c99d7d1583
    lastState:
      terminated:
        containerID: containerd://34147edc4b6ddab8cc6f0c7e5a74633a69946fb6d1a3c490a4690301e2a7876a
        exitCode: 1
        finishedAt: "2024-05-03T02:39:00Z"
        reason: Error
        startedAt: "2024-05-03T02:38:27Z"
    name: kafka
    ready: true
    restartCount: 1
    started: true
    state:
      running:
        startedAt: "2024-05-03T02:39:01Z"
  hostIP: 10.123.33.2
  hostIPs:
  - ip: 10.123.33.2
  phase: Running
  podIP: 192.168.0.54
  podIPs:
  - ip: 192.168.0.54
  qosClass: Burstable
  startTime: "2024-05-03T02:38:27Z"

 status:
    clusterId: y-FS0TwGRmag5pUS5sTYRg
    conditions:
    - lastTransitionTime: "2024-05-07T18:54:14.082768572Z"
      message: The .spec.kafka.replicas property in the Kafka custom resource is ignored
        when node pools are used and should be removed from the custom resource.
      reason: UnusedReplicasConfiguration
      status: "True"
      type: Warning
    - lastTransitionTime: "2024-05-07T18:54:15.872445580Z"
      status: "True"
      type: Ready
    kafkaMetadataState: ZooKeeper
    kafkaNodePools:
    - name: kafka
  ...
    observedGeneration: 7
    operatorLastSuccessfulVersion: 0.40.0