Kafka

---
kind: configuration/kafka
title: "Kafka"
name: default
specification:
  kafka_var:
    enabled: True
    admin: kafka
    admin_pwd: LambdaStack
    # javax_net_debug: all # uncomment to activate debugging, other debug options: https://colinpaice.blog/2020/04/05/using-java-djavax-net-debug-to-examine-data-flows-including-tls/
    security:
      ssl:
        enabled: False
        port: 9093
        server:
          local_cert_download_path: kafka-certs
          keystore_location: /var/private/ssl/kafka.server.keystore.jks
          truststore_location: /var/private/ssl/kafka.server.truststore.jks
          cert_validity: 365
          passwords:
            keystore: PasswordToChange
            truststore: PasswordToChange
            key: PasswordToChange
        endpoint_identification_algorithm: HTTPS
        client_auth: required
      encrypt_at_rest: False
      inter_broker_protocol: PLAINTEXT
      authorization:
        enabled: False
        authorizer_class_name: kafka.security.auth.SimpleAclAuthorizer
        allow_everyone_if_no_acl_found: False
        super_users:
          - tester01
          - tester02
        users:
          - name: test_user
            topic: test_topic
      authentication:
        enabled: False
        authentication_method: certificates
        sasl_mechanism_inter_broker_protocol:
        sasl_enabled_mechanisms: PLAIN
    sha: "b28e81705e30528f1abb6766e22dfe9dae50b1e1e93330c880928ff7a08e6b38ee71cbfc96ec14369b2dfd24293938702cab422173c8e01955a9d1746ae43f98"
    port: 9092
    min_insync_replicas: 1 # Minimum number of replicas (ack write)
    default_replication_factor: 1 # Minimum number of automatically created topics
    offsets_topic_replication_factor: 1 # Minimum number of offsets topic (consider higher value for HA)
    num_recovery_threads_per_data_dir: 1 # Minimum number of recovery threads per data dir
    num_replica_fetchers: 1 # Minimum number of replica fetchers
    replica_fetch_max_bytes: 1048576
    replica_socket_receive_buffer_bytes: 65536
    partitions: 8 # 100 x brokers x replicas for reasonable size cluster. Small clusters can be less
    log_retention_hours: 168 # The minimum age of a log file to be eligible for deletion due to age
    log_retention_bytes: -1 # -1 is no size limit only a time limit (log_retention_hours). This limit is enforced at the partition level, multiply it by the number of partitions to compute the topic retention in bytes.
    offset_retention_minutes: 10080 # Offsets older than this retention period will be discarded
    heap_opts: "-Xmx2G -Xms2G"
    opts: "-Djavax.net.debug=all"
    jmx_opts:
    max_incremental_fetch_session_cache_slots: 1000
    controlled_shutdown_enable: true
    group: kafka
    user: kafka
    conf_dir: /opt/kafka/config
    data_dir: /var/lib/kafka
    log_dir: /var/log/kafka
    socket_settings:
      network_threads: 3 # The number of threads handling network requests
      io_threads: 8 # The number of threads doing disk I/O
      send_buffer_bytes: 102400 # The send buffer (SO_SNDBUF) used by the socket server
      receive_buffer_bytes: 102400 # The receive buffer (SO_RCVBUF) used by the socket server      
      request_max_bytes: 104857600 # The maximum size of a request that the socket server will accept (protection against OOM)
  zookeeper_set_acl: false
  zookeeper_hosts: "{{ groups['zookeeper']|join(':2181,') }}:2181"
  jmx_exporter_user: jmx-exporter
  jmx_exporter_group: jmx-exporter
  prometheus_jmx_path: /opt/jmx-exporter/jmx_prometheus_javaagent.jar
  prometheus_jmx_exporter_web_listen_port: 7071
  prometheus_jmx_config: /opt/kafka/config/jmx-kafka.config.yml
  prometheus_config_dir: /etc/prometheus
  prometheus_kafka_jmx_file_sd_labels:
    "job": "jmx-kafka"