feat: add test case for Kerberos Kafka authentication

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -16,6 +16,8 @@ services:
       - "9093:9093"
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
+    depends_on:
+      - "kdc" 
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
@@ -23,14 +25,32 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
+      KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
       KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: true
-
       KAFKA_CREATE_TOPICS: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
+      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
     networks:
       emqx_bridge:
     volumes:
+      - emqx-shared-secret:/var/lib/secret
       - ./kafka/jaas.conf:/etc/kafka/jaas.conf
       - ./kafka/run_add_scram_users.sh:/bin/run_add_scram_users.sh
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
     command: run_add_scram_users.sh
+  kdc:
+    hostname: kdc.emqx.net
+    image:  ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04
+    container_name: kdc.emqx.net
+    networks:
+      emqx_bridge:
+    volumes:
+      - emqx-shared-secret:/var/lib/secret
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
+      - ./kerberos/run.sh:/usr/bin/run.sh
+    command: run.sh
+

.ci/docker-compose-file/docker-compose.yaml

@@ -18,6 +18,9 @@ services:
       - emqx_bridge
     volumes:
       - ../..:/emqx
+      - emqx-shared-secret:/var/lib/secret
+      - ./kerberos/krb5.conf:/etc/kdc/krb5.conf
+      - ./kerberos/krb5.conf:/etc/krb5.conf
     working_dir: /emqx
     tty: true
 
@@ -33,3 +36,6 @@ networks:
           gateway: 172.100.239.1
         - subnet: 2001:3200:3200::/64
           gateway: 2001:3200:3200::1
+
+volumes:   # add this section
+  emqx-shared-secret:    # does not need anything underneath this

.ci/docker-compose-file/kafka/jaas.conf

@@ -6,4 +6,11 @@ KafkaServer {
    org.apache.kafka.common.security.scram.ScramLoginModule required
    username="admin"
    password="password";
+
+   com.sun.security.auth.module.Krb5LoginModule required
+   useKeyTab=true
+   storeKey=true
+   keyTab="/var/lib/secret/kafka.key"
+   principal="kafka/kafka-1.emqx.net@KDC.EMQX.NET";
+
 };

.ci/docker-compose-file/kafka/run_add_scram_users.sh

@@ -2,6 +2,15 @@
 
 set -euo pipefail
 
+
+TIMEOUT=60
+
+echo "+++++++ Wait until Kerberos Keytab is created ++++++++"
+
+timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.key ]; do sleep 1; done'
+
+sleep 3
+
 echo "+++++++ Starting Kafka ++++++++"
 
 start-kafka.sh &

.ci/docker-compose-file/kerberos/krb5.conf

@@ -0,0 +1,23 @@
+[libdefaults]
+  default_realm = KDC.EMQX.NET
+  ticket_lifetime = 24h
+  renew_lifetime = 7d
+  forwardable = true
+  rdns = false
+  dns_lookup_kdc   = no
+  dns_lookup_realm = no
+
+[realms]
+  KDC.EMQX.NET = {
+    kdc = kdc
+    admin_server = kadmin
+  }
+
+[domain_realm]
+  kdc.emqx.net = KDC.EMQX.NET
+  .kdc.emqx.net = KDC.EMQX.NET
+
+[logging]
+  kdc = FILE:/var/log/kerberos/krb5kdc.log
+  admin_server = FILE:/var/log/kerberos/kadmin.log
+  default = FILE:/var/log/kerberos/krb5lib.log

.ci/docker-compose-file/kerberos/run.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+
+echo "Remove old keytabs"
+
+rm -f /var/lib/secret/kafka.key 2>&1 > /dev/null
+rm -f /var/lib/secret/rig.key 2>&1 > /dev/null
+
+echo "Create realm"
+
+kdb5_util -P emqx -r KDC.EMQX.NET create -s
+
+echo "Add principals"
+
+kadmin.local -w password -q "add_principal -randkey kafka/kafka-1.emqx.net@KDC.EMQX.NET"
+kadmin.local -w password -q "add_principal -randkey rig@KDC.EMQX.NET"  > /dev/null
+
+
+echo "Create keytabs"
+
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/kafka.key -norandkey kafka/kafka-1.emqx.net@KDC.EMQX.NET " > /dev/null
+kadmin.local -w password -q "ktadd  -k /var/lib/secret/rig.key -norandkey rig@KDC.EMQX.NET " > /dev/null
+
+echo STARTING KDC
+/usr/sbin/krb5kdc -n

lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl

@@ -94,6 +94,18 @@ t_publish_sasl_scram512(_CtConfig) ->
     }),
     do_publish(Conf, KafkaTopic).
 
+t_publish_sasl_kerberos(_CtConfig) ->
+    KafkaTopic = "test-topic-one-partition",
+    Conf = config(#{
+        "authentication" => #{
+            "kerberos_principal" => "rig@KDC.EMQX.NET",
+            "kerberos_keytab_file" => "/var/lib/secret/rig.key"
+        },
+        "kafka_hosts_string" => kafka_hosts_string_sasl(),
+        "kafka_topic" => KafkaTopic
+    }),
+    do_publish(Conf, KafkaTopic).
+
 config(Args) ->
     {ok, Conf} = hocon:binary(hocon_config(Args)),
     #{config := Parsed} = hocon_tconf:check_plain(
@@ -139,6 +151,13 @@ hocon_config_template_authentication(#{"mechanism" := _}) ->
     password = {{ password }}
     username = {{ username }}
 }
+""";
+hocon_config_template_authentication(#{"kerberos_principal" := _}) ->
+"""
+{
+    kerberos_principal = \"{{ kerberos_principal }}\"
+    kerberos_keytab_file = \"{{ kerberos_keytab_file }}\"
+}
 """.
 
 kafka_hosts_string() ->