From 1051df7af801aacc35e4fad8cd7fe836cc126174 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Feb 2023 14:10:58 -0300 Subject: [PATCH] test: add kafka to toxiproxy --- .../docker-compose-kafka.yaml | 26 +++++++++++-------- .../docker-compose-toxiproxy.yaml | 5 +++- .ci/docker-compose-file/docker-compose.yaml | 5 +--- .ci/docker-compose-file/toxiproxy.json | 24 +++++++++++++++++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 63e74fa11..e54f1377d 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -13,7 +13,7 @@ services: image: fredrikhgrelland/alpine-jdk11-openssl container_name: ssl_cert_gen volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kafka/generate-certs.sh:/bin/generate-certs.sh entrypoint: /bin/sh command: /bin/generate-certs.sh @@ -21,21 +21,24 @@ services: hostname: kdc.emqx.net image: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 container_name: kdc.emqx.net + expose: + - 88 # kdc + - 749 # admin server + # ports: + # - 88:88 + # - 749:749 networks: emqx_bridge: volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf - ./kerberos/run.sh:/usr/bin/run.sh command: run.sh kafka_1: image: wurstmeister/kafka:2.13-2.7.0 - ports: - - "9092:9092" - - "9093:9093" - - "9094:9094" - - "9095:9095" + # ports: + # - "9192-9195:9192-9195" container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: @@ -48,9 +51,9 @@ services: environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093,SSL://:9094,SASL_SSL://:9095 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093,SSL://kafka-1.emqx.net:9094,SASL_SSL://kafka-1.emqx.net:9095 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL + KAFKA_LISTENERS: PLAINTEXT://:9092,SASL_PLAINTEXT://:9093,SSL://:9094,SASL_SSL://:9095,LOCAL_PLAINTEXT://:9192,LOCAL_SASL_PLAINTEXT://:9193,LOCAL_SSL://:9194,LOCAL_SASL_SSL://:9195,TOXIPROXY_PLAINTEXT://:9292,TOXIPROXY_SASL_PLAINTEXT://:9293,TOXIPROXY_SSL://:9294,TOXIPROXY_SASL_SSL://:9295 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1.emqx.net:9092,SASL_PLAINTEXT://kafka-1.emqx.net:9093,SSL://kafka-1.emqx.net:9094,SASL_SSL://kafka-1.emqx.net:9095,LOCAL_PLAINTEXT://localhost:9192,LOCAL_SASL_PLAINTEXT://localhost:9193,LOCAL_SSL://localhost:9194,LOCAL_SASL_SSL://localhost:9195,TOXIPROXY_PLAINTEXT://toxiproxy.emqx.net:9292,TOXIPROXY_SASL_PLAINTEXT://toxiproxy.emqx.net:9293,TOXIPROXY_SSL://toxiproxy.emqx.net:9294,TOXIPROXY_SASL_SSL://toxiproxy.emqx.net:9295 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,LOCAL_PLAINTEXT:PLAINTEXT,LOCAL_SASL_PLAINTEXT:SASL_PLAINTEXT,LOCAL_SSL:SSL,LOCAL_SASL_SSL:SASL_SSL,TOXIPROXY_PLAINTEXT:PLAINTEXT,TOXIPROXY_SASL_PLAINTEXT:SASL_PLAINTEXT,TOXIPROXY_SSL:SSL,TOXIPROXY_SASL_SSL:SASL_SSL KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka @@ -58,6 +61,7 @@ services: KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer KAFKA_SSL_TRUSTSTORE_LOCATION: /var/lib/secret/kafka.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD: password @@ -67,7 +71,7 @@ services: networks: emqx_bridge: volumes: - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kafka/jaas.conf:/etc/kafka/jaas.conf - ./kafka/kafka-entrypoint.sh:/bin/kafka-entrypoint.sh - ./kerberos/krb5.conf:/etc/kdc/krb5.conf diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 3dd30af52..16f18b6c2 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -6,7 +6,10 @@ services: image: ghcr.io/shopify/toxiproxy:2.5.0 restart: always networks: - - emqx_bridge + emqx_bridge: + aliases: + - toxiproxy + - toxiproxy.emqx.net volumes: - "./toxiproxy.json:/config/toxiproxy.json" ports: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index ff330872d..42003fcb7 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -18,7 +18,7 @@ services: - emqx_bridge volumes: - ../..:/emqx - - emqx-shared-secret:/var/lib/secret + - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf working_dir: /emqx @@ -37,6 +37,3 @@ networks: gateway: 172.100.239.1 - subnet: 2001:3200:3200::/64 gateway: 2001:3200:3200::1 - -volumes: # add this section - emqx-shared-secret: # does not need anything underneath this diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 6188eab17..2f8c4341b 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -53,5 +53,29 @@ "listen": "0.0.0.0:8000", "upstream": "dynamo:8000", "enabled": true + }, + { + "name": "kafka_plain", + "listen": "0.0.0.0:9292", + "upstream": "kafka-1.emqx.net:9292", + "enabled": true + }, + { + "name": "kafka_sasl_plain", + "listen": "0.0.0.0:9293", + "upstream": "kafka-1.emqx.net:9293", + "enabled": true + }, + { + "name": "kafka_ssl", + "listen": "0.0.0.0:9294", + "upstream": "kafka-1.emqx.net:9294", + "enabled": true + }, + { + "name": "kafka_sasl_ssl", + "listen": "0.0.0.0:9295", + "upstream": "kafka-1.emqx.net:9295", + "enabled": true } ]