From 0cba1b2ed0a10c7b1b2f18ecd4c0bd7307a24917 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 22 Nov 2022 15:11:43 +0100 Subject: [PATCH] ci: run tests as non-root in erlang container --- .../docker-compose-kafka.yaml | 13 ++-- .ci/docker-compose-file/docker-compose.yaml | 2 +- .../kafka/generate-certs.sh | 7 ++- .../kafka/run_add_scram_users.sh | 3 +- .../kafka/ssl_cert_gen.Dockerfile | 9 +++ .github/workflows/run_test_cases.yaml | 61 +++++++++++-------- build | 3 - .../emqx_bridge_impl_kafka_producer_SUITE.erl | 3 + lib-ee/emqx_license/etc/emqx_license.conf | 0 scripts/ct/run.sh | 37 ++++------- scripts/find-apps.sh | 2 +- 11 files changed, 76 insertions(+), 64 deletions(-) create mode 100644 .ci/docker-compose-file/kafka/ssl_cert_gen.Dockerfile create mode 100644 lib-ee/emqx_license/etc/emqx_license.conf diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index ba0161293..65eb51b3b 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -10,7 +10,9 @@ services: networks: emqx_bridge: ssl_cert_gen: - image: fredrikhgrelland/alpine-jdk11-openssl + build: + context: ./kafka + dockerfile: ssl_cert_gen.Dockerfile container_name: ssl_cert_gen volumes: - emqx-shared-secret:/var/lib/secret @@ -39,9 +41,12 @@ services: container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: - - "kdc" - - "zookeeper" - - "ssl_cert_gen" + kdc: + condition: service_started + zookeeper: + condition: service_started + ssl_cert_gen: + condition: service_started environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 4a5ef7070..72f1d52a1 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -23,7 +23,7 @@ services: - ./kerberos/krb5.conf:/etc/krb5.conf working_dir: /emqx tty: true - user: "${UID_GID}" + user: "${DOCKER_USER:-root}" networks: emqx_bridge: diff --git a/.ci/docker-compose-file/kafka/generate-certs.sh b/.ci/docker-compose-file/kafka/generate-certs.sh index 3f1c75550..e5e2eaf32 100755 --- a/.ci/docker-compose-file/kafka/generate-certs.sh +++ b/.ci/docker-compose-file/kafka/generate-certs.sh @@ -1,4 +1,4 @@ -#!/usr/bin/bash +#!/usr/bin/env bash set -euo pipefail @@ -10,10 +10,10 @@ HOST="*." DAYS=3650 PASS="password" -cd /var/lib/secret/ +cd /var/lib/secret # Delete old files -(rm ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks 2>/dev/null || true) +rm -f ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks ls @@ -44,3 +44,4 @@ echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeysto echo '= Import CA into java truststore' echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass "$PASS" +chmod g+r /var/lib/secret/* diff --git a/.ci/docker-compose-file/kafka/run_add_scram_users.sh b/.ci/docker-compose-file/kafka/run_add_scram_users.sh index 4b51fee0d..2584168d7 100755 --- a/.ci/docker-compose-file/kafka/run_add_scram_users.sh +++ b/.ci/docker-compose-file/kafka/run_add_scram_users.sh @@ -2,7 +2,6 @@ set -euo pipefail - TIMEOUT=60 echo "+++++++ Sleep for a while to make sure that old keytab and truststore is deleted ++++++++" @@ -17,7 +16,7 @@ timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.keytab ]; do sleep 1; echo "+++++++ Wait until SSL certs are generated ++++++++" timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.truststore.jks ]; do sleep 1; done' - +keytool -list -v -keystore /var/lib/secret/kafka.keystore.jks -storepass password sleep 3 echo "+++++++ Starting Kafka ++++++++" diff --git a/.ci/docker-compose-file/kafka/ssl_cert_gen.Dockerfile b/.ci/docker-compose-file/kafka/ssl_cert_gen.Dockerfile new file mode 100644 index 000000000..9efc016ab --- /dev/null +++ b/.ci/docker-compose-file/kafka/ssl_cert_gen.Dockerfile @@ -0,0 +1,9 @@ +# jdk version must be <= jdk in kafka container for java keystore to work +FROM azul/zulu-openjdk-alpine:8-jre-headless + +ARG TEST_DOWNLOAD_BUILD_ARGUMENT=https://nrk.no + +RUN apk --update --no-cache add curl=~7 ca-certificates=20220614-r0 openssl=1.1.1s-r1 +RUN find /usr/local/share/ca-certificates -not -name "*.crt" -type f -delete +RUN update-ca-certificates 2>/dev/null || true && echo "NOTE: CA warnings suppressed." +RUN curl -s -L -o /dev/null ${TEST_DOWNLOAD_BUILD_ARGUMENT} || { printf "\n###############\nERROR: You are probably behind a corporate proxy. Add your custom ca .crt in the conf/certificates docker build folder\n###############\n"; exit 1; } diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 07ae00a50..b727f750a 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -18,10 +18,13 @@ jobs: runs-on: aws-amd64 # prepare source with any OTP version, no need for a matrix container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" + env: + PROFILE: emqx outputs: fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }} steps: + - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/checkout@v3 with: path: source @@ -33,46 +36,41 @@ jobs: docker_ct_apps="$(./scripts/find-apps.sh --ci docker)" echo "fast: $fast_ct_apps" echo "docker: $docker_ct_apps" - echo "::set-output name=fast_ct_apps::$fast_ct_apps" - echo "::set-output name=docker_ct_apps::$docker_ct_apps" - - name: get_all_deps + echo "fast_ct_apps=${fast_ct_apps}" >> $GITHUB_OUTPUT + echo "docker_ct_apps=${docker_ct_apps}" >> $GITHUB_OUTPUT + - name: Get deps and compile working-directory: source env: PROFILE: emqx - #DIAGNOSTIC: 1 run: | - make ensure-rebar3 - # this will fetch all deps and compile - ./rebar3 as test compile + make cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: - name: source-emqx + name: source-${{ env.PROFILE }} path: source.zip prepare_ee: runs-on: aws-amd64 # prepare source with any OTP version, no need for a matrix container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" + env: + PROFILE: emqx-enterprise steps: + - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/checkout@v3 with: path: source - - name: get_all_deps + - name: Get deps and compile working-directory: source - env: - PROFILE: emqx-enterprise - #DIAGNOSTIC: 1 run: | - make ensure-rebar3 - # this will fetch all deps and compile - ./rebar3 as test compile + make cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: - name: source-emqx-enterprise + name: source-${{ env.PROFILE }} path: source.zip eunit_and_proper: @@ -92,7 +90,7 @@ jobs: container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - - uses: AutoModality/action-clean@v1 + - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/download-artifact@v3 with: name: source-${{ matrix.profile }} @@ -128,8 +126,6 @@ jobs: fail-fast: false matrix: app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} - otp_release: - - "erlang24" runs-on: aws-amd64 defaults: @@ -137,7 +133,9 @@ jobs: shell: bash steps: - - uses: AutoModality/action-clean@v1 + - uses: emqx/self-hosted-cleanup-action@v1.0.3 + with: + cleanup_docker: true - uses: actions/download-artifact@v3 with: name: source-${{ matrix.app[1] }} @@ -156,16 +154,22 @@ jobs: PROFILE: ${{ matrix.app[1] }} run: | echo $PROFILE - rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt ./scripts/ct/run.sh --app $WHICH_APP - uses: actions/upload-artifact@v3 + if: success() with: name: coverdata path: source/_build/test/cover + - id: logs_artifact_name + if: failure() + run: | + PRE_NAME="logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}" + ARTIFACT_NAME=$(echo "$PRE_NAME" | sed 's/\//-/g') + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@v3 if: failure() with: - name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }} + name: ${{ steps.logs_artifact_name.outputs.artifact_name }} path: source/_build/test/logs ct: @@ -183,7 +187,7 @@ jobs: shell: bash steps: - - uses: AutoModality/action-clean@v1 + - uses: emqx/self-hosted-cleanup-action@v1.0.3 - uses: actions/download-artifact@v3 with: name: source-${{ matrix.app[1] }} @@ -200,14 +204,21 @@ jobs: run: | make "${WHICH_APP}-ct" - uses: actions/upload-artifact@v3 + if: success() with: name: coverdata path: source/_build/test/cover if-no-files-found: warn # do not fail if no coverdata found + - id: logs_artifact_name + if: failure() + run: | + PRE_NAME="logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}" + ARTIFACT_NAME=$(echo "$PRE_NAME" | sed 's/\//-/g') + echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT - uses: actions/upload-artifact@v3 if: failure() with: - name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }} + name: ${{ steps.logs_artifact_name.outputs.artifact_name }} path: source/_build/test/logs make_cover: @@ -218,7 +229,7 @@ jobs: runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - - uses: AutoModality/action-clean@v1 + - uses: AutoModality/action-clean@v1.1.0 - uses: actions/download-artifact@v3 with: name: source-emqx-enterprise diff --git a/build b/build index 7e5f65b59..ba4315764 100755 --- a/build +++ b/build @@ -4,9 +4,6 @@ # arg1: profile, e.g. emqx | emqx-pkg # arg2: artifact, e.g. rel | relup | tgz | pkg -if [[ -n "$DEBUG" ]]; then - set -x -fi set -euo pipefail DEBUG="${DEBUG:-0}" diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 2eef1170d..b7f0de226 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -71,6 +71,9 @@ init_per_suite(Config) -> [emqx_conf, emqx_rule_engine, emqx_bridge, emqx_management, emqx_dashboard], fun set_special_configs/1 ), + %% disable postgres backend so that it does not spam in the logs + application:load(system_monitor), + application:set_env(system_monitor, db_hostname, undefined), application:set_env(emqx_machine, applications, [ emqx_prometheus, emqx_modules, diff --git a/lib-ee/emqx_license/etc/emqx_license.conf b/lib-ee/emqx_license/etc/emqx_license.conf new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index edfaf6a9e..70b0e63cc 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -83,7 +83,7 @@ if [ -f "$DOCKER_CT_ENVS_FILE" ]; then # shellcheck disable=SC2002 CT_DEPS="$(cat "$DOCKER_CT_ENVS_FILE" | xargs)" fi -CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS}" +CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS:-}" FILES=( ) @@ -120,9 +120,6 @@ for dep in ${CT_DEPS}; do '.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' ) ;; kafka) - # Kafka container generates root owned ssl files - # the files are shared with EMQX (with a docker volume) - NEED_ROOT=yes FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' ) ;; *) @@ -138,44 +135,34 @@ for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done -if [[ "${NEED_ROOT:-}" == 'yes' ]]; then - export UID_GID='root:root' -else - # Passing $UID to docker-compose to be used in erlang container - # as owner of the main process to avoid git repo permissions issue. - # Permissions issue happens because we are mounting local filesystem - # where files are owned by $UID to docker container where it's using - # root (UID=0) by default, and git is not happy about it. - export UID_GID="$UID:$UID" -fi - +DOCKER_USER="$(id -u):root" +export DOCKER_USER # shellcheck disable=2086 # no quotes for F_OPTIONS docker-compose $F_OPTIONS up -d --build -# /emqx is where the source dir is mounted to the Erlang container -# in .ci/docker-compose-file/docker-compose.yaml TTY='' if [[ -t 1 ]]; then TTY='-t' fi -echo "Fixing file owners and permissions for $UID_GID" -# rebar and hex cache directory need to be writable by $UID -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $UID_GID /.cache && chown -R $UID_GID /emqx" -# need to initialize .erlang.cookie manually here because / is not writable by $UID -docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie" +echo "Fixing file owners and permissions in $ERLANG_CONTAINER" +# rebar and hex cache directory need to be writable by $DOCKER_USER +docker exec -i $TTY -u root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $DOCKER_USER /.cache" +# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER +docker exec -i $TTY -u root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie" if [ "$ONLY_UP" = 'yes' ]; then exit 0 fi if [ "$ATTACH" = 'yes' ]; then - docker exec -it "$ERLANG_CONTAINER" bash + docker exec -u "$DOCKER_USER" -it "$ERLANG_CONTAINER" bash elif [ "$CONSOLE" = 'yes' ]; then - docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run" + docker exec -u "$DOCKER_USER" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run" else set +e - docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" + docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "rm -f _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt" + docker exec -u "$DOCKER_USER" -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct" RESULT=$? if [ "$KEEP_UP" = 'yes' ]; then exit $RESULT diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index fefb73568..d49e9829a 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -8,7 +8,7 @@ cd -P -- "$(dirname -- "$0")/.." help() { echo echo "-h|--help: To display this usage info" - echo "--ci fast|docker: Print apps in json format for github ci mtrix" + echo "--ci fast|docker: Print apps in json format for github ci matrix" } CI='novalue'