yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: run tests as non-root in erlang container

This commit is contained in:
Ivan Dyachkov 2022-11-22 15:11:43 +01:00
parent c559334e69
commit 0cba1b2ed0
11 changed files with 76 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.ci/docker-compose-file/docker-compose-kafka.yaml
View File

@ -10,7 +10,9 @@ services:
networks:
emqx_bridge:
ssl_cert_gen:
image: fredrikhgrelland/alpine-jdk11-openssl
build:
context: ./kafka
dockerfile: ssl_cert_gen.Dockerfile
container_name: ssl_cert_gen
volumes:
- emqx-shared-secret:/var/lib/secret
@ -39,9 +41,12 @@ services:
container_name: kafka-1.emqx.net
hostname: kafka-1.emqx.net
depends_on:
- "kdc"
- "zookeeper"
- "ssl_cert_gen"
kdc:
condition: service_started
zookeeper:
condition: service_started
ssl_cert_gen:
condition: service_started
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

2
.ci/docker-compose-file/docker-compose.yaml
View File

@ -23,7 +23,7 @@ services:
- ./kerberos/krb5.conf:/etc/krb5.conf
working_dir: /emqx
tty: true
user: "${UID_GID}"
user: "${DOCKER_USER:-root}"
networks:
emqx_bridge:

7
.ci/docker-compose-file/kafka/generate-certs.sh
View File

@ -1,4 +1,4 @@
#!/usr/bin/bash
#!/usr/bin/env bash
set -euo pipefail
@ -10,10 +10,10 @@ HOST="*."
DAYS=3650
PASS="password"
cd /var/lib/secret/
cd /var/lib/secret
# Delete old files
(rm ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks 2>/dev/null || true)
rm -f ca.key ca.crt server.key server.csr server.crt client.key client.csr client.crt server.p12 kafka.keystore.jks kafka.truststore.jks
ls
@ -44,3 +44,4 @@ echo $PASS | keytool -importkeystore -destkeystore kafka.keystore.jks -srckeysto
echo '= Import CA into java truststore'
echo yes | keytool -keystore kafka.truststore.jks -alias CARoot -import -file ca.crt -storepass "$PASS"
chmod g+r /var/lib/secret/*

3
.ci/docker-compose-file/kafka/run_add_scram_users.sh
View File

@ -2,7 +2,6 @@
set -euo pipefail
TIMEOUT=60
echo "+++++++ Sleep for a while to make sure that old keytab and truststore is deleted ++++++++"
@ -17,7 +16,7 @@ timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.keytab ]; do sleep 1;
echo "+++++++ Wait until SSL certs are generated ++++++++"
timeout $TIMEOUT bash -c 'until [ -f /var/lib/secret/kafka.truststore.jks ]; do sleep 1; done'
keytool -list -v -keystore /var/lib/secret/kafka.keystore.jks -storepass password
sleep 3
echo "+++++++ Starting Kafka ++++++++"

9
.ci/docker-compose-file/kafka/ssl_cert_gen.Dockerfile Normal file
View File

@ -0,0 +1,9 @@
# jdk version must be <= jdk in kafka container for java keystore to work
FROM azul/zulu-openjdk-alpine:8-jre-headless
ARG TEST_DOWNLOAD_BUILD_ARGUMENT=https://nrk.no
RUN apk --update --no-cache add curl=~7 ca-certificates=20220614-r0 openssl=1.1.1s-r1
RUN find /usr/local/share/ca-certificates -not -name "*.crt" -type f -delete
RUN update-ca-certificates 2>/dev/null || true && echo "NOTE: CA warnings suppressed."
RUN curl -s -L -o /dev/null ${TEST_DOWNLOAD_BUILD_ARGUMENT} || { printf "\n###############\nERROR: You are probably behind a corporate proxy. Add your custom ca .crt in the conf/certificates docker build folder\n###############\n"; exit 1; }

61
.github/workflows/run_test_cases.yaml vendored
View File

@ -18,10 +18,13 @@ jobs:
runs-on: aws-amd64
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
env:
PROFILE: emqx
outputs:
fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }}
docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }}
steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3
with:
path: source
@ -33,46 +36,41 @@ jobs:
docker_ct_apps="$(./scripts/find-apps.sh --ci docker)"
echo "fast: $fast_ct_apps"
echo "docker: $docker_ct_apps"
echo "::set-output name=fast_ct_apps::$fast_ct_apps"
echo "::set-output name=docker_ct_apps::$docker_ct_apps"
- name: get_all_deps
echo "fast_ct_apps=${fast_ct_apps}" >> $GITHUB_OUTPUT
echo "docker_ct_apps=${docker_ct_apps}" >> $GITHUB_OUTPUT
- name: Get deps and compile
working-directory: source
env:
PROFILE: emqx
#DIAGNOSTIC: 1
run: |
make ensure-rebar3
# this will fetch all deps and compile
./rebar3 as test compile
make
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-emqx
name: source-${{ env.PROFILE }}
path: source.zip
prepare_ee:
runs-on: aws-amd64
# prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
env:
PROFILE: emqx-enterprise
steps:
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/checkout@v3
with:
path: source
- name: get_all_deps
- name: Get deps and compile
working-directory: source
env:
PROFILE: emqx-enterprise
#DIAGNOSTIC: 1
run: |
make ensure-rebar3
# this will fetch all deps and compile
./rebar3 as test compile
make
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source-emqx-enterprise
name: source-${{ env.PROFILE }}
path: source.zip
eunit_and_proper:
@ -92,7 +90,7 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.profile }}
@ -128,8 +126,6 @@ jobs:
fail-fast: false
matrix:
app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
otp_release:
- "erlang24"
runs-on: aws-amd64
defaults:
@ -137,7 +133,9 @@ jobs:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: emqx/self-hosted-cleanup-action@v1.0.3
with:
cleanup_docker: true
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.app[1] }}
@ -156,16 +154,22 @@ jobs:
PROFILE: ${{ matrix.app[1] }}
run: |
echo $PROFILE
rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt
./scripts/ct/run.sh --app $WHICH_APP
- uses: actions/upload-artifact@v3
if: success()
with:
name: coverdata
path: source/_build/test/cover
- id: logs_artifact_name
if: failure()
run: |
PRE_NAME="logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}"
ARTIFACT_NAME=$(echo "$PRE_NAME" | sed 's/\//-/g')
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
name: ${{ steps.logs_artifact_name.outputs.artifact_name }}
path: source/_build/test/logs
ct:
@ -183,7 +187,7 @@ jobs:
shell: bash
steps:
- uses: AutoModality/action-clean@v1
- uses: emqx/self-hosted-cleanup-action@v1.0.3
- uses: actions/download-artifact@v3
with:
name: source-${{ matrix.app[1] }}
@ -200,14 +204,21 @@ jobs:
run: |
make "${WHICH_APP}-ct"
- uses: actions/upload-artifact@v3
if: success()
with:
name: coverdata
path: source/_build/test/cover
if-no-files-found: warn # do not fail if no coverdata found
- id: logs_artifact_name
if: failure()
run: |
PRE_NAME="logs-${{ matrix.app[0] }}-${{ matrix.app[1] }}"
ARTIFACT_NAME=$(echo "$PRE_NAME" | sed 's/\//-/g')
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v3
if: failure()
with:
name: logs_${{ matrix.otp_release }}-${{ matrix.app[0] }}-${{ matrix.app[1] }}
name: ${{ steps.logs_artifact_name.outputs.artifact_name }}
path: source/_build/test/logs
make_cover:
@ -218,7 +229,7 @@ jobs:
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04"
steps:
- uses: AutoModality/action-clean@v1
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/download-artifact@v3
with:
name: source-emqx-enterprise

3
build
View File

@ -4,9 +4,6 @@
# arg1: profile, e.g. emqx | emqx-pkg
# arg2: artifact, e.g. rel | relup | tgz | pkg
if [[ -n "$DEBUG" ]]; then
set -x
fi
set -euo pipefail
DEBUG="${DEBUG:-0}"

3
lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
View File

@ -71,6 +71,9 @@ init_per_suite(Config) ->
[emqx_conf, emqx_rule_engine, emqx_bridge, emqx_management, emqx_dashboard],
fun set_special_configs/1
),
%% disable postgres backend so that it does not spam in the logs
application:load(system_monitor),
application:set_env(system_monitor, db_hostname, undefined),
application:set_env(emqx_machine, applications, [
emqx_prometheus,
emqx_modules,

0
lib-ee/emqx_license/etc/emqx_license.conf Normal file
View File

37
scripts/ct/run.sh
View File

@ -83,7 +83,7 @@ if [ -f "$DOCKER_CT_ENVS_FILE" ]; then
# shellcheck disable=SC2002
CT_DEPS="$(cat "$DOCKER_CT_ENVS_FILE" | xargs)"
fi
CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS}"
CT_DEPS="${ERLANG_CONTAINER} ${CT_DEPS:-}"
FILES=( )
@ -120,9 +120,6 @@ for dep in ${CT_DEPS}; do
'.ci/docker-compose-file/docker-compose-pgsql-tls.yaml' )
;;
kafka)
# Kafka container generates root owned ssl files
# the files are shared with EMQX (with a docker volume)
NEED_ROOT=yes
FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' )
;;
*)
@ -138,44 +135,34 @@ for file in "${FILES[@]}"; do
F_OPTIONS="$F_OPTIONS -f $file"
done
if [[ "${NEED_ROOT:-}" == 'yes' ]]; then
export UID_GID='root:root'
else
# Passing $UID to docker-compose to be used in erlang container
# as owner of the main process to avoid git repo permissions issue.
# Permissions issue happens because we are mounting local filesystem
# where files are owned by $UID to docker container where it's using
# root (UID=0) by default, and git is not happy about it.
export UID_GID="$UID:$UID"
fi
DOCKER_USER="$(id -u):root"
export DOCKER_USER
# shellcheck disable=2086 # no quotes for F_OPTIONS
docker-compose $F_OPTIONS up -d --build
# /emqx is where the source dir is mounted to the Erlang container
# in .ci/docker-compose-file/docker-compose.yaml
TTY=''
if [[ -t 1 ]]; then
TTY='-t'
fi
echo "Fixing file owners and permissions for $UID_GID"
# rebar and hex cache directory need to be writable by $UID
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $UID_GID /.cache && chown -R $UID_GID /emqx"
# need to initialize .erlang.cookie manually here because / is not writable by $UID
docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID_GID /.erlang.cookie && chmod 0400 /.erlang.cookie"
echo "Fixing file owners and permissions in $ERLANG_CONTAINER"
# rebar and hex cache directory need to be writable by $DOCKER_USER
docker exec -i $TTY -u root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache && chown $DOCKER_USER /.cache"
# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER
docker exec -i $TTY -u root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie"
if [ "$ONLY_UP" = 'yes' ]; then
exit 0
fi
if [ "$ATTACH" = 'yes' ]; then
docker exec -it "$ERLANG_CONTAINER" bash
docker exec -u "$DOCKER_USER" -it "$ERLANG_CONTAINER" bash
elif [ "$CONSOLE" = 'yes' ]; then
docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
docker exec -u "$DOCKER_USER" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
else
set +e
docker exec -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "rm -f _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt"
docker exec -u "$DOCKER_USER" -e PROFILE="$PROFILE" -i $TTY -e EMQX_CT_SUITES="$SUITES" "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
RESULT=$?
if [ "$KEEP_UP" = 'yes' ]; then
exit $RESULT

2
scripts/find-apps.sh
View File

@ -8,7 +8,7 @@ cd -P -- "$(dirname -- "$0")/.."
help() {
echo
echo "-h|--help: To display this usage info"
echo "--ci fast|docker: Print apps in json format for github ci mtrix"
echo "--ci fast|docker: Print apps in json format for github ci matrix"
}
CI='novalue'