diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 2612eb8d8..3d99d3969 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -20,6 +20,7 @@ services: - ../..:/emqx working_dir: /emqx tty: true + user: "${UID_GID}" networks: emqx_bridge: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 803b50cfc..3372fefc9 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -20,7 +20,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} @@ -112,7 +112,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest @@ -164,7 +164,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -189,7 +189,7 @@ jobs: os: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest @@ -232,7 +232,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -257,7 +257,7 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: only support latest otp version, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest registry: - 'docker.io' - 'public.ecr.aws' @@ -319,7 +319,7 @@ jobs: - ${{ needs.prepare.outputs.BUILD_PROFILE }} # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.2.1-1 # update to latest + - 24.3.4.2-1 # update to latest elixir: - 1.13.4 # update to latest registry: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e4b435a3e..36ec90283 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -140,7 +140,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 + - 24.3.4.2-1 os: - macos-11 runs-on: ${{ matrix.os }} @@ -175,7 +175,7 @@ jobs: needs: prepare runs-on: ${{ matrix.build_machine }} container: - image: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + image: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false @@ -183,7 +183,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 # we test with OTP 23, but only build package on OTP 24 versions + - 24.3.4.2-1 # we test with OTP 23, but only build package on OTP 24 versions elixir: - 1.13.4 # used to split elixir packages into a separate job, since the @@ -232,14 +232,14 @@ jobs: profile: emqx-enterprise include: - profile: emqx - otp: 24.2.1-1 + otp: 24.3.4.2-1 elixir: 1.13.4 build_elixir: with_elixir arch: amd64 os: ubuntu20.04 build_machine: ubuntu-20.04 - profile: emqx - otp: 24.2.1-1 + otp: 24.3.4.2-1 elixir: 1.13.4 build_elixir: with_elixir arch: amd64 @@ -290,7 +290,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IsElixir}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-17:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/5.0-18:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v3 with: @@ -307,7 +307,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.2.1-1 + - 24.3.4.2-1 include: - profile: emqx otp: windows # otp version on windows is rather fixed @@ -320,7 +320,7 @@ jobs: run: sudo apt-get update && sudo apt install -y dos2unix - name: get packages run: | - DEFAULT_BEAM_PLATFORM='otp24.2.1-1' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1' set -e -u cd packages/${{ matrix.profile }} # Make a copy of the default OTP version package to a file without OTP version infix diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 8f2e43f45..0e3e3d036 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -32,14 +32,14 @@ jobs: - emqx - emqx-enterprise otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 os: - ubuntu20.04 - el8 - container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: AutoModality/action-clean@v1 @@ -132,7 +132,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.2.1-1 + - 24.3.4.2-1 os: - macos-11 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index fdf8903d4..c5c509f0c 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 1417960ed..bc15d696f 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 2ec54ce63..8dc9e54cd 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -8,7 +8,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 2cecade04..210eda570 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 36ba5496d..cef1095a2 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -12,7 +12,7 @@ on: jobs: elixir_release_build: runs-on: ubuntu-latest - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5569aa300..3e7dd4ada 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -32,7 +32,7 @@ jobs: esac aws s3 cp --recursive s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages cd packages - DEFAULT_BEAM_PLATFORM='otp24.2.1-1' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1' # all packages including full-name and default-name are uploaded to s3 # but we only upload default-name packages (and elixir) as github artifacts # so we rename (overwrite) non-default packages before uploading diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index c05fa7a3a..5b68e3dda 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: otp: - - 24.2.1-1 + - 24.3.4.2-1 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify # the base image. @@ -24,7 +24,7 @@ jobs: - amd64 runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 6a6466442..0464b5e50 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-alpine3.15.1 + container: ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-alpine3.15.1 steps: - uses: actions/checkout@v3 @@ -49,7 +49,7 @@ jobs: os: - ["alpine3.15.1", "alpine:3.15.1"] otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 arch: @@ -68,7 +68,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker @@ -120,7 +120,7 @@ jobs: os: - ["debian11", "debian:11-slim"] otp: - - 24.2.1-1 + - 24.3.4.2-1 elixir: - 1.13.4 arch: @@ -141,7 +141,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-17:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-18:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 6f88c78e4..ea0dff9e9 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -16,7 +16,7 @@ on: jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 53939aaf3..03030908c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} @@ -60,7 +60,7 @@ jobs: defaults: run: shell: bash - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 @@ -121,6 +121,7 @@ jobs: PGSQL_TAG: 13 REDIS_TAG: 6 run: | + rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt ./scripts/ct/run.sh --app ${{ matrix.app_name }} - uses: actions/upload-artifact@v3 with: @@ -143,7 +144,7 @@ jobs: - emqx-enterprise runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" defaults: run: shell: bash @@ -200,7 +201,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 879efc434..aa6f55022 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -89,8 +89,13 @@ for file in "${FILES[@]}"; do F_OPTIONS="$F_OPTIONS -f $file" done +# Passing $UID to docker-compose to be used in erlang container +# as owner of the main process to avoid git repo permissions issue. +# Permissions issue happens because we are mounting local filesystem +# where files are owned by $UID to docker container where it's using +# root (UID=0) by default, and git is not happy about it. # shellcheck disable=2086 # no quotes for F_OPTIONS -docker-compose $F_OPTIONS up -d --build +UID_GID="$UID:$UID" docker-compose $F_OPTIONS up -d --build # /emqx is where the source dir is mounted to the Erlang container # in .ci/docker-compose-file/docker-compose.yaml @@ -98,8 +103,11 @@ TTY='' if [[ -t 1 ]]; then TTY='-t' fi -docker exec -i $TTY "$ERLANG_CONTAINER" bash -c 'git config --global --add safe.directory /emqx' +# rebar and hex cache directory need to be writable by $UID +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir /.cache && chown $UID:$UID /.cache" +# need to initialize .erlang.cookie manually here because / is not writable by $UID +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 16 > /.erlang.cookie && chown $UID:$UID /.erlang.cookie && chmod 0400 /.erlang.cookie" if [ "$CONSOLE" = 'yes' ]; then docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make run" else @@ -107,6 +115,6 @@ else docker exec -i $TTY "$ERLANG_CONTAINER" bash -c "make ${WHICH_APP}-ct" RESULT=$? # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down + UID_GID="$UID:$UID" docker-compose $F_OPTIONS down exit $RESULT fi