From 74c33cd4e5fdac1a73309ff34960944212eabd55 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 23 Mar 2022 15:16:57 +0100 Subject: [PATCH] feat(rule_engine): add jq function to the rule engine This commit adds a function to the rule engine that alows users to transform text or JSON objects using [jq filter programs][1]. [jq][1] is a command line tool that can be used to transform and filter JSON text using jq's built-in language. The rule engine function that is added with this commit uses the [Erlang jq NIF library][2] that wraps the jq C library in an Erlang NIF function. [1]: https://stedolan.github.io/jq/ [2]: https://github.com/emqx/jq --- .ci/docker-compose-file/docker-compose.yaml | 4 +- .github/workflows/apps_version_check.yaml | 4 +- .github/workflows/build_packages.yaml | 17 +- .github/workflows/build_slim_packages.yaml | 18 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 5 +- .github/workflows/elixir_apps_check.yaml | 11 +- .github/workflows/elixir_deps_check.yaml | 7 +- .github/workflows/elixir_release.yml | 7 +- .github/workflows/run_emqx_app_tests.yaml | 11 +- .github/workflows/run_fvt_tests.yaml | 6 +- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 10 +- Makefile | 2 +- apps/emqx_rule_engine/rebar.config | 4 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 20 +- .../test/emqx_rule_engine_SUITE.erl | 38 ++ .../test/emqx_rule_funcs_SUITE.erl | 18 + deploy/docker/Dockerfile | 29 +- elvis.config | 6 + mix.exs | 13 +- pkg-vsn.sh | 2 + rebar.config.erl | 637 ++++++++++-------- scripts/buildx.sh | 11 +- 24 files changed, 528 insertions(+), 356 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index baf636d27..a19acaca8 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang23: container_name: erlang23 - image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-23.3.4.9-4-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-23.3.4.9-4-ubuntu20.04 env_file: - conf.env environment: @@ -23,7 +23,7 @@ services: erlang24: container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 4975e5c11..94d28dd87 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Check apps version run: ./scripts/apps-version-check.sh diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4a5458fd4..37549501c 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -22,7 +22,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04" outputs: BUILD_PROFILES: ${{ steps.get_profiles.outputs.BUILD_PROFILES }} @@ -139,7 +139,8 @@ jobs: - name: prepare run: | brew update - brew install curl zip unzip gnu-sed kerl unixodbc freetds + brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison + echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store - uses: actions/cache@v2 @@ -205,7 +206,7 @@ jobs: needs: prepare runs-on: ${{ matrix.build_machine }} container: - image: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + image: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false @@ -234,8 +235,8 @@ jobs: - debian11 - debian10 - debian9 - - rockylinux8 - - centos7 + - el8 + - el7 - raspbian10 build_machine: - aws-arm64 @@ -272,7 +273,7 @@ jobs: elixir: 1.13.3 build_elixir: with_elixir arch: amd64 - os: centos7 + os: el7 build_machine: ubuntu-20.04 defaults: @@ -325,7 +326,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IsElixir}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-10:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/5.0-14:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') @@ -462,7 +463,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} RUN_FROM=${{ steps.pre-meta.outputs.img }} EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }} file: source/deploy/docker/Dockerfile diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index df1d41614..f3ed58a1b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -38,15 +38,15 @@ jobs: - 1.13.3 os: - ubuntu20.04 - - rockylinux8 + - el8 - container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - - name: cleanup - run: | - rm -rf "${GITHUB_WORKSPACE}/" - - uses: actions/checkout@v1 + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: prepare run: | echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV @@ -63,6 +63,9 @@ jobs: _build/default/lib/quicer/ deps/quicer/ key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: build and test tgz package run: | make ${EMQX_NAME}-tgz @@ -148,7 +151,8 @@ jobs: - name: prepare run: | brew update - brew install curl zip unzip gnu-sed kerl unixodbc freetds + brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison + echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - uses: actions/cache@v2 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index dbf4b5923..195d78047 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 323e4cb64..c52dd77cc 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04" steps: - uses: actions/checkout@v2 with: @@ -13,6 +13,9 @@ jobs: - name: Check line-break at EOF run: | ./scripts/check-nl-at-eof.sh + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Check Elixir code formatting run: | mix format --check-formatted diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 2f40a1f81..a63966765 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -6,9 +6,9 @@ on: [pull_request, push] jobs: elixir_apps_check: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04" strategy: fail-fast: false @@ -31,12 +31,17 @@ jobs: edition_type: enterprise steps: + - name: fix_git_permission + run: git config --global --add safe.directory '/__w/emqx/emqx' - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: ensure rebar run: ./scripts/ensure-rebar3.sh + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: check applications run: ./scripts/check-elixir-applications.exs - name: check applications started with emqx_machine diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 5cc9c25d2..2f843930e 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,13 +7,16 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04 steps: - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v2 - name: ensure rebar run: ./scripts/ensure-rebar3.sh + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: setup mix run: | mix local.hex --force diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 3ce943633..6fb0d8355 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -12,13 +12,16 @@ on: jobs: elixir_release_build: runs-on: ubuntu-latest - container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04 steps: - name: Checkout - uses: actions/checkout@v2.4.0 + uses: actions/checkout@v2 - name: install tools run: apt update && apt install netcat-openbsd + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: elixir release run: make emqx-elixir - name: start release diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 6c80a25bd..fd598eea3 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -25,7 +25,7 @@ jobs: - amd64 runs-on: aws-amd64 - container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: @@ -47,14 +47,19 @@ jobs: key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - name: run run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" echo "git diff base: $GITHUB_BASE_REF" if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then # base is a commit sha1 compare_base="$GITHUB_BASE_REF" else - compare_base="origin/$GITHUB_BASE_REF" + repo="${GITHUB_REPOSITORY}" + git remote -v + remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')" + git fetch "$remote" "$GITHUB_BASE_REF" + compare_base="$remote/$GITHUB_BASE_REF" fi - changed_files="$(git diff --name-only ${compare_base}...HEAD apps/emqx)" + changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)" if [ "$changed_files" = '' ]; then echo "nothing changed in apps/emqx, ignored." exit 0 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 857682b0e..9698e40a7 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 + container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1 steps: - uses: actions/checkout@v2 @@ -84,7 +84,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker - name: run emqx @@ -162,7 +162,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index b98d8dd53..ce514f583 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -33,7 +33,7 @@ jobs: - amd64 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index db491b89d..196e251b2 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -27,7 +27,7 @@ jobs: - amd64 runs-on: aws-amd64 - container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 @@ -58,7 +58,7 @@ jobs: find_apps: runs-on: aws-amd64 - container: emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 + container: emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04 outputs: fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} @@ -114,7 +114,7 @@ jobs: # produces .coverdata - name: run common test run: | - docker exec -i ${{ matrix.otp_release }} bash -c "make ${{ matrix.app_name }}-ct" + docker exec -i ${{ matrix.otp_release }} bash -c "git config --global --add safe.directory \"$GITHUB_WORKSPACE\" && make ${{ matrix.app_name }}-ct" - uses: actions/upload-artifact@v1 if: matrix.otp_release == 'erlang24' with: @@ -141,7 +141,7 @@ jobs: arch: - amd64 runs-on: aws-amd64 - container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: shell: bash @@ -180,7 +180,7 @@ jobs: - amd64 runs-on: aws-amd64 - container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index db61428f4..77c96cc98 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 2bc40e977..5e05444ba 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,6 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}} +]}. {erl_opts, [ warn_unused_vars, diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 12a2af127..f33e3b77d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -147,7 +147,8 @@ regex_replace/3, ascii/1, find/2, - find/3 + find/3, + jq/2 ]). %% Map Funcs @@ -779,6 +780,23 @@ find_s(S, P, Dir) -> SubStr -> SubStr end. +-spec jq(FilterProgram, JSON) -> Result when + FilterProgram :: binary(), + JSON :: binary() | term(), + Result :: [term()]. +jq(FilterProgram, JSONBin) when + is_binary(FilterProgram), is_binary(JSONBin) +-> + case jq:parse(FilterProgram, JSONBin) of + {ok, Result} -> + [json_decode(JSONString) || JSONString <- Result]; + {error, ErrorReason} -> + erlang:throw({jq_exception, ErrorReason}) + end; +jq(FilterProgram, JSONTerm) when is_binary(FilterProgram) -> + JSONBin = json_encode(JSONTerm), + jq(FilterProgram, JSONBin). + %%------------------------------------------------------------------------------ %% Array Funcs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index af19a24f1..bf274b624 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -57,6 +57,7 @@ groups() -> t_match_atom_and_binary, t_sqlselect_0, t_sqlselect_00, + t_sqlselect_001, t_sqlselect_01, t_sqlselect_02, t_sqlselect_1, @@ -728,6 +729,43 @@ t_sqlselect_00(_Config) -> ) ). +t_sqlselect_001(_Config) -> + %% Verify that the jq function can be called from SQL + Sql = + "select jq('.what + .what', payload) as ans " + "from \"t/#\" ", + ?assertMatch( + {ok, #{<<"ans">> := [8]}}, + emqx_rule_sqltester:test( + #{ + sql => Sql, + context => + #{ + payload => #{<<"what">> => 4}, + topic => <<"t/a">> + } + } + ) + ), + Sql2 = + "SELECT jq('.a|.[]', " + "'{\"a\": [{\"b\": 1}, {\"b\": 2}, {\"b\": 3}]}') " + "as jq_output, " + " jq_output[1].b as first_b from \"t/#\" ", + ?assertMatch( + {ok, #{<<"first_b">> := 1}}, + emqx_rule_sqltester:test( + #{ + sql => Sql2, + context => + #{ + payload => #{<<"what">> => 4}, + topic => <<"t/a">> + } + } + ) + ). + t_sqlselect_01(_Config) -> SQL = "SELECT json_decode(payload) as p, payload " diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 132f874e4..5b27e3efa 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -629,6 +629,24 @@ t_regex_replace(_) -> ?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])), ?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])). +jq_1_elm_res(JSONString) -> + Bin = list_to_binary(JSONString), + [apply_func(json_decode, [Bin])]. + +t_jq(_) -> + ?assertEqual( + jq_1_elm_res("{\"b\":2}"), + apply_func(jq, [<<".">>, apply_func(json_decode, [<<"{\"b\": 2}">>])]) + ), + ?assertEqual( + jq_1_elm_res("6"), + apply_func(jq, [<<".+1">>, apply_func(json_decode, [<<"5">>])]) + ), + ?assertEqual( + jq_1_elm_res("{\"b\":2}"), + apply_func(jq, [<<".">>, <<"{\"b\": 2}">>]) + ). + ascii_string() -> list(range(0, 127)). bin(S) -> iolist_to_binary(S). diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index e9ed73df6..2e7dc0c8f 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,22 +1,27 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder RUN apk add --no-cache \ - git \ - curl \ - gcc \ - g++ \ - make \ - perl \ - ncurses-dev \ - openssl-dev \ - coreutils \ + autoconf \ + automake \ + bash \ + bison \ bsd-compat-headers \ + coreutils \ + curl \ + flex \ + g++ \ + gcc \ + git \ + jq \ libc-dev \ libstdc++ \ - bash \ - jq + libtool \ + make \ + ncurses-dev \ + openssl-dev \ + perl COPY . /emqx diff --git a/elvis.config b/elvis.config index 189d2ae72..ee7eaeaee 100644 --- a/elvis.config +++ b/elvis.config @@ -37,6 +37,12 @@ {elvis_style, nesting_level, #{ level => 6 }} ] }, + #{dirs => ["apps/emqx_rule_engine/src"], + filter => "*_rule_funcs.erl", + rules => [ + {elvis_style, god_modules, disable} + ] + }, #{dirs => ["."], filter => "Makefile", ruleset => makefiles diff --git a/mix.exs b/mix.exs index 5ad69b7d6..ea67ff714 100644 --- a/mix.exs +++ b/mix.exs @@ -91,7 +91,7 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.11.2", override: true, runtime: false} - ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep() + ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() end defp umbrella_apps() do @@ -202,6 +202,7 @@ defmodule EMQXUmbrella.MixProject do ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ + if(enable_jq?(), do: [jq: :permanent], else: []) ++ if(edition_type == :enterprise, do: [ emqx_enterprise_conf: :load, @@ -608,6 +609,12 @@ defmodule EMQXUmbrella.MixProject do else: [] end + defp jq_dep() do + if enable_jq?(), + do: [{:jq, github: "emqx/jq", tag: "v0.1.0", override: true}], + else: [] + end + defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt @@ -619,6 +626,10 @@ defmodule EMQXUmbrella.MixProject do not win32?() end + defp enable_jq?() do + not win32?() + end + defp enable_quicer?() do not Enum.any?([ build_without_quic?(), diff --git a/pkg-vsn.sh b/pkg-vsn.sh index d0bcb6b93..5894646b7 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -84,6 +84,8 @@ RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.h git_exact_vsn() { local tag + ## Needed to avoid error in github action + git config --global --add safe.directory "/__w/emqx/emqx" tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)" echo "${tag#[v|e]}" } diff --git a/rebar.config.erl b/rebar.config.erl index 834f7988e..bf9d4fce5 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -19,12 +19,17 @@ assert_otp() -> OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < Oldest orelse OtpRelease > Latest of true -> - io:format(standard_error, "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n", - [OtpRelease, Oldest, Latest]), + io:format( + standard_error, + "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n", + [OtpRelease, Oldest, Latest] + ), halt(1); false when OtpRelease =/= Latest -> - io:format("WARNING: Erlang/OTP version ~p found, recommended==~p~n", - [OtpRelease, Latest]); + io:format( + "WARNING: Erlang/OTP version ~p found, recommended==~p~n", + [OtpRelease, Latest] + ); false -> ok end. @@ -35,16 +40,19 @@ bcrypt() -> quicer() -> {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}. +jq() -> + {jq, {git, "https://github.com/emqx/jq", {tag, "v0.1.0"}}}. + deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), - MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++ - [quicer() || is_quicer_supported()], + MoreDeps = + [bcrypt() || provide_bcrypt_dep()] ++ + [jq() || provide_jq()] ++ + [quicer() || is_quicer_supported()], lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}). overrides() -> - [ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]} - ]} - ] ++ snabbkaffe_overrides(). + [{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}] ++ snabbkaffe_overrides(). %% Temporary workaround for a rebar3 erl_opts duplication %% bug. Ideally, we want to set this define globally @@ -53,14 +61,15 @@ snabbkaffe_overrides() -> [{add, App, [{erl_opts, [{d, snk_kind, msg}]}]} || App <- Apps]. config() -> - [ {cover_enabled, is_cover_enabled()} - , {profiles, profiles()} - , {plugins, plugins()} + [ + {cover_enabled, is_cover_enabled()}, + {profiles, profiles()}, + {plugins, plugins()} ]. is_cover_enabled() -> case os:getenv("ENABLE_COVER_COMPILE") of - "1"-> true; + "1" -> true; "true" -> true; _ -> false end. @@ -70,14 +79,13 @@ is_enterprise(ee) -> true. is_quicer_supported() -> not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse - is_win32() orelse is_centos_6() - ). + is_win32() orelse is_centos_6()). is_centos_6() -> %% reason: %% glibc is too old case file:read_file("/etc/centos-release") of - {ok, <<"CentOS release 6", _/binary >>} -> + {ok, <<"CentOS release 6", _/binary>>} -> true; _ -> false @@ -88,145 +96,158 @@ is_win32() -> project_app_dirs(Edition) -> ["apps/*"] ++ - case is_enterprise(Edition) of - true -> ["lib-ee/*"]; - false -> [] - end. + case is_enterprise(Edition) of + true -> ["lib-ee/*"]; + false -> [] + end. plugins() -> - [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} - , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}} - %% emqx main project does not require port-compiler - %% pin at root level for deterministic - , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} - ] - %% test plugins are concatenated to default profile plugins - %% otherwise rebar3 test profile runs are super slow - ++ test_plugins(). + [ + {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}, + {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}, + %% emqx main project does not require port-compiler + %% pin at root level for deterministic + {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} + ] ++ + %% test plugins are concatenated to default profile plugins + %% otherwise rebar3 test profile runs are super slow + test_plugins(). test_plugins() -> - [ {rebar3_proper, "0.12.1"} - , {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} + [ + {rebar3_proper, "0.12.1"}, + {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} ]. test_deps() -> - [ {bbmustache, "1.10.0"} - , {meck, "0.9.2"} - , {proper, "1.4.0"} + [ + {bbmustache, "1.10.0"}, + {meck, "0.9.2"}, + {proper, "1.4.0"} ]. common_compile_opts(Edition, Vsn) -> - [ debug_info % always include debug_info - , {compile_info, [{emqx_vsn, Vsn}]} - , {d, 'EMQX_RELEASE_EDITION', Edition} + % always include debug_info + [ + debug_info, + {compile_info, [{emqx_vsn, Vsn}]}, + {d, 'EMQX_RELEASE_EDITION', Edition} ] ++ - [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. + [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"]. prod_compile_opts(Edition, Vsn) -> - [ compressed - , deterministic - , warnings_as_errors - | common_compile_opts(Edition, Vsn) + [ + compressed, + deterministic, + warnings_as_errors + | common_compile_opts(Edition, Vsn) ]. prod_overrides() -> - [{add, [ {erl_opts, [deterministic]}]}]. + [{add, [{erl_opts, [deterministic]}]}]. profiles() -> profiles_ce() ++ profiles_ee() ++ profiles_dev(). profiles_ce() -> Vsn = get_vsn(emqx), - [ {'emqx', - [ {erl_opts, prod_compile_opts(ce, Vsn)} - , {relx, relx(Vsn, cloud, bin, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{compile, "bash build emqx doc"}]} - ]} - , {'emqx-pkg', - [ {erl_opts, prod_compile_opts(ce, Vsn)} - , {relx, relx(Vsn, cloud, pkg, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{compile, "bash build emqx-pkg doc"}]} - ]} - , {'emqx-edge', - [ {erl_opts, prod_compile_opts(edge, Vsn)} - , {relx, relx(Vsn, edge, bin, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{compile, "bash build emqx-edge doc"}]} - ]} - , {'emqx-edge-pkg', - [ {erl_opts, prod_compile_opts(edge, Vsn)} - , {relx, relx(Vsn, edge, pkg, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]} - ]} + [ + {'emqx', [ + {erl_opts, prod_compile_opts(ce, Vsn)}, + {relx, relx(Vsn, cloud, bin, ce)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ce)}, + {post_hooks, [{compile, "bash build emqx doc"}]} + ]}, + {'emqx-pkg', [ + {erl_opts, prod_compile_opts(ce, Vsn)}, + {relx, relx(Vsn, cloud, pkg, ce)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ce)}, + {post_hooks, [{compile, "bash build emqx-pkg doc"}]} + ]}, + {'emqx-edge', [ + {erl_opts, prod_compile_opts(edge, Vsn)}, + {relx, relx(Vsn, edge, bin, ce)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ce)}, + {post_hooks, [{compile, "bash build emqx-edge doc"}]} + ]}, + {'emqx-edge-pkg', [ + {erl_opts, prod_compile_opts(edge, Vsn)}, + {relx, relx(Vsn, edge, pkg, ce)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ce)}, + {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]} + ]} ]. profiles_ee() -> Vsn = get_vsn('emqx-enterprise'), - [ {'emqx-enterprise', - [ {erl_opts, prod_compile_opts(ee, Vsn)} - , {relx, relx(Vsn, cloud, bin, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} - ]} - , {'emqx-enterprise-pkg', - [ {erl_opts, prod_compile_opts(ee, Vsn)} - , {relx, relx(Vsn, cloud, pkg, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} - ]} + [ + {'emqx-enterprise', [ + {erl_opts, prod_compile_opts(ee, Vsn)}, + {relx, relx(Vsn, cloud, bin, ee)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ee)}, + {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} + ]}, + {'emqx-enterprise-pkg', [ + {erl_opts, prod_compile_opts(ee, Vsn)}, + {relx, relx(Vsn, cloud, pkg, ee)}, + {overrides, prod_overrides()}, + {project_app_dirs, project_app_dirs(ee)}, + {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} + ]} ]. %% EE has more files than CE, always test/check with EE options. profiles_dev() -> Vsn = get_vsn('emqx-enterprise'), - [ {check, - [ {erl_opts, common_compile_opts(ee, Vsn)} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {test, - [ {deps, test_deps()} - , {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()} - , {extra_src_dirs, [{"test", [{recursive, true}]}]} - , {project_app_dirs, project_app_dirs(ee)} - ]} + [ + {check, [ + {erl_opts, common_compile_opts(ee, Vsn)}, + {project_app_dirs, project_app_dirs(ee)} + ]}, + {test, [ + {deps, test_deps()}, + {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()}, + {extra_src_dirs, [{"test", [{recursive, true}]}]}, + {project_app_dirs, project_app_dirs(ee)} + ]} ]. %% RelType: cloud (full size) | edge (slim size) %% PkgType: bin | pkg %% Edition: ce (community) | ee (enterprise) relx(Vsn, RelType, PkgType, Edition) -> - [ {include_src,false} - , {include_erts, true} - , {extended_start_script,false} - , {generate_start_script,false} - , {sys_config,false} - , {vm_args,false} - , {release, {emqx, Vsn}, relx_apps(RelType, Edition)} - , {overlay, relx_overlay(RelType, Edition)} - , {overlay_vars, build_info() ++ - [ {emqx_description, emqx_description(RelType, Edition)} - | overlay_vars(RelType, PkgType, Edition) - ]} + [ + {include_src, false}, + {include_erts, true}, + {extended_start_script, false}, + {generate_start_script, false}, + {sys_config, false}, + {vm_args, false}, + {release, {emqx, Vsn}, relx_apps(RelType, Edition)}, + {overlay, relx_overlay(RelType, Edition)}, + {overlay_vars, + build_info() ++ + [ + {emqx_description, emqx_description(RelType, Edition)} + | overlay_vars(RelType, PkgType, Edition) + ]} ]. %% Make a HOCON compatible format build_info() -> Os = os_cmd("./scripts/get-distro.sh"), - [ {build_info_arch, erlang:system_info(system_architecture)} - , {build_info_wordsize, rebar_utils:wordsize()} - , {build_info_os, Os} - , {build_info_erlang, rebar_utils:otp_release()} - , {build_info_elixir, none} - , {build_info_relform, relform()} + [ + {build_info_arch, erlang:system_info(system_architecture)}, + {build_info_wordsize, rebar_utils:wordsize()}, + {build_info_os, Os}, + {build_info_erlang, rebar_utils:otp_release()}, + {build_info_elixir, none}, + {build_info_relform, relform()} ]. relform() -> @@ -237,111 +258,120 @@ relform() -> emqx_description(cloud, ee) -> "EMQX Enterprise"; emqx_description(cloud, ce) -> "EMQX"; -emqx_description(edge, ce) -> "EMQX Edge". +emqx_description(edge, ce) -> "EMQX Edge". overlay_vars(RelType, PkgType, Edition) -> - overlay_vars_rel(RelType) - ++ overlay_vars_pkg(PkgType) - ++ overlay_vars_edition(Edition). + overlay_vars_rel(RelType) ++ + overlay_vars_pkg(PkgType) ++ + overlay_vars_edition(Edition). %% vars per release type, cloud or edge overlay_vars_rel(RelType) -> - VmArgs = case RelType of - cloud -> "vm.args"; - edge -> "vm.args.edge" - end, + VmArgs = + case RelType of + cloud -> "vm.args"; + edge -> "vm.args.edge" + end, - [ {vm_args_file, VmArgs} - ]. + [{vm_args_file, VmArgs}]. overlay_vars_edition(ce) -> - [ {emqx_schema_mod, emqx_conf_schema} - , {is_enterprise, "no"} - , {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)} + [ + {emqx_schema_mod, emqx_conf_schema}, + {is_enterprise, "no"}, + {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)} ]; overlay_vars_edition(ee) -> - [ {emqx_schema_mod, emqx_enterprise_conf_schema} - , {is_enterprise, "yes"} - , {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)} + [ + {emqx_schema_mod, emqx_enterprise_conf_schema}, + {is_enterprise, "yes"}, + {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)} ]. %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) overlay_vars_pkg(bin) -> - [ {platform_data_dir, "data"} - , {platform_etc_dir, "etc"} - , {platform_log_dir, "log"} - , {platform_plugins_dir, "plugins"} - , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} - , {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"} - , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} - , {runner_log_dir, "$RUNNER_ROOT_DIR/log"} - , {runner_user, ""} - , {is_elixir, "no"} + [ + {platform_data_dir, "data"}, + {platform_etc_dir, "etc"}, + {platform_log_dir, "log"}, + {platform_plugins_dir, "plugins"}, + {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}, + {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"}, + {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}, + {runner_log_dir, "$RUNNER_ROOT_DIR/log"}, + {runner_user, ""}, + {is_elixir, "no"} ]; overlay_vars_pkg(pkg) -> - [ {platform_data_dir, "/var/lib/emqx"} - , {platform_etc_dir, "/etc/emqx"} - , {platform_log_dir, "/var/log/emqx"} - , {platform_plugins_dir, "/var/lib/emqx/plugins"} - , {runner_bin_dir, "/usr/bin"} - , {emqx_etc_dir, "/etc/emqx"} - , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} - , {runner_log_dir, "/var/log/emqx"} - , {runner_user, "emqx"} - , {is_elixir, "no"} + [ + {platform_data_dir, "/var/lib/emqx"}, + {platform_etc_dir, "/etc/emqx"}, + {platform_log_dir, "/var/log/emqx"}, + {platform_plugins_dir, "/var/lib/emqx/plugins"}, + {runner_bin_dir, "/usr/bin"}, + {emqx_etc_dir, "/etc/emqx"}, + {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}, + {runner_log_dir, "/var/log/emqx"}, + {runner_user, "emqx"}, + {is_elixir, "no"} ]. relx_apps(ReleaseType, Edition) -> - [ kernel - , sasl - , crypto - , public_key - , asn1 - , syntax_tools - , ssl - , os_mon - , inets - , compiler - , runtime_tools - , redbug - , {hocon, load} - , {emqx, load} % started by emqx_machine - , {emqx_conf, load} - , emqx_machine - , {mnesia, load} - , {ekka, load} - , {emqx_plugin_libs, load} - , {esasl, load} - , observer_cli - , {system_monitor, load} % started by emqx_machine - , emqx_http_lib - , emqx_resource - , emqx_connector - , emqx_authn - , emqx_authz - , emqx_auto_subscribe - , emqx_gateway - , emqx_exhook - , emqx_bridge - , emqx_rule_engine - , emqx_modules - , emqx_management - , emqx_dashboard - , emqx_retainer - , emqx_statsd - , emqx_prometheus - , emqx_psk - , emqx_slow_subs - , emqx_plugins - ] - ++ [quicer || is_quicer_supported()] - ++ [bcrypt || provide_bcrypt_release(ReleaseType)] - ++ relx_apps_per_rel(ReleaseType) - ++ relx_additional_apps(ReleaseType, Edition). + [ + kernel, + sasl, + crypto, + public_key, + asn1, + syntax_tools, + ssl, + os_mon, + inets, + compiler, + runtime_tools, + redbug, + {hocon, load}, + % started by emqx_machine + {emqx, load}, + {emqx_conf, load}, + emqx_machine, + {mnesia, load}, + {ekka, load}, + {emqx_plugin_libs, load}, + {esasl, load}, + observer_cli, + % started by emqx_machine + {system_monitor, load}, + emqx_http_lib, + emqx_resource, + emqx_connector, + emqx_authn, + emqx_authz, + emqx_auto_subscribe, + emqx_gateway, + emqx_exhook, + emqx_bridge, + emqx_rule_engine, + emqx_modules, + emqx_management, + emqx_dashboard, + emqx_retainer, + emqx_statsd, + emqx_prometheus, + emqx_psk, + emqx_slow_subs, + emqx_plugins + ] ++ + [quicer || is_quicer_supported()] ++ + [bcrypt || provide_bcrypt_release(ReleaseType)] ++ + [jq || provide_jq()] ++ + relx_apps_per_rel(ReleaseType) ++ + relx_additional_apps(ReleaseType, Edition). relx_apps_per_rel(cloud) -> - [ xmerl - | [{observer, load} || is_app(observer)] + [ + xmerl + | [{observer, load} || is_app(observer)] ]; relx_apps_per_rel(edge) -> []. @@ -349,13 +379,13 @@ relx_apps_per_rel(edge) -> is_app(Name) -> case application:load(Name) of ok -> true; - {error,{already_loaded, _}} -> true; + {error, {already_loaded, _}} -> true; _ -> false end. relx_additional_apps(ReleaseType, Edition) -> - relx_plugin_apps_per_rel(ReleaseType) - ++ relx_apps_per_edition(Edition). + relx_plugin_apps_per_rel(ReleaseType) ++ + relx_apps_per_edition(Edition). relx_plugin_apps_per_rel(cloud) -> []; @@ -363,101 +393,110 @@ relx_plugin_apps_per_rel(edge) -> []. relx_apps_per_edition(ee) -> - [ emqx_license - , {emqx_enterprise_conf, load} + [ + emqx_license, + {emqx_enterprise_conf, load} ]; - -relx_apps_per_edition(ce) -> []. +relx_apps_per_edition(ce) -> + []. emqx_machine_boot_apps(ce) -> - [ emqx_prometheus - , emqx_modules - , emqx_dashboard - , emqx_connector - , emqx_gateway - , emqx_statsd - , emqx_resource - , emqx_rule_engine - , emqx_bridge - , emqx_plugin_libs - , emqx_management - , emqx_retainer - , emqx_exhook - , emqx_authn - , emqx_authz - , emqx_slow_subs - , emqx_auto_subscribe - , emqx_plugins + [ + emqx_prometheus, + emqx_modules, + emqx_dashboard, + emqx_connector, + emqx_gateway, + emqx_statsd, + emqx_resource, + emqx_rule_engine, + emqx_bridge, + emqx_plugin_libs, + emqx_management, + emqx_retainer, + emqx_exhook, + emqx_authn, + emqx_authz, + emqx_slow_subs, + emqx_auto_subscribe, + emqx_plugins ]; - emqx_machine_boot_apps(ee) -> emqx_machine_boot_apps(ce) ++ - []. + []. emqx_machine_boot_app_list(Edition) -> string:join( - [atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)], - ", "). + [atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)], + ", " + ). relx_overlay(ReleaseType, Edition) -> - [ {mkdir, "log/"} - , {mkdir, "data/"} - , {mkdir, "plugins"} - , {mkdir, "data/mnesia"} - , {mkdir, "data/configs"} - , {mkdir, "data/patches"} - , {mkdir, "data/scripts"} - , {template, "rel/emqx_vars", "releases/emqx_vars"} - , {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"} - , {copy, "bin/emqx", "bin/emqx"} - , {copy, "bin/emqx_ctl", "bin/emqx_ctl"} - , {copy, "bin/node_dump", "bin/node_dump"} - , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"} - , {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup - , {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup - , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup - , {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"} - , {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"} - , {template, "bin/emqx.cmd", "bin/emqx.cmd"} - , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} - , {copy, "bin/nodetool", "bin/nodetool"} - , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} + [ + {mkdir, "log/"}, + {mkdir, "data/"}, + {mkdir, "plugins"}, + {mkdir, "data/mnesia"}, + {mkdir, "data/configs"}, + {mkdir, "data/patches"}, + {mkdir, "data/scripts"}, + {template, "rel/emqx_vars", "releases/emqx_vars"}, + {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"}, + {copy, "bin/emqx", "bin/emqx"}, + {copy, "bin/emqx_ctl", "bin/emqx_ctl"}, + {copy, "bin/node_dump", "bin/node_dump"}, + {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}, + %% for relup + {copy, "bin/emqx", "bin/emqx-{{release_version}}"}, + %% for relup + {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, + %% for relup + {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, + {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, + {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, + {template, "bin/emqx.cmd", "bin/emqx.cmd"}, + {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}, + {copy, "bin/nodetool", "bin/nodetool"}, + {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} ] ++ etc_overlay(ReleaseType, Edition). etc_overlay(ReleaseType, Edition) -> Templates = emqx_etc_overlay(ReleaseType, Edition), - [ {mkdir, "etc/"} - , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"} + [ + {mkdir, "etc/"}, + {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"} ] ++ - lists:map( - fun({From, To}) -> {template, From, To}; - (FromTo) -> {template, FromTo, FromTo} - end, Templates). + lists:map( + fun + ({From, To}) -> {template, From, To}; + (FromTo) -> {template, FromTo, FromTo} + end, + Templates + ). emqx_etc_overlay(ReleaseType, Edition) -> - emqx_etc_overlay_per_rel(ReleaseType) - ++ emqx_etc_overlay_per_edition(Edition) - ++ emqx_etc_overlay_common(). + emqx_etc_overlay_per_rel(ReleaseType) ++ + emqx_etc_overlay_per_edition(Edition) ++ + emqx_etc_overlay_common(). emqx_etc_overlay_per_rel(cloud) -> - [ {"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args","etc/vm.args"} - ]; + [{"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args", "etc/vm.args"}]; emqx_etc_overlay_per_rel(edge) -> - [ {"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args","etc/vm.args"} - ]. + [{"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args", "etc/vm.args"}]. emqx_etc_overlay_common() -> - [ {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"} - ]. + [{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}]. emqx_etc_overlay_per_edition(ce) -> - [ {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} - , {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} + [ + {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}, + {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} ]; emqx_etc_overlay_per_edition(ee) -> - [ {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"} - , {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} - , {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} + [ + {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"}, + {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}, + {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} ]. get_vsn(Profile) -> @@ -468,10 +507,11 @@ get_vsn(Profile) -> os_cmd(Cmd) -> Output = os:cmd("bash " ++ Cmd), - re:replace(Output, "\n", "", [{return ,list}]). + re:replace(Output, "\n", "", [{return, list}]). maybe_dump(Config) -> - is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), + is_debug() andalso + file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), Config. is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC"). @@ -486,54 +526,67 @@ is_debug(VarName) -> provide_bcrypt_dep() -> not is_win32(). +provide_jq() -> + not is_win32(). + provide_bcrypt_release(ReleaseType) -> provide_bcrypt_dep() andalso ReleaseType =:= cloud. erl_opts_i() -> [{i, "apps"}] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))]. + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))]. dialyzer(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), - AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of - false -> - []; - Value -> - [ list_to_atom(App) || App <- string:tokens(Value, ",")] - end, + AppsToAnalyse = + case os:getenv("DIALYZER_ANALYSE_APP") of + false -> + []; + Value -> + [list_to_atom(App) || App <- string:tokens(Value, ",")] + end, AppNames = app_names(), - KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], + KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], AppsToExclude = AppNames -- KnownApps, case length(AppsToAnalyse) > 0 of true -> - lists:keystore(dialyzer, 1, Config, {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}); + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]} + ); false -> Config end. coveralls() -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of - {"true", Token} when is_list(Token) -> - Cfgs = [{coveralls_repo_token, Token}, + {"true", Token} when is_list(Token) -> + Cfgs = [ + {coveralls_repo_token, Token}, {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, {coveralls_commit_sha, os:getenv("GITHUB_SHA")}, {coveralls_coverdata, "_build/test/cover/*.coverdata"}, - {coveralls_service_name, "github"}], - case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" - andalso string:tokens(os:getenv("GITHUB_REF"), "/") of - [_, "pull", PRNO, _] -> - [{coveralls_service_pull_request, PRNO} | Cfgs]; - _ -> - Cfgs - end; - _ -> - [] + {coveralls_service_name, "github"} + ], + case + os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso + string:tokens(os:getenv("GITHUB_REF"), "/") + of + [_, "pull", PRNO, _] -> + [{coveralls_service_pull_request, PRNO} | Cfgs]; + _ -> + Cfgs + end; + _ -> + [] end. app_names() -> list_dir("apps") ++ list_dir("lib-ee"). diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 301723fcb..0751458a0 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10 +## --builder ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10 set -euo pipefail @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source ode in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10" } while [ "$#" -gt 0 ]; do @@ -120,13 +120,6 @@ fi HOST_SYSTEM="$(./scripts/get-distro.sh)" BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')" -# quick workaround before builder image is renamed -if [ "$BUILDER_SYSTEM" = 'centos7' ]; then - BUILDER_SYSTEM='el7' -elif [ "$BUILDER_SYSTEM" = 'rockylinux8' ]; then - BUILDER_SYSTEM='el8' -fi - CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" IS_NATIVE_SYSTEM='no'