yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rule_engine): add jq function to the rule engine 

This commit adds a function to the rule engine that alows users
to transform text or JSON objects using [jq filter programs][1].

[jq][1] is a command line tool that can be used to transform
and filter JSON text using jq's built-in language. The rule engine
function that is added with this commit uses the
[Erlang jq NIF library][2] that wraps the jq C library in an
Erlang NIF function.

[1]: https://stedolan.github.io/jq/
[2]: https://github.com/emqx/jq
This commit is contained in:
Kjell Winblad 2022-03-23 15:16:57 +01:00
parent 8ff552c9cf
commit 74c33cd4e5
24 changed files with 528 additions and 356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services: services:
erlang23: erlang23:
container_name: erlang23 container_name: erlang23
image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-23.3.4.9-4-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-23.3.4.9-4-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:
@ -23,7 +23,7 @@ services:
erlang24: erlang24:
container_name: erlang24 container_name: erlang24
image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:

4
.github/workflows/apps_version_check.yaml vendored
View File

@ -7,6 +7,8 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Check apps version - name: Check apps version
run: ./scripts/apps-version-check.sh run: ./scripts/apps-version-check.sh

17
.github/workflows/build_packages.yaml vendored
View File

@ -22,7 +22,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
outputs: outputs:
BUILD_PROFILES: ${{ steps.get_profiles.outputs.BUILD_PROFILES }} BUILD_PROFILES: ${{ steps.get_profiles.outputs.BUILD_PROFILES }}
@ -139,7 +139,8 @@ jobs:
- name: prepare - name: prepare
run: | run: |
brew update brew update
brew install curl zip unzip gnu-sed kerl unixodbc freetds brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store git config --global credential.helper store
- uses: actions/cache@v2 - uses: actions/cache@v2
@ -205,7 +206,7 @@ jobs:
needs: prepare needs: prepare
runs-on: ${{ matrix.build_machine }} runs-on: ${{ matrix.build_machine }}
container: container:
image: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" image: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
strategy: strategy:
fail-fast: false fail-fast: false
@ -234,8 +235,8 @@ jobs:
- debian11 - debian11
- debian10 - debian10
- debian9 - debian9
- rockylinux8 - el8
- centos7 - el7
- raspbian10 - raspbian10
build_machine: build_machine:
- aws-arm64 - aws-arm64
@ -272,7 +273,7 @@ jobs:
elixir: 1.13.3 elixir: 1.13.3
build_elixir: with_elixir build_elixir: with_elixir
arch: amd64 arch: amd64
os: centos7 os: el7
build_machine: ubuntu-20.04 build_machine: ubuntu-20.04
defaults: defaults:
@ -325,7 +326,7 @@ jobs:
--pkgtype "${PKGTYPE}" \ --pkgtype "${PKGTYPE}" \
--arch "${ARCH}" \ --arch "${ARCH}" \
--elixir "${IsElixir}" \ --elixir "${IsElixir}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-10:${ELIXIR}-${OTP}-${SYSTEM}" --builder "ghcr.io/emqx/emqx-builder/5.0-14:${ELIXIR}-${OTP}-${SYSTEM}"
done done
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
@ -462,7 +463,7 @@ jobs:
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
RUN_FROM=${{ steps.pre-meta.outputs.img }} RUN_FROM=${{ steps.pre-meta.outputs.img }}
EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }} EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }}
file: source/deploy/docker/Dockerfile file: source/deploy/docker/Dockerfile

18
.github/workflows/build_slim_packages.yaml vendored
View File

@ -38,15 +38,15 @@ jobs:
- 1.13.3 - 1.13.3
os: os:
- ubuntu20.04 - ubuntu20.04
- rockylinux8 - el8
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- name: cleanup - uses: AutoModality/action-clean@v1
run: | - uses: actions/checkout@v2
rm -rf "${GITHUB_WORKSPACE}/" with:
- uses: actions/checkout@v1 fetch-depth: 0
- name: prepare - name: prepare
run: | run: |
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
@ -63,6 +63,9 @@ jobs:
_build/default/lib/quicer/ _build/default/lib/quicer/
deps/quicer/ deps/quicer/
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: build and test tgz package - name: build and test tgz package
run: | run: |
make ${EMQX_NAME}-tgz make ${EMQX_NAME}-tgz
@ -148,7 +151,8 @@ jobs:
- name: prepare - name: prepare
run: | run: |
brew update brew update
brew install curl zip unzip gnu-sed kerl unixodbc freetds brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
- uses: actions/cache@v2 - uses: actions/cache@v2

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request, push]
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

5
.github/workflows/code_style_check.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs: jobs:
code_style_check: code_style_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -13,6 +13,9 @@ jobs:
- name: Check line-break at EOF - name: Check line-break at EOF
run: | run: |
./scripts/check-nl-at-eof.sh ./scripts/check-nl-at-eof.sh
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Check Elixir code formatting - name: Check Elixir code formatting
run: | run: |
mix format --check-formatted mix format --check-formatted

11
.github/workflows/elixir_apps_check.yaml vendored
View File

@ -6,9 +6,9 @@ on: [pull_request, push]
jobs: jobs:
elixir_apps_check: elixir_apps_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-latest
# just use the latest builder # just use the latest builder
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
strategy: strategy:
fail-fast: false fail-fast: false
@ -31,12 +31,17 @@ jobs:
edition_type: enterprise edition_type: enterprise
steps: steps:
- name: fix_git_permission
run: git config --global --add safe.directory '/__w/emqx/emqx'
- name: Checkout - name: Checkout
uses: actions/checkout@v2.4.0 uses: actions/checkout@v2
with: with:
fetch-depth: 0 fetch-depth: 0
- name: ensure rebar - name: ensure rebar
run: ./scripts/ensure-rebar3.sh run: ./scripts/ensure-rebar3.sh
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: check applications - name: check applications
run: ./scripts/check-elixir-applications.exs run: ./scripts/check-elixir-applications.exs
- name: check applications started with emqx_machine - name: check applications started with emqx_machine

7
.github/workflows/elixir_deps_check.yaml vendored
View File

@ -7,13 +7,16 @@ on: [pull_request, push]
jobs: jobs:
elixir_deps_check: elixir_deps_check:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2.4.0 uses: actions/checkout@v2
- name: ensure rebar - name: ensure rebar
run: ./scripts/ensure-rebar3.sh run: ./scripts/ensure-rebar3.sh
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: setup mix - name: setup mix
run: | run: |
mix local.hex --force mix local.hex --force

7
.github/workflows/elixir_release.yml vendored
View File

@ -12,13 +12,16 @@ on:
jobs: jobs:
elixir_release_build: elixir_release_build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2.4.0 uses: actions/checkout@v2
- name: install tools - name: install tools
run: apt update && apt install netcat-openbsd run: apt update && apt install netcat-openbsd
- name: Work around https://github.com/actions/checkout/issues/766
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: elixir release - name: elixir release
run: make emqx-elixir run: make emqx-elixir
- name: start release - name: start release

11
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -25,7 +25,7 @@ jobs:
- amd64 - amd64
runs-on: aws-amd64 runs-on: aws-amd64
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
defaults: defaults:
run: run:
@ -47,14 +47,19 @@ jobs:
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: run - name: run
run: | run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
echo "git diff base: $GITHUB_BASE_REF" echo "git diff base: $GITHUB_BASE_REF"
if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then
# base is a commit sha1 # base is a commit sha1
compare_base="$GITHUB_BASE_REF" compare_base="$GITHUB_BASE_REF"
else else
compare_base="origin/$GITHUB_BASE_REF" repo="${GITHUB_REPOSITORY}"
git remote -v
remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')"
git fetch "$remote" "$GITHUB_BASE_REF"
compare_base="$remote/$GITHUB_BASE_REF"
fi fi
changed_files="$(git diff --name-only ${compare_base}...HEAD apps/emqx)" changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)"
if [ "$changed_files" = '' ]; then if [ "$changed_files" = '' ]; then
echo "nothing changed in apps/emqx, ignored." echo "nothing changed in apps/emqx, ignored."
exit 0 exit 0

6
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix # prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -84,7 +84,7 @@ jobs:
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker
- name: run emqx - name: run emqx
@ -162,7 +162,7 @@ jobs:
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
run: | run: |
make ${{ matrix.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV

2
.github/workflows/run_relup_tests.yaml vendored
View File

@ -33,7 +33,7 @@ jobs:
- amd64 - amd64
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
defaults: defaults:
run: run:

10
.github/workflows/run_test_cases.yaml vendored
View File

@ -27,7 +27,7 @@ jobs:
- amd64 - amd64
runs-on: aws-amd64 runs-on: aws-amd64
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -58,7 +58,7 @@ jobs:
find_apps: find_apps:
runs-on: aws-amd64 runs-on: aws-amd64
container: emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04 container: emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
outputs: outputs:
fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
@ -114,7 +114,7 @@ jobs:
# produces <app-name>.coverdata # produces <app-name>.coverdata
- name: run common test - name: run common test
run: | run: |
docker exec -i ${{ matrix.otp_release }} bash -c "make ${{ matrix.app_name }}-ct" docker exec -i ${{ matrix.otp_release }} bash -c "git config --global --add safe.directory \"$GITHUB_WORKSPACE\" && make ${{ matrix.app_name }}-ct"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: matrix.otp_release == 'erlang24' if: matrix.otp_release == 'erlang24'
with: with:
@ -141,7 +141,7 @@ jobs:
arch: arch:
- amd64 - amd64
runs-on: aws-amd64 runs-on: aws-amd64
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
defaults: defaults:
run: run:
shell: bash shell: bash
@ -180,7 +180,7 @@ jobs:
- amd64 - amd64
runs-on: aws-amd64 runs-on: aws-amd64
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

2
Makefile
View File

@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export EMQX_DEFAULT_RUNNER = alpine:3.15.1
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)

4
apps/emqx_rule_engine/rebar.config
View File

@ -1,6 +1,8 @@
%% -*- mode: erlang -*- %% -*- mode: erlang -*-
{deps, [{emqx, {path, "../emqx"}}]}. {deps, [
{emqx, {path, "../emqx"}}
]}.
{erl_opts, [ {erl_opts, [
warn_unused_vars, warn_unused_vars,

20
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -147,7 +147,8 @@
regex_replace/3, regex_replace/3,
ascii/1, ascii/1,
find/2, find/2,
find/3 find/3,
jq/2
]). ]).
%% Map Funcs %% Map Funcs
@ -779,6 +780,23 @@ find_s(S, P, Dir) ->
SubStr -> SubStr SubStr -> SubStr
end. end.
-spec jq(FilterProgram, JSON) -> Result when
FilterProgram :: binary(),
JSON :: binary() | term(),
Result :: [term()].
jq(FilterProgram, JSONBin) when
is_binary(FilterProgram), is_binary(JSONBin)
->
case jq:parse(FilterProgram, JSONBin) of
{ok, Result} ->
[json_decode(JSONString) || JSONString <- Result];
{error, ErrorReason} ->
erlang:throw({jq_exception, ErrorReason})
end;
jq(FilterProgram, JSONTerm) when is_binary(FilterProgram) ->
JSONBin = json_encode(JSONTerm),
jq(FilterProgram, JSONBin).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Array Funcs %% Array Funcs
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------

38
apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
View File

@ -57,6 +57,7 @@ groups() ->
t_match_atom_and_binary, t_match_atom_and_binary,
t_sqlselect_0, t_sqlselect_0,
t_sqlselect_00, t_sqlselect_00,
t_sqlselect_001,
t_sqlselect_01, t_sqlselect_01,
t_sqlselect_02, t_sqlselect_02,
t_sqlselect_1, t_sqlselect_1,
@ -728,6 +729,43 @@ t_sqlselect_00(_Config) ->
) )
). ).
t_sqlselect_001(_Config) ->
%% Verify that the jq function can be called from SQL
Sql =
"select jq('.what + .what', payload) as ans "
"from \"t/#\" ",
?assertMatch(
{ok, #{<<"ans">> := [8]}},
emqx_rule_sqltester:test(
#{
sql => Sql,
context =>
#{
payload => #{<<"what">> => 4},
topic => <<"t/a">>
}
}
)
),
Sql2 =
"SELECT jq('.a|.[]', "
"'{\"a\": [{\"b\": 1}, {\"b\": 2}, {\"b\": 3}]}') "
"as jq_output, "
" jq_output[1].b as first_b from \"t/#\" ",
?assertMatch(
{ok, #{<<"first_b">> := 1}},
emqx_rule_sqltester:test(
#{
sql => Sql2,
context =>
#{
payload => #{<<"what">> => 4},
topic => <<"t/a">>
}
}
)
).
t_sqlselect_01(_Config) -> t_sqlselect_01(_Config) ->
SQL = SQL =
"SELECT json_decode(payload) as p, payload " "SELECT json_decode(payload) as p, payload "

18
apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl
View File

@ -629,6 +629,24 @@ t_regex_replace(_) ->
?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])), ?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])),
?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])). ?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])).
jq_1_elm_res(JSONString) ->
Bin = list_to_binary(JSONString),
[apply_func(json_decode, [Bin])].
t_jq(_) ->
?assertEqual(
jq_1_elm_res("{\"b\":2}"),
apply_func(jq, [<<".">>, apply_func(json_decode, [<<"{\"b\": 2}">>])])
),
?assertEqual(
jq_1_elm_res("6"),
apply_func(jq, [<<".+1">>, apply_func(json_decode, [<<"5">>])])
),
?assertEqual(
jq_1_elm_res("{\"b\":2}"),
apply_func(jq, [<<".">>, <<"{\"b\": 2}">>])
).
ascii_string() -> list(range(0, 127)). ascii_string() -> list(range(0, 127)).
bin(S) -> iolist_to_binary(S). bin(S) -> iolist_to_binary(S).

29
deploy/docker/Dockerfile
View File

@ -1,22 +1,27 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
ARG RUN_FROM=alpine:3.15.1 ARG RUN_FROM=alpine:3.15.1
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder
RUN apk add --no-cache \ RUN apk add --no-cache \
git \ autoconf \
curl \ automake \
gcc \ bash \
g++ \ bison \
make \
perl \
ncurses-dev \
openssl-dev \
coreutils \
bsd-compat-headers \ bsd-compat-headers \
coreutils \
curl \
flex \
g++ \
gcc \
git \
jq \
libc-dev \ libc-dev \
libstdc++ \ libstdc++ \
bash \ libtool \
jq make \
ncurses-dev \
openssl-dev \
perl
COPY . /emqx COPY . /emqx

6
elvis.config
View File

@ -37,6 +37,12 @@
{elvis_style, nesting_level, #{ level => 6 }} {elvis_style, nesting_level, #{ level => 6 }}
] ]
}, },
#{dirs => ["apps/emqx_rule_engine/src"],
filter => "*_rule_funcs.erl",
rules => [
{elvis_style, god_modules, disable}
]
},
#{dirs => ["."], #{dirs => ["."],
filter => "Makefile", filter => "Makefile",
ruleset => makefiles ruleset => makefiles

13
mix.exs
View File

@ -91,7 +91,7 @@ defmodule EMQXUmbrella.MixProject do
github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
# in conflict by grpc and eetcd # in conflict by grpc and eetcd
{:gpb, "4.11.2", override: true, runtime: false} {:gpb, "4.11.2", override: true, runtime: false}
] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep() ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
end end
defp umbrella_apps() do defp umbrella_apps() do
@ -202,6 +202,7 @@ defmodule EMQXUmbrella.MixProject do
] ++ ] ++
if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
if(enable_jq?(), do: [jq: :permanent], else: []) ++
if(edition_type == :enterprise, if(edition_type == :enterprise,
do: [ do: [
emqx_enterprise_conf: :load, emqx_enterprise_conf: :load,
@ -608,6 +609,12 @@ defmodule EMQXUmbrella.MixProject do
else: [] else: []
end end
defp jq_dep() do
if enable_jq?(),
do: [{:jq, github: "emqx/jq", tag: "v0.1.0", override: true}],
else: []
end
defp quicer_dep() do defp quicer_dep() do
if enable_quicer?(), if enable_quicer?(),
# in conflict with emqx and emqtt # in conflict with emqx and emqtt
@ -619,6 +626,10 @@ defmodule EMQXUmbrella.MixProject do
not win32?() not win32?()
end end
defp enable_jq?() do
not win32?()
end
defp enable_quicer?() do defp enable_quicer?() do
not Enum.any?([ not Enum.any?([
build_without_quic?(), build_without_quic?(),

2
pkg-vsn.sh
View File

@ -84,6 +84,8 @@ RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.h
git_exact_vsn() { git_exact_vsn() {
local tag local tag
## Needed to avoid error in github action
git config --global --add safe.directory "/__w/emqx/emqx"
tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)" tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)"
echo "${tag#[v|e]}" echo "${tag#[v|e]}"
} }

559
rebar.config.erl
View File

@ -19,12 +19,17 @@ assert_otp() ->
OtpRelease = list_to_integer(erlang:system_info(otp_release)), OtpRelease = list_to_integer(erlang:system_info(otp_release)),
case OtpRelease < Oldest orelse OtpRelease > Latest of case OtpRelease < Oldest orelse OtpRelease > Latest of
true -> true ->
io:format(standard_error, "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n", io:format(
[OtpRelease, Oldest, Latest]), standard_error,
"ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
[OtpRelease, Oldest, Latest]
),
halt(1); halt(1);
false when OtpRelease =/= Latest -> false when OtpRelease =/= Latest ->
io:format("WARNING: Erlang/OTP version ~p found, recommended==~p~n", io:format(
[OtpRelease, Latest]); "WARNING: Erlang/OTP version ~p found, recommended==~p~n",
[OtpRelease, Latest]
);
false -> false ->
ok ok
end. end.
@ -35,16 +40,19 @@ bcrypt() ->
quicer() -> quicer() ->
{quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}. {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}.
jq() ->
{jq, {git, "https://github.com/emqx/jq", {tag, "v0.1.0"}}}.
deps(Config) -> deps(Config) ->
{deps, OldDeps} = lists:keyfind(deps, 1, Config), {deps, OldDeps} = lists:keyfind(deps, 1, Config),
MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++ MoreDeps =
[bcrypt() || provide_bcrypt_dep()] ++
[jq() || provide_jq()] ++
[quicer() || is_quicer_supported()], [quicer() || is_quicer_supported()],
lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}). lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}).
overrides() -> overrides() ->
[ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]} [{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}] ++ snabbkaffe_overrides().
]}
] ++ snabbkaffe_overrides().
%% Temporary workaround for a rebar3 erl_opts duplication %% Temporary workaround for a rebar3 erl_opts duplication
%% bug. Ideally, we want to set this define globally %% bug. Ideally, we want to set this define globally
@ -53,14 +61,15 @@ snabbkaffe_overrides() ->
[{add, App, [{erl_opts, [{d, snk_kind, msg}]}]} || App <- Apps]. [{add, App, [{erl_opts, [{d, snk_kind, msg}]}]} || App <- Apps].
config() -> config() ->
[ {cover_enabled, is_cover_enabled()} [
, {profiles, profiles()} {cover_enabled, is_cover_enabled()},
, {plugins, plugins()} {profiles, profiles()},
{plugins, plugins()}
]. ].
is_cover_enabled() -> is_cover_enabled() ->
case os:getenv("ENABLE_COVER_COMPILE") of case os:getenv("ENABLE_COVER_COMPILE") of
"1"-> true; "1" -> true;
"true" -> true; "true" -> true;
_ -> false _ -> false
end. end.
@ -70,14 +79,13 @@ is_enterprise(ee) -> true.
is_quicer_supported() -> is_quicer_supported() ->
not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse
is_win32() orelse is_centos_6() is_win32() orelse is_centos_6()).
).
is_centos_6() -> is_centos_6() ->
%% reason: %% reason:
%% glibc is too old %% glibc is too old
case file:read_file("/etc/centos-release") of case file:read_file("/etc/centos-release") of
{ok, <<"CentOS release 6", _/binary >>} -> {ok, <<"CentOS release 6", _/binary>>} ->
true; true;
_ -> _ ->
false false
@ -94,109 +102,118 @@ project_app_dirs(Edition) ->
end. end.
plugins() -> plugins() ->
[ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} [
, {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}} {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.0.0"}}},
{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}},
%% emqx main project does not require port-compiler %% emqx main project does not require port-compiler
%% pin at root level for deterministic %% pin at root level for deterministic
, {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}}
] ] ++
%% test plugins are concatenated to default profile plugins %% test plugins are concatenated to default profile plugins
%% otherwise rebar3 test profile runs are super slow %% otherwise rebar3 test profile runs are super slow
++ test_plugins(). test_plugins().
test_plugins() -> test_plugins() ->
[ {rebar3_proper, "0.12.1"} [
, {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} {rebar3_proper, "0.12.1"},
{coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
]. ].
test_deps() -> test_deps() ->
[ {bbmustache, "1.10.0"} [
, {meck, "0.9.2"} {bbmustache, "1.10.0"},
, {proper, "1.4.0"} {meck, "0.9.2"},
{proper, "1.4.0"}
]. ].
common_compile_opts(Edition, Vsn) -> common_compile_opts(Edition, Vsn) ->
[ debug_info % always include debug_info % always include debug_info
, {compile_info, [{emqx_vsn, Vsn}]} [
, {d, 'EMQX_RELEASE_EDITION', Edition} debug_info,
{compile_info, [{emqx_vsn, Vsn}]},
{d, 'EMQX_RELEASE_EDITION', Edition}
] ++ ] ++
[{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"].
prod_compile_opts(Edition, Vsn) -> prod_compile_opts(Edition, Vsn) ->
[ compressed [
, deterministic compressed,
, warnings_as_errors deterministic,
warnings_as_errors
| common_compile_opts(Edition, Vsn) | common_compile_opts(Edition, Vsn)
]. ].
prod_overrides() -> prod_overrides() ->
[{add, [ {erl_opts, [deterministic]}]}]. [{add, [{erl_opts, [deterministic]}]}].
profiles() -> profiles() ->
profiles_ce() ++ profiles_ee() ++ profiles_dev(). profiles_ce() ++ profiles_ee() ++ profiles_dev().
profiles_ce() -> profiles_ce() ->
Vsn = get_vsn(emqx), Vsn = get_vsn(emqx),
[ {'emqx', [
[ {erl_opts, prod_compile_opts(ce, Vsn)} {'emqx', [
, {relx, relx(Vsn, cloud, bin, ce)} {erl_opts, prod_compile_opts(ce, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, cloud, bin, ce)},
, {project_app_dirs, project_app_dirs(ce)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx doc"}]} {project_app_dirs, project_app_dirs(ce)},
]} {post_hooks, [{compile, "bash build emqx doc"}]}
, {'emqx-pkg', ]},
[ {erl_opts, prod_compile_opts(ce, Vsn)} {'emqx-pkg', [
, {relx, relx(Vsn, cloud, pkg, ce)} {erl_opts, prod_compile_opts(ce, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, cloud, pkg, ce)},
, {project_app_dirs, project_app_dirs(ce)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx-pkg doc"}]} {project_app_dirs, project_app_dirs(ce)},
]} {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
, {'emqx-edge', ]},
[ {erl_opts, prod_compile_opts(edge, Vsn)} {'emqx-edge', [
, {relx, relx(Vsn, edge, bin, ce)} {erl_opts, prod_compile_opts(edge, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, edge, bin, ce)},
, {project_app_dirs, project_app_dirs(ce)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx-edge doc"}]} {project_app_dirs, project_app_dirs(ce)},
]} {post_hooks, [{compile, "bash build emqx-edge doc"}]}
, {'emqx-edge-pkg', ]},
[ {erl_opts, prod_compile_opts(edge, Vsn)} {'emqx-edge-pkg', [
, {relx, relx(Vsn, edge, pkg, ce)} {erl_opts, prod_compile_opts(edge, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, edge, pkg, ce)},
, {project_app_dirs, project_app_dirs(ce)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]} {project_app_dirs, project_app_dirs(ce)},
{post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
]} ]}
]. ].
profiles_ee() -> profiles_ee() ->
Vsn = get_vsn('emqx-enterprise'), Vsn = get_vsn('emqx-enterprise'),
[ {'emqx-enterprise', [
[ {erl_opts, prod_compile_opts(ee, Vsn)} {'emqx-enterprise', [
, {relx, relx(Vsn, cloud, bin, ee)} {erl_opts, prod_compile_opts(ee, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, cloud, bin, ee)},
, {project_app_dirs, project_app_dirs(ee)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} {project_app_dirs, project_app_dirs(ee)},
]} {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
, {'emqx-enterprise-pkg', ]},
[ {erl_opts, prod_compile_opts(ee, Vsn)} {'emqx-enterprise-pkg', [
, {relx, relx(Vsn, cloud, pkg, ee)} {erl_opts, prod_compile_opts(ee, Vsn)},
, {overrides, prod_overrides()} {relx, relx(Vsn, cloud, pkg, ee)},
, {project_app_dirs, project_app_dirs(ee)} {overrides, prod_overrides()},
, {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} {project_app_dirs, project_app_dirs(ee)},
{post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
]} ]}
]. ].
%% EE has more files than CE, always test/check with EE options. %% EE has more files than CE, always test/check with EE options.
profiles_dev() -> profiles_dev() ->
Vsn = get_vsn('emqx-enterprise'), Vsn = get_vsn('emqx-enterprise'),
[ {check, [
[ {erl_opts, common_compile_opts(ee, Vsn)} {check, [
, {project_app_dirs, project_app_dirs(ee)} {erl_opts, common_compile_opts(ee, Vsn)},
]} {project_app_dirs, project_app_dirs(ee)}
, {test, ]},
[ {deps, test_deps()} {test, [
, {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()} {deps, test_deps()},
, {extra_src_dirs, [{"test", [{recursive, true}]}]} {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()},
, {project_app_dirs, project_app_dirs(ee)} {extra_src_dirs, [{"test", [{recursive, true}]}]},
{project_app_dirs, project_app_dirs(ee)}
]} ]}
]. ].
@ -204,16 +221,19 @@ profiles_dev() ->
%% PkgType: bin | pkg %% PkgType: bin | pkg
%% Edition: ce (community) | ee (enterprise) %% Edition: ce (community) | ee (enterprise)
relx(Vsn, RelType, PkgType, Edition) -> relx(Vsn, RelType, PkgType, Edition) ->
[ {include_src,false} [
, {include_erts, true} {include_src, false},
, {extended_start_script,false} {include_erts, true},
, {generate_start_script,false} {extended_start_script, false},
, {sys_config,false} {generate_start_script, false},
, {vm_args,false} {sys_config, false},
, {release, {emqx, Vsn}, relx_apps(RelType, Edition)} {vm_args, false},
, {overlay, relx_overlay(RelType, Edition)} {release, {emqx, Vsn}, relx_apps(RelType, Edition)},
, {overlay_vars, build_info() ++ {overlay, relx_overlay(RelType, Edition)},
[ {emqx_description, emqx_description(RelType, Edition)} {overlay_vars,
build_info() ++
[
{emqx_description, emqx_description(RelType, Edition)}
| overlay_vars(RelType, PkgType, Edition) | overlay_vars(RelType, PkgType, Edition)
]} ]}
]. ].
@ -221,12 +241,13 @@ relx(Vsn, RelType, PkgType, Edition) ->
%% Make a HOCON compatible format %% Make a HOCON compatible format
build_info() -> build_info() ->
Os = os_cmd("./scripts/get-distro.sh"), Os = os_cmd("./scripts/get-distro.sh"),
[ {build_info_arch, erlang:system_info(system_architecture)} [
, {build_info_wordsize, rebar_utils:wordsize()} {build_info_arch, erlang:system_info(system_architecture)},
, {build_info_os, Os} {build_info_wordsize, rebar_utils:wordsize()},
, {build_info_erlang, rebar_utils:otp_release()} {build_info_os, Os},
, {build_info_elixir, none} {build_info_erlang, rebar_utils:otp_release()},
, {build_info_relform, relform()} {build_info_elixir, none},
{build_info_relform, relform()}
]. ].
relform() -> relform() ->
@ -240,107 +261,116 @@ emqx_description(cloud, ce) -> "EMQX";
emqx_description(edge, ce) -> "EMQX Edge". emqx_description(edge, ce) -> "EMQX Edge".
overlay_vars(RelType, PkgType, Edition) -> overlay_vars(RelType, PkgType, Edition) ->
overlay_vars_rel(RelType) overlay_vars_rel(RelType) ++
++ overlay_vars_pkg(PkgType) overlay_vars_pkg(PkgType) ++
++ overlay_vars_edition(Edition). overlay_vars_edition(Edition).
%% vars per release type, cloud or edge %% vars per release type, cloud or edge
overlay_vars_rel(RelType) -> overlay_vars_rel(RelType) ->
VmArgs = case RelType of VmArgs =
case RelType of
cloud -> "vm.args"; cloud -> "vm.args";
edge -> "vm.args.edge" edge -> "vm.args.edge"
end, end,
[ {vm_args_file, VmArgs} [{vm_args_file, VmArgs}].
].
overlay_vars_edition(ce) -> overlay_vars_edition(ce) ->
[ {emqx_schema_mod, emqx_conf_schema} [
, {is_enterprise, "no"} {emqx_schema_mod, emqx_conf_schema},
, {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)} {is_enterprise, "no"},
{emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)}
]; ];
overlay_vars_edition(ee) -> overlay_vars_edition(ee) ->
[ {emqx_schema_mod, emqx_enterprise_conf_schema} [
, {is_enterprise, "yes"} {emqx_schema_mod, emqx_enterprise_conf_schema},
, {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)} {is_enterprise, "yes"},
{emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)}
]. ].
%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
overlay_vars_pkg(bin) -> overlay_vars_pkg(bin) ->
[ {platform_data_dir, "data"} [
, {platform_etc_dir, "etc"} {platform_data_dir, "data"},
, {platform_log_dir, "log"} {platform_etc_dir, "etc"},
, {platform_plugins_dir, "plugins"} {platform_log_dir, "log"},
, {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} {platform_plugins_dir, "plugins"},
, {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"} {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"},
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"},
, {runner_log_dir, "$RUNNER_ROOT_DIR/log"} {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
, {runner_user, ""} {runner_log_dir, "$RUNNER_ROOT_DIR/log"},
, {is_elixir, "no"} {runner_user, ""},
{is_elixir, "no"}
]; ];
overlay_vars_pkg(pkg) -> overlay_vars_pkg(pkg) ->
[ {platform_data_dir, "/var/lib/emqx"} [
, {platform_etc_dir, "/etc/emqx"} {platform_data_dir, "/var/lib/emqx"},
, {platform_log_dir, "/var/log/emqx"} {platform_etc_dir, "/etc/emqx"},
, {platform_plugins_dir, "/var/lib/emqx/plugins"} {platform_log_dir, "/var/log/emqx"},
, {runner_bin_dir, "/usr/bin"} {platform_plugins_dir, "/var/lib/emqx/plugins"},
, {emqx_etc_dir, "/etc/emqx"} {runner_bin_dir, "/usr/bin"},
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} {emqx_etc_dir, "/etc/emqx"},
, {runner_log_dir, "/var/log/emqx"} {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
, {runner_user, "emqx"} {runner_log_dir, "/var/log/emqx"},
, {is_elixir, "no"} {runner_user, "emqx"},
{is_elixir, "no"}
]. ].
relx_apps(ReleaseType, Edition) -> relx_apps(ReleaseType, Edition) ->
[ kernel [
, sasl kernel,
, crypto sasl,
, public_key crypto,
, asn1 public_key,
, syntax_tools asn1,
, ssl syntax_tools,
, os_mon ssl,
, inets os_mon,
, compiler inets,
, runtime_tools compiler,
, redbug runtime_tools,
, {hocon, load} redbug,
, {emqx, load} % started by emqx_machine {hocon, load},
, {emqx_conf, load} % started by emqx_machine
, emqx_machine {emqx, load},
, {mnesia, load} {emqx_conf, load},
, {ekka, load} emqx_machine,
, {emqx_plugin_libs, load} {mnesia, load},
, {esasl, load} {ekka, load},
, observer_cli {emqx_plugin_libs, load},
, {system_monitor, load} % started by emqx_machine {esasl, load},
, emqx_http_lib observer_cli,
, emqx_resource % started by emqx_machine
, emqx_connector {system_monitor, load},
, emqx_authn emqx_http_lib,
, emqx_authz emqx_resource,
, emqx_auto_subscribe emqx_connector,
, emqx_gateway emqx_authn,
, emqx_exhook emqx_authz,
, emqx_bridge emqx_auto_subscribe,
, emqx_rule_engine emqx_gateway,
, emqx_modules emqx_exhook,
, emqx_management emqx_bridge,
, emqx_dashboard emqx_rule_engine,
, emqx_retainer emqx_modules,
, emqx_statsd emqx_management,
, emqx_prometheus emqx_dashboard,
, emqx_psk emqx_retainer,
, emqx_slow_subs emqx_statsd,
, emqx_plugins emqx_prometheus,
] emqx_psk,
++ [quicer || is_quicer_supported()] emqx_slow_subs,
++ [bcrypt || provide_bcrypt_release(ReleaseType)] emqx_plugins
++ relx_apps_per_rel(ReleaseType) ] ++
++ relx_additional_apps(ReleaseType, Edition). [quicer || is_quicer_supported()] ++
[bcrypt || provide_bcrypt_release(ReleaseType)] ++
[jq || provide_jq()] ++
relx_apps_per_rel(ReleaseType) ++
relx_additional_apps(ReleaseType, Edition).
relx_apps_per_rel(cloud) -> relx_apps_per_rel(cloud) ->
[ xmerl [
xmerl
| [{observer, load} || is_app(observer)] | [{observer, load} || is_app(observer)]
]; ];
relx_apps_per_rel(edge) -> relx_apps_per_rel(edge) ->
@ -349,13 +379,13 @@ relx_apps_per_rel(edge) ->
is_app(Name) -> is_app(Name) ->
case application:load(Name) of case application:load(Name) of
ok -> true; ok -> true;
{error,{already_loaded, _}} -> true; {error, {already_loaded, _}} -> true;
_ -> false _ -> false
end. end.
relx_additional_apps(ReleaseType, Edition) -> relx_additional_apps(ReleaseType, Edition) ->
relx_plugin_apps_per_rel(ReleaseType) relx_plugin_apps_per_rel(ReleaseType) ++
++ relx_apps_per_edition(Edition). relx_apps_per_edition(Edition).
relx_plugin_apps_per_rel(cloud) -> relx_plugin_apps_per_rel(cloud) ->
[]; [];
@ -363,33 +393,34 @@ relx_plugin_apps_per_rel(edge) ->
[]. [].
relx_apps_per_edition(ee) -> relx_apps_per_edition(ee) ->
[ emqx_license [
, {emqx_enterprise_conf, load} emqx_license,
{emqx_enterprise_conf, load}
]; ];
relx_apps_per_edition(ce) ->
relx_apps_per_edition(ce) -> []. [].
emqx_machine_boot_apps(ce) -> emqx_machine_boot_apps(ce) ->
[ emqx_prometheus [
, emqx_modules emqx_prometheus,
, emqx_dashboard emqx_modules,
, emqx_connector emqx_dashboard,
, emqx_gateway emqx_connector,
, emqx_statsd emqx_gateway,
, emqx_resource emqx_statsd,
, emqx_rule_engine emqx_resource,
, emqx_bridge emqx_rule_engine,
, emqx_plugin_libs emqx_bridge,
, emqx_management emqx_plugin_libs,
, emqx_retainer emqx_management,
, emqx_exhook emqx_retainer,
, emqx_authn emqx_exhook,
, emqx_authz emqx_authn,
, emqx_slow_subs emqx_authz,
, emqx_auto_subscribe emqx_slow_subs,
, emqx_plugins emqx_auto_subscribe,
emqx_plugins
]; ];
emqx_machine_boot_apps(ee) -> emqx_machine_boot_apps(ee) ->
emqx_machine_boot_apps(ce) ++ emqx_machine_boot_apps(ce) ++
[]. [].
@ -397,67 +428,75 @@ emqx_machine_boot_apps(ee) ->
emqx_machine_boot_app_list(Edition) -> emqx_machine_boot_app_list(Edition) ->
string:join( string:join(
[atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)], [atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)],
", "). ", "
).
relx_overlay(ReleaseType, Edition) -> relx_overlay(ReleaseType, Edition) ->
[ {mkdir, "log/"} [
, {mkdir, "data/"} {mkdir, "log/"},
, {mkdir, "plugins"} {mkdir, "data/"},
, {mkdir, "data/mnesia"} {mkdir, "plugins"},
, {mkdir, "data/configs"} {mkdir, "data/mnesia"},
, {mkdir, "data/patches"} {mkdir, "data/configs"},
, {mkdir, "data/scripts"} {mkdir, "data/patches"},
, {template, "rel/emqx_vars", "releases/emqx_vars"} {mkdir, "data/scripts"},
, {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"} {template, "rel/emqx_vars", "releases/emqx_vars"},
, {copy, "bin/emqx", "bin/emqx"} {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"},
, {copy, "bin/emqx_ctl", "bin/emqx_ctl"} {copy, "bin/emqx", "bin/emqx"},
, {copy, "bin/node_dump", "bin/node_dump"} {copy, "bin/emqx_ctl", "bin/emqx_ctl"},
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"} {copy, "bin/node_dump", "bin/node_dump"},
, {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"},
, {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup %% for relup
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup {copy, "bin/emqx", "bin/emqx-{{release_version}}"},
, {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"} %% for relup
, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"} {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"},
, {template, "bin/emqx.cmd", "bin/emqx.cmd"} %% for relup
, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"},
, {copy, "bin/nodetool", "bin/nodetool"} {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"},
, {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"},
{template, "bin/emqx.cmd", "bin/emqx.cmd"},
{template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"},
{copy, "bin/nodetool", "bin/nodetool"},
{copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
] ++ etc_overlay(ReleaseType, Edition). ] ++ etc_overlay(ReleaseType, Edition).
etc_overlay(ReleaseType, Edition) -> etc_overlay(ReleaseType, Edition) ->
Templates = emqx_etc_overlay(ReleaseType, Edition), Templates = emqx_etc_overlay(ReleaseType, Edition),
[ {mkdir, "etc/"} [
, {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"} {mkdir, "etc/"},
{copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}
] ++ ] ++
lists:map( lists:map(
fun({From, To}) -> {template, From, To}; fun
({From, To}) -> {template, From, To};
(FromTo) -> {template, FromTo, FromTo} (FromTo) -> {template, FromTo, FromTo}
end, Templates). end,
Templates
).
emqx_etc_overlay(ReleaseType, Edition) -> emqx_etc_overlay(ReleaseType, Edition) ->
emqx_etc_overlay_per_rel(ReleaseType) emqx_etc_overlay_per_rel(ReleaseType) ++
++ emqx_etc_overlay_per_edition(Edition) emqx_etc_overlay_per_edition(Edition) ++
++ emqx_etc_overlay_common(). emqx_etc_overlay_common().
emqx_etc_overlay_per_rel(cloud) -> emqx_etc_overlay_per_rel(cloud) ->
[ {"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args","etc/vm.args"} [{"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args", "etc/vm.args"}];
];
emqx_etc_overlay_per_rel(edge) -> emqx_etc_overlay_per_rel(edge) ->
[ {"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args","etc/vm.args"} [{"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args", "etc/vm.args"}].
].
emqx_etc_overlay_common() -> emqx_etc_overlay_common() ->
[ {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"} [{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}].
].
emqx_etc_overlay_per_edition(ce) -> emqx_etc_overlay_per_edition(ce) ->
[ {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} [
, {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
{"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
]; ];
emqx_etc_overlay_per_edition(ee) -> emqx_etc_overlay_per_edition(ee) ->
[ {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"} [
, {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"} {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"},
, {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"} {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
{"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
]. ].
get_vsn(Profile) -> get_vsn(Profile) ->
@ -468,10 +507,11 @@ get_vsn(Profile) ->
os_cmd(Cmd) -> os_cmd(Cmd) ->
Output = os:cmd("bash " ++ Cmd), Output = os:cmd("bash " ++ Cmd),
re:replace(Output, "\n", "", [{return ,list}]). re:replace(Output, "\n", "", [{return, list}]).
maybe_dump(Config) -> maybe_dump(Config) ->
is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]), is_debug() andalso
file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
Config. Config.
is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC"). is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC").
@ -486,6 +526,9 @@ is_debug(VarName) ->
provide_bcrypt_dep() -> provide_bcrypt_dep() ->
not is_win32(). not is_win32().
provide_jq() ->
not is_win32().
provide_bcrypt_release(ReleaseType) -> provide_bcrypt_release(ReleaseType) ->
provide_bcrypt_dep() andalso ReleaseType =:= cloud. provide_bcrypt_dep() andalso ReleaseType =:= cloud.
@ -497,11 +540,12 @@ erl_opts_i() ->
dialyzer(Config) -> dialyzer(Config) ->
{dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of AppsToAnalyse =
case os:getenv("DIALYZER_ANALYSE_APP") of
false -> false ->
[]; [];
Value -> Value ->
[ list_to_atom(App) || App <- string:tokens(Value, ",")] [list_to_atom(App) || App <- string:tokens(Value, ",")]
end, end,
AppNames = app_names(), AppNames = app_names(),
@ -512,7 +556,12 @@ dialyzer(Config) ->
case length(AppsToAnalyse) > 0 of case length(AppsToAnalyse) > 0 of
true -> true ->
lists:keystore(dialyzer, 1, Config, {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}); lists:keystore(
dialyzer,
1,
Config,
{dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}
);
false -> false ->
Config Config
end. end.
@ -520,13 +569,17 @@ dialyzer(Config) ->
coveralls() -> coveralls() ->
case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of
{"true", Token} when is_list(Token) -> {"true", Token} when is_list(Token) ->
Cfgs = [{coveralls_repo_token, Token}, Cfgs = [
{coveralls_repo_token, Token},
{coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")}, {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")},
{coveralls_commit_sha, os:getenv("GITHUB_SHA")}, {coveralls_commit_sha, os:getenv("GITHUB_SHA")},
{coveralls_coverdata, "_build/test/cover/*.coverdata"}, {coveralls_coverdata, "_build/test/cover/*.coverdata"},
{coveralls_service_name, "github"}], {coveralls_service_name, "github"}
case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" ],
andalso string:tokens(os:getenv("GITHUB_REF"), "/") of case
os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso
string:tokens(os:getenv("GITHUB_REF"), "/")
of
[_, "pull", PRNO, _] -> [_, "pull", PRNO, _] ->
[{coveralls_service_pull_request, PRNO} | Cfgs]; [{coveralls_service_pull_request, PRNO} | Cfgs];
_ -> _ ->

11
scripts/buildx.sh
View File

@ -9,7 +9,7 @@
## example: ## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10 ## --builder ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10
set -euo pipefail set -euo pipefail
@ -24,7 +24,7 @@ help() {
echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD" echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull" echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10" echo " E.g. ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10"
} }
while [ "$#" -gt 0 ]; do while [ "$#" -gt 0 ]; do
@ -120,13 +120,6 @@ fi
HOST_SYSTEM="$(./scripts/get-distro.sh)" HOST_SYSTEM="$(./scripts/get-distro.sh)"
BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')" BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')"
# quick workaround before builder image is renamed
if [ "$BUILDER_SYSTEM" = 'centos7' ]; then
BUILDER_SYSTEM='el7'
elif [ "$BUILDER_SYSTEM" = 'rockylinux8' ]; then
BUILDER_SYSTEM='el8'
fi
CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
IS_NATIVE_SYSTEM='no' IS_NATIVE_SYSTEM='no'