feat(rule_engine): add jq function to the rule engine
This commit adds a function to the rule engine that alows users to transform text or JSON objects using [jq filter programs][1]. [jq][1] is a command line tool that can be used to transform and filter JSON text using jq's built-in language. The rule engine function that is added with this commit uses the [Erlang jq NIF library][2] that wraps the jq C library in an Erlang NIF function. [1]: https://stedolan.github.io/jq/ [2]: https://github.com/emqx/jq
This commit is contained in:
Kjell Winblad
parent
8ff552c9cf
commit
74c33cd4e5
|
|
@ -3,7 +3,7 @@ version: '3.9'
|
|||
services:
|
||||
erlang23:
|
||||
container_name: erlang23
|
||||
image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-23.3.4.9-4-ubuntu20.04
|
||||
image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-23.3.4.9-4-ubuntu20.04
|
||||
env_file:
|
||||
- conf.env
|
||||
environment:
|
||||
|
|
@ -23,7 +23,7 @@ services:
|
|||
|
||||
erlang24:
|
||||
container_name: erlang24
|
||||
image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
|
||||
image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
|
||||
env_file:
|
||||
- conf.env
|
||||
environment:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ jobs:
|
|||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Check apps version
|
||||
run: ./scripts/apps-version-check.sh
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ jobs:
|
|||
prepare:
|
||||
runs-on: ubuntu-20.04
|
||||
# prepare source with any OTP version, no need for a matrix
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
|
||||
outputs:
|
||||
BUILD_PROFILES: ${{ steps.get_profiles.outputs.BUILD_PROFILES }}
|
||||
|
|
@ -139,7 +139,8 @@ jobs:
|
|||
- name: prepare
|
||||
run: |
|
||||
brew update
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
|
||||
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
|
||||
echo "/usr/local/bin" >> $GITHUB_PATH
|
||||
git config --global credential.helper store
|
||||
- uses: actions/cache@v2
|
||||
|
|
@ -205,7 +206,7 @@ jobs:
|
|||
needs: prepare
|
||||
runs-on: ${{ matrix.build_machine }}
|
||||
container:
|
||||
image: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
image: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
|
@ -234,8 +235,8 @@ jobs:
|
|||
- debian11
|
||||
- debian10
|
||||
- debian9
|
||||
- rockylinux8
|
||||
- centos7
|
||||
- el8
|
||||
- el7
|
||||
- raspbian10
|
||||
build_machine:
|
||||
- aws-arm64
|
||||
|
|
@ -272,7 +273,7 @@ jobs:
|
|||
elixir: 1.13.3
|
||||
build_elixir: with_elixir
|
||||
arch: amd64
|
||||
os: centos7
|
||||
os: el7
|
||||
build_machine: ubuntu-20.04
|
||||
|
||||
defaults:
|
||||
|
|
@ -325,7 +326,7 @@ jobs:
|
|||
--pkgtype "${PKGTYPE}" \
|
||||
--arch "${ARCH}" \
|
||||
--elixir "${IsElixir}" \
|
||||
--builder "ghcr.io/emqx/emqx-builder/5.0-10:${ELIXIR}-${OTP}-${SYSTEM}"
|
||||
--builder "ghcr.io/emqx/emqx-builder/5.0-14:${ELIXIR}-${OTP}-${SYSTEM}"
|
||||
done
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
|
|
@ -462,7 +463,7 @@ jobs:
|
|||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
RUN_FROM=${{ steps.pre-meta.outputs.img }}
|
||||
EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }}
|
||||
file: source/deploy/docker/Dockerfile
|
||||
|
|
|
|||
|
|
@ -38,15 +38,15 @@ jobs:
|
|||
- 1.13.3
|
||||
os:
|
||||
- ubuntu20.04
|
||||
- rockylinux8
|
||||
- el8
|
||||
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
|
||||
steps:
|
||||
- name: cleanup
|
||||
run: |
|
||||
rm -rf "${GITHUB_WORKSPACE}/"
|
||||
- uses: actions/checkout@v1
|
||||
- uses: AutoModality/action-clean@v1
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: prepare
|
||||
run: |
|
||||
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
|
||||
|
|
@ -63,6 +63,9 @@ jobs:
|
|||
_build/default/lib/quicer/
|
||||
deps/quicer/
|
||||
key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
|
||||
- name: Work around https://github.com/actions/checkout/issues/766
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
- name: build and test tgz package
|
||||
run: |
|
||||
make ${EMQX_NAME}-tgz
|
||||
|
|
@ -148,7 +151,8 @@ jobs:
|
|||
- name: prepare
|
||||
run: |
|
||||
brew update
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds
|
||||
brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
|
||||
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
|
||||
echo "/usr/local/bin" >> $GITHUB_PATH
|
||||
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
|
||||
- uses: actions/cache@v2
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ on: [pull_request, push]
|
|||
jobs:
|
||||
check_deps_integrity:
|
||||
runs-on: ubuntu-20.04
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ on: [pull_request]
|
|||
jobs:
|
||||
code_style_check:
|
||||
runs-on: ubuntu-20.04
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
|
|
@ -13,6 +13,9 @@ jobs:
|
|||
- name: Check line-break at EOF
|
||||
run: |
|
||||
./scripts/check-nl-at-eof.sh
|
||||
- name: Work around https://github.com/actions/checkout/issues/766
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
- name: Check Elixir code formatting
|
||||
run: |
|
||||
mix format --check-formatted
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ on: [pull_request, push]
|
|||
|
||||
jobs:
|
||||
elixir_apps_check:
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-latest
|
||||
# just use the latest builder
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
|
@ -31,12 +31,17 @@ jobs:
|
|||
edition_type: enterprise
|
||||
|
||||
steps:
|
||||
- name: fix_git_permission
|
||||
run: git config --global --add safe.directory '/__w/emqx/emqx'
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2.4.0
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: ensure rebar
|
||||
run: ./scripts/ensure-rebar3.sh
|
||||
- name: Work around https://github.com/actions/checkout/issues/766
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
- name: check applications
|
||||
run: ./scripts/check-elixir-applications.exs
|
||||
- name: check applications started with emqx_machine
|
||||
|
|
|
|||
|
|
@ -7,13 +7,16 @@ on: [pull_request, push]
|
|||
jobs:
|
||||
elixir_deps_check:
|
||||
runs-on: ubuntu-20.04
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2.4.0
|
||||
uses: actions/checkout@v2
|
||||
- name: ensure rebar
|
||||
run: ./scripts/ensure-rebar3.sh
|
||||
- name: Work around https://github.com/actions/checkout/issues/766
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
- name: setup mix
|
||||
run: |
|
||||
mix local.hex --force
|
||||
|
|
|
|||
|
|
@ -12,13 +12,16 @@ on:
|
|||
jobs:
|
||||
elixir_release_build:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2.4.0
|
||||
uses: actions/checkout@v2
|
||||
- name: install tools
|
||||
run: apt update && apt install netcat-openbsd
|
||||
- name: Work around https://github.com/actions/checkout/issues/766
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
- name: elixir release
|
||||
run: make emqx-elixir
|
||||
- name: start release
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
- amd64
|
||||
|
||||
runs-on: aws-amd64
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
|
@ -47,14 +47,19 @@ jobs:
|
|||
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
|
||||
- name: run
|
||||
run: |
|
||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
echo "git diff base: $GITHUB_BASE_REF"
|
||||
if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then
|
||||
# base is a commit sha1
|
||||
compare_base="$GITHUB_BASE_REF"
|
||||
else
|
||||
compare_base="origin/$GITHUB_BASE_REF"
|
||||
repo="${GITHUB_REPOSITORY}"
|
||||
git remote -v
|
||||
remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')"
|
||||
git fetch "$remote" "$GITHUB_BASE_REF"
|
||||
compare_base="$remote/$GITHUB_BASE_REF"
|
||||
fi
|
||||
changed_files="$(git diff --name-only ${compare_base}...HEAD apps/emqx)"
|
||||
changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)"
|
||||
if [ "$changed_files" = '' ]; then
|
||||
echo "nothing changed in apps/emqx, ignored."
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ jobs:
|
|||
prepare:
|
||||
runs-on: ubuntu-20.04
|
||||
# prepare source with any OTP version, no need for a matrix
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
|
||||
container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -84,7 +84,7 @@ jobs:
|
|||
- name: make docker image
|
||||
working-directory: source
|
||||
env:
|
||||
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
run: |
|
||||
make ${{ matrix.profile }}-docker
|
||||
- name: run emqx
|
||||
|
|
@ -162,7 +162,7 @@ jobs:
|
|||
- name: make docker image
|
||||
working-directory: source
|
||||
env:
|
||||
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
|
||||
run: |
|
||||
make ${{ matrix.profile }}-docker
|
||||
echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ jobs:
|
|||
- amd64
|
||||
|
||||
runs-on: ubuntu-20.04
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
|
||||
defaults:
|
||||
run:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ jobs:
|
|||
- amd64
|
||||
|
||||
runs-on: aws-amd64
|
||||
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
|
@ -58,7 +58,7 @@ jobs:
|
|||
|
||||
find_apps:
|
||||
runs-on: aws-amd64
|
||||
container: emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
|
||||
container: emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
|
||||
outputs:
|
||||
fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
|
||||
docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
|
||||
|
|
@ -114,7 +114,7 @@ jobs:
|
|||
# produces <app-name>.coverdata
|
||||
- name: run common test
|
||||
run: |
|
||||
docker exec -i ${{ matrix.otp_release }} bash -c "make ${{ matrix.app_name }}-ct"
|
||||
docker exec -i ${{ matrix.otp_release }} bash -c "git config --global --add safe.directory \"$GITHUB_WORKSPACE\" && make ${{ matrix.app_name }}-ct"
|
||||
- uses: actions/upload-artifact@v1
|
||||
if: matrix.otp_release == 'erlang24'
|
||||
with:
|
||||
|
|
@ -141,7 +141,7 @@ jobs:
|
|||
arch:
|
||||
- amd64
|
||||
runs-on: aws-amd64
|
||||
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
|
@ -180,7 +180,7 @@ jobs:
|
|||
- amd64
|
||||
|
||||
runs-on: aws-amd64
|
||||
container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
|
|||
BUILD = $(CURDIR)/build
|
||||
SCRIPTS = $(CURDIR)/scripts
|
||||
export EMQX_RELUP ?= true
|
||||
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
|
||||
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
|
||||
export EMQX_DEFAULT_RUNNER = alpine:3.15.1
|
||||
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
|
||||
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
%% -*- mode: erlang -*-
|
||||
|
||||
{deps, [{emqx, {path, "../emqx"}}]}.
|
||||
{deps, [
|
||||
{emqx, {path, "../emqx"}}
|
||||
]}.
|
||||
|
||||
{erl_opts, [
|
||||
warn_unused_vars,
|
||||
|
|
|
|||
|
|
@ -147,7 +147,8 @@
|
|||
regex_replace/3,
|
||||
ascii/1,
|
||||
find/2,
|
||||
find/3
|
||||
find/3,
|
||||
jq/2
|
||||
]).
|
||||
|
||||
%% Map Funcs
|
||||
|
|
@ -779,6 +780,23 @@ find_s(S, P, Dir) ->
|
|||
SubStr -> SubStr
|
||||
end.
|
||||
|
||||
-spec jq(FilterProgram, JSON) -> Result when
|
||||
FilterProgram :: binary(),
|
||||
JSON :: binary() | term(),
|
||||
Result :: [term()].
|
||||
jq(FilterProgram, JSONBin) when
|
||||
is_binary(FilterProgram), is_binary(JSONBin)
|
||||
->
|
||||
case jq:parse(FilterProgram, JSONBin) of
|
||||
{ok, Result} ->
|
||||
[json_decode(JSONString) || JSONString <- Result];
|
||||
{error, ErrorReason} ->
|
||||
erlang:throw({jq_exception, ErrorReason})
|
||||
end;
|
||||
jq(FilterProgram, JSONTerm) when is_binary(FilterProgram) ->
|
||||
JSONBin = json_encode(JSONTerm),
|
||||
jq(FilterProgram, JSONBin).
|
||||
|
||||
%%------------------------------------------------------------------------------
|
||||
%% Array Funcs
|
||||
%%------------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ groups() ->
|
|||
t_match_atom_and_binary,
|
||||
t_sqlselect_0,
|
||||
t_sqlselect_00,
|
||||
t_sqlselect_001,
|
||||
t_sqlselect_01,
|
||||
t_sqlselect_02,
|
||||
t_sqlselect_1,
|
||||
|
|
@ -728,6 +729,43 @@ t_sqlselect_00(_Config) ->
|
|||
)
|
||||
).
|
||||
|
||||
t_sqlselect_001(_Config) ->
|
||||
%% Verify that the jq function can be called from SQL
|
||||
Sql =
|
||||
"select jq('.what + .what', payload) as ans "
|
||||
"from \"t/#\" ",
|
||||
?assertMatch(
|
||||
{ok, #{<<"ans">> := [8]}},
|
||||
emqx_rule_sqltester:test(
|
||||
#{
|
||||
sql => Sql,
|
||||
context =>
|
||||
#{
|
||||
payload => #{<<"what">> => 4},
|
||||
topic => <<"t/a">>
|
||||
}
|
||||
}
|
||||
)
|
||||
),
|
||||
Sql2 =
|
||||
"SELECT jq('.a|.[]', "
|
||||
"'{\"a\": [{\"b\": 1}, {\"b\": 2}, {\"b\": 3}]}') "
|
||||
"as jq_output, "
|
||||
" jq_output[1].b as first_b from \"t/#\" ",
|
||||
?assertMatch(
|
||||
{ok, #{<<"first_b">> := 1}},
|
||||
emqx_rule_sqltester:test(
|
||||
#{
|
||||
sql => Sql2,
|
||||
context =>
|
||||
#{
|
||||
payload => #{<<"what">> => 4},
|
||||
topic => <<"t/a">>
|
||||
}
|
||||
}
|
||||
)
|
||||
).
|
||||
|
||||
t_sqlselect_01(_Config) ->
|
||||
SQL =
|
||||
"SELECT json_decode(payload) as p, payload "
|
||||
|
|
|
|||
|
|
@ -629,6 +629,24 @@ t_regex_replace(_) ->
|
|||
?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])),
|
||||
?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])).
|
||||
|
||||
jq_1_elm_res(JSONString) ->
|
||||
Bin = list_to_binary(JSONString),
|
||||
[apply_func(json_decode, [Bin])].
|
||||
|
||||
t_jq(_) ->
|
||||
?assertEqual(
|
||||
jq_1_elm_res("{\"b\":2}"),
|
||||
apply_func(jq, [<<".">>, apply_func(json_decode, [<<"{\"b\": 2}">>])])
|
||||
),
|
||||
?assertEqual(
|
||||
jq_1_elm_res("6"),
|
||||
apply_func(jq, [<<".+1">>, apply_func(json_decode, [<<"5">>])])
|
||||
),
|
||||
?assertEqual(
|
||||
jq_1_elm_res("{\"b\":2}"),
|
||||
apply_func(jq, [<<".">>, <<"{\"b\": 2}">>])
|
||||
).
|
||||
|
||||
ascii_string() -> list(range(0, 127)).
|
||||
|
||||
bin(S) -> iolist_to_binary(S).
|
||||
|
|
|
|||
|
|
@ -1,22 +1,27 @@
|
|||
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
|
||||
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
|
||||
ARG RUN_FROM=alpine:3.15.1
|
||||
FROM ${BUILD_FROM} AS builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
git \
|
||||
curl \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
perl \
|
||||
ncurses-dev \
|
||||
openssl-dev \
|
||||
coreutils \
|
||||
autoconf \
|
||||
automake \
|
||||
bash \
|
||||
bison \
|
||||
bsd-compat-headers \
|
||||
coreutils \
|
||||
curl \
|
||||
flex \
|
||||
g++ \
|
||||
gcc \
|
||||
git \
|
||||
jq \
|
||||
libc-dev \
|
||||
libstdc++ \
|
||||
bash \
|
||||
jq
|
||||
libtool \
|
||||
make \
|
||||
ncurses-dev \
|
||||
openssl-dev \
|
||||
perl
|
||||
|
||||
COPY . /emqx
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,12 @@
|
|||
{elvis_style, nesting_level, #{ level => 6 }}
|
||||
]
|
||||
},
|
||||
#{dirs => ["apps/emqx_rule_engine/src"],
|
||||
filter => "*_rule_funcs.erl",
|
||||
rules => [
|
||||
{elvis_style, god_modules, disable}
|
||||
]
|
||||
},
|
||||
#{dirs => ["."],
|
||||
filter => "Makefile",
|
||||
ruleset => makefiles
|
||||
|
|
|
|||
13
mix.exs
13
mix.exs
|
|
@ -91,7 +91,7 @@ defmodule EMQXUmbrella.MixProject do
|
|||
github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
|
||||
# in conflict by grpc and eetcd
|
||||
{:gpb, "4.11.2", override: true, runtime: false}
|
||||
] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
|
||||
] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
|
||||
end
|
||||
|
||||
defp umbrella_apps() do
|
||||
|
|
@ -202,6 +202,7 @@ defmodule EMQXUmbrella.MixProject do
|
|||
] ++
|
||||
if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
|
||||
if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
|
||||
if(enable_jq?(), do: [jq: :permanent], else: []) ++
|
||||
if(edition_type == :enterprise,
|
||||
do: [
|
||||
emqx_enterprise_conf: :load,
|
||||
|
|
@ -608,6 +609,12 @@ defmodule EMQXUmbrella.MixProject do
|
|||
else: []
|
||||
end
|
||||
|
||||
defp jq_dep() do
|
||||
if enable_jq?(),
|
||||
do: [{:jq, github: "emqx/jq", tag: "v0.1.0", override: true}],
|
||||
else: []
|
||||
end
|
||||
|
||||
defp quicer_dep() do
|
||||
if enable_quicer?(),
|
||||
# in conflict with emqx and emqtt
|
||||
|
|
@ -619,6 +626,10 @@ defmodule EMQXUmbrella.MixProject do
|
|||
not win32?()
|
||||
end
|
||||
|
||||
defp enable_jq?() do
|
||||
not win32?()
|
||||
end
|
||||
|
||||
defp enable_quicer?() do
|
||||
not Enum.any?([
|
||||
build_without_quic?(),
|
||||
|
|
|
|||
|
|
@ -84,6 +84,8 @@ RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.h
|
|||
|
||||
git_exact_vsn() {
|
||||
local tag
|
||||
## Needed to avoid error in github action
|
||||
git config --global --add safe.directory "/__w/emqx/emqx"
|
||||
tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)"
|
||||
echo "${tag#[v|e]}"
|
||||
}
|
||||
|
|
|
|||
545
rebar.config.erl
545
rebar.config.erl
|
|
@ -19,12 +19,17 @@ assert_otp() ->
|
|||
OtpRelease = list_to_integer(erlang:system_info(otp_release)),
|
||||
case OtpRelease < Oldest orelse OtpRelease > Latest of
|
||||
true ->
|
||||
io:format(standard_error, "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
|
||||
[OtpRelease, Oldest, Latest]),
|
||||
io:format(
|
||||
standard_error,
|
||||
"ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
|
||||
[OtpRelease, Oldest, Latest]
|
||||
),
|
||||
halt(1);
|
||||
false when OtpRelease =/= Latest ->
|
||||
io:format("WARNING: Erlang/OTP version ~p found, recommended==~p~n",
|
||||
[OtpRelease, Latest]);
|
||||
io:format(
|
||||
"WARNING: Erlang/OTP version ~p found, recommended==~p~n",
|
||||
[OtpRelease, Latest]
|
||||
);
|
||||
false ->
|
||||
ok
|
||||
end.
|
||||
|
|
@ -35,16 +40,19 @@ bcrypt() ->
|
|||
quicer() ->
|
||||
{quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}.
|
||||
|
||||
jq() ->
|
||||
{jq, {git, "https://github.com/emqx/jq", {tag, "v0.1.0"}}}.
|
||||
|
||||
deps(Config) ->
|
||||
{deps, OldDeps} = lists:keyfind(deps, 1, Config),
|
||||
MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++
|
||||
MoreDeps =
|
||||
[bcrypt() || provide_bcrypt_dep()] ++
|
||||
[jq() || provide_jq()] ++
|
||||
[quicer() || is_quicer_supported()],
|
||||
lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}).
|
||||
|
||||
overrides() ->
|
||||
[ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]}
|
||||
]}
|
||||
] ++ snabbkaffe_overrides().
|
||||
[{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}] ++ snabbkaffe_overrides().
|
||||
|
||||
%% Temporary workaround for a rebar3 erl_opts duplication
|
||||
%% bug. Ideally, we want to set this define globally
|
||||
|
|
@ -53,9 +61,10 @@ snabbkaffe_overrides() ->
|
|||
[{add, App, [{erl_opts, [{d, snk_kind, msg}]}]} || App <- Apps].
|
||||
|
||||
config() ->
|
||||
[ {cover_enabled, is_cover_enabled()}
|
||||
, {profiles, profiles()}
|
||||
, {plugins, plugins()}
|
||||
[
|
||||
{cover_enabled, is_cover_enabled()},
|
||||
{profiles, profiles()},
|
||||
{plugins, plugins()}
|
||||
].
|
||||
|
||||
is_cover_enabled() ->
|
||||
|
|
@ -70,8 +79,7 @@ is_enterprise(ee) -> true.
|
|||
|
||||
is_quicer_supported() ->
|
||||
not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse
|
||||
is_win32() orelse is_centos_6()
|
||||
).
|
||||
is_win32() orelse is_centos_6()).
|
||||
|
||||
is_centos_6() ->
|
||||
%% reason:
|
||||
|
|
@ -94,38 +102,44 @@ project_app_dirs(Edition) ->
|
|||
end.
|
||||
|
||||
plugins() ->
|
||||
[ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}
|
||||
, {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}
|
||||
[
|
||||
{relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.0.0"}}},
|
||||
{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}},
|
||||
%% emqx main project does not require port-compiler
|
||||
%% pin at root level for deterministic
|
||||
, {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}}
|
||||
]
|
||||
{pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}}
|
||||
] ++
|
||||
%% test plugins are concatenated to default profile plugins
|
||||
%% otherwise rebar3 test profile runs are super slow
|
||||
++ test_plugins().
|
||||
test_plugins().
|
||||
|
||||
test_plugins() ->
|
||||
[ {rebar3_proper, "0.12.1"}
|
||||
, {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
|
||||
[
|
||||
{rebar3_proper, "0.12.1"},
|
||||
{coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
|
||||
].
|
||||
|
||||
test_deps() ->
|
||||
[ {bbmustache, "1.10.0"}
|
||||
, {meck, "0.9.2"}
|
||||
, {proper, "1.4.0"}
|
||||
[
|
||||
{bbmustache, "1.10.0"},
|
||||
{meck, "0.9.2"},
|
||||
{proper, "1.4.0"}
|
||||
].
|
||||
|
||||
common_compile_opts(Edition, Vsn) ->
|
||||
[ debug_info % always include debug_info
|
||||
, {compile_info, [{emqx_vsn, Vsn}]}
|
||||
, {d, 'EMQX_RELEASE_EDITION', Edition}
|
||||
% always include debug_info
|
||||
[
|
||||
debug_info,
|
||||
{compile_info, [{emqx_vsn, Vsn}]},
|
||||
{d, 'EMQX_RELEASE_EDITION', Edition}
|
||||
] ++
|
||||
[{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"].
|
||||
|
||||
prod_compile_opts(Edition, Vsn) ->
|
||||
[ compressed
|
||||
, deterministic
|
||||
, warnings_as_errors
|
||||
[
|
||||
compressed,
|
||||
deterministic,
|
||||
warnings_as_errors
|
||||
| common_compile_opts(Edition, Vsn)
|
||||
].
|
||||
|
||||
|
|
@ -137,66 +151,69 @@ profiles() ->
|
|||
|
||||
profiles_ce() ->
|
||||
Vsn = get_vsn(emqx),
|
||||
[ {'emqx',
|
||||
[ {erl_opts, prod_compile_opts(ce, Vsn)}
|
||||
, {relx, relx(Vsn, cloud, bin, ce)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ce)}
|
||||
, {post_hooks, [{compile, "bash build emqx doc"}]}
|
||||
]}
|
||||
, {'emqx-pkg',
|
||||
[ {erl_opts, prod_compile_opts(ce, Vsn)}
|
||||
, {relx, relx(Vsn, cloud, pkg, ce)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ce)}
|
||||
, {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
|
||||
]}
|
||||
, {'emqx-edge',
|
||||
[ {erl_opts, prod_compile_opts(edge, Vsn)}
|
||||
, {relx, relx(Vsn, edge, bin, ce)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ce)}
|
||||
, {post_hooks, [{compile, "bash build emqx-edge doc"}]}
|
||||
]}
|
||||
, {'emqx-edge-pkg',
|
||||
[ {erl_opts, prod_compile_opts(edge, Vsn)}
|
||||
, {relx, relx(Vsn, edge, pkg, ce)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ce)}
|
||||
, {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
|
||||
[
|
||||
{'emqx', [
|
||||
{erl_opts, prod_compile_opts(ce, Vsn)},
|
||||
{relx, relx(Vsn, cloud, bin, ce)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ce)},
|
||||
{post_hooks, [{compile, "bash build emqx doc"}]}
|
||||
]},
|
||||
{'emqx-pkg', [
|
||||
{erl_opts, prod_compile_opts(ce, Vsn)},
|
||||
{relx, relx(Vsn, cloud, pkg, ce)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ce)},
|
||||
{post_hooks, [{compile, "bash build emqx-pkg doc"}]}
|
||||
]},
|
||||
{'emqx-edge', [
|
||||
{erl_opts, prod_compile_opts(edge, Vsn)},
|
||||
{relx, relx(Vsn, edge, bin, ce)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ce)},
|
||||
{post_hooks, [{compile, "bash build emqx-edge doc"}]}
|
||||
]},
|
||||
{'emqx-edge-pkg', [
|
||||
{erl_opts, prod_compile_opts(edge, Vsn)},
|
||||
{relx, relx(Vsn, edge, pkg, ce)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ce)},
|
||||
{post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
|
||||
]}
|
||||
].
|
||||
|
||||
profiles_ee() ->
|
||||
Vsn = get_vsn('emqx-enterprise'),
|
||||
[ {'emqx-enterprise',
|
||||
[ {erl_opts, prod_compile_opts(ee, Vsn)}
|
||||
, {relx, relx(Vsn, cloud, bin, ee)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ee)}
|
||||
, {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
|
||||
]}
|
||||
, {'emqx-enterprise-pkg',
|
||||
[ {erl_opts, prod_compile_opts(ee, Vsn)}
|
||||
, {relx, relx(Vsn, cloud, pkg, ee)}
|
||||
, {overrides, prod_overrides()}
|
||||
, {project_app_dirs, project_app_dirs(ee)}
|
||||
, {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
|
||||
[
|
||||
{'emqx-enterprise', [
|
||||
{erl_opts, prod_compile_opts(ee, Vsn)},
|
||||
{relx, relx(Vsn, cloud, bin, ee)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ee)},
|
||||
{post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
|
||||
]},
|
||||
{'emqx-enterprise-pkg', [
|
||||
{erl_opts, prod_compile_opts(ee, Vsn)},
|
||||
{relx, relx(Vsn, cloud, pkg, ee)},
|
||||
{overrides, prod_overrides()},
|
||||
{project_app_dirs, project_app_dirs(ee)},
|
||||
{post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
|
||||
]}
|
||||
].
|
||||
|
||||
%% EE has more files than CE, always test/check with EE options.
|
||||
profiles_dev() ->
|
||||
Vsn = get_vsn('emqx-enterprise'),
|
||||
[ {check,
|
||||
[ {erl_opts, common_compile_opts(ee, Vsn)}
|
||||
, {project_app_dirs, project_app_dirs(ee)}
|
||||
]}
|
||||
, {test,
|
||||
[ {deps, test_deps()}
|
||||
, {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()}
|
||||
, {extra_src_dirs, [{"test", [{recursive, true}]}]}
|
||||
, {project_app_dirs, project_app_dirs(ee)}
|
||||
[
|
||||
{check, [
|
||||
{erl_opts, common_compile_opts(ee, Vsn)},
|
||||
{project_app_dirs, project_app_dirs(ee)}
|
||||
]},
|
||||
{test, [
|
||||
{deps, test_deps()},
|
||||
{erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()},
|
||||
{extra_src_dirs, [{"test", [{recursive, true}]}]},
|
||||
{project_app_dirs, project_app_dirs(ee)}
|
||||
]}
|
||||
].
|
||||
|
||||
|
|
@ -204,16 +221,19 @@ profiles_dev() ->
|
|||
%% PkgType: bin | pkg
|
||||
%% Edition: ce (community) | ee (enterprise)
|
||||
relx(Vsn, RelType, PkgType, Edition) ->
|
||||
[ {include_src,false}
|
||||
, {include_erts, true}
|
||||
, {extended_start_script,false}
|
||||
, {generate_start_script,false}
|
||||
, {sys_config,false}
|
||||
, {vm_args,false}
|
||||
, {release, {emqx, Vsn}, relx_apps(RelType, Edition)}
|
||||
, {overlay, relx_overlay(RelType, Edition)}
|
||||
, {overlay_vars, build_info() ++
|
||||
[ {emqx_description, emqx_description(RelType, Edition)}
|
||||
[
|
||||
{include_src, false},
|
||||
{include_erts, true},
|
||||
{extended_start_script, false},
|
||||
{generate_start_script, false},
|
||||
{sys_config, false},
|
||||
{vm_args, false},
|
||||
{release, {emqx, Vsn}, relx_apps(RelType, Edition)},
|
||||
{overlay, relx_overlay(RelType, Edition)},
|
||||
{overlay_vars,
|
||||
build_info() ++
|
||||
[
|
||||
{emqx_description, emqx_description(RelType, Edition)}
|
||||
| overlay_vars(RelType, PkgType, Edition)
|
||||
]}
|
||||
].
|
||||
|
|
@ -221,12 +241,13 @@ relx(Vsn, RelType, PkgType, Edition) ->
|
|||
%% Make a HOCON compatible format
|
||||
build_info() ->
|
||||
Os = os_cmd("./scripts/get-distro.sh"),
|
||||
[ {build_info_arch, erlang:system_info(system_architecture)}
|
||||
, {build_info_wordsize, rebar_utils:wordsize()}
|
||||
, {build_info_os, Os}
|
||||
, {build_info_erlang, rebar_utils:otp_release()}
|
||||
, {build_info_elixir, none}
|
||||
, {build_info_relform, relform()}
|
||||
[
|
||||
{build_info_arch, erlang:system_info(system_architecture)},
|
||||
{build_info_wordsize, rebar_utils:wordsize()},
|
||||
{build_info_os, Os},
|
||||
{build_info_erlang, rebar_utils:otp_release()},
|
||||
{build_info_elixir, none},
|
||||
{build_info_relform, relform()}
|
||||
].
|
||||
|
||||
relform() ->
|
||||
|
|
@ -240,107 +261,116 @@ emqx_description(cloud, ce) -> "EMQX";
|
|||
emqx_description(edge, ce) -> "EMQX Edge".
|
||||
|
||||
overlay_vars(RelType, PkgType, Edition) ->
|
||||
overlay_vars_rel(RelType)
|
||||
++ overlay_vars_pkg(PkgType)
|
||||
++ overlay_vars_edition(Edition).
|
||||
overlay_vars_rel(RelType) ++
|
||||
overlay_vars_pkg(PkgType) ++
|
||||
overlay_vars_edition(Edition).
|
||||
|
||||
%% vars per release type, cloud or edge
|
||||
overlay_vars_rel(RelType) ->
|
||||
VmArgs = case RelType of
|
||||
VmArgs =
|
||||
case RelType of
|
||||
cloud -> "vm.args";
|
||||
edge -> "vm.args.edge"
|
||||
end,
|
||||
|
||||
[ {vm_args_file, VmArgs}
|
||||
].
|
||||
[{vm_args_file, VmArgs}].
|
||||
|
||||
overlay_vars_edition(ce) ->
|
||||
[ {emqx_schema_mod, emqx_conf_schema}
|
||||
, {is_enterprise, "no"}
|
||||
, {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)}
|
||||
[
|
||||
{emqx_schema_mod, emqx_conf_schema},
|
||||
{is_enterprise, "no"},
|
||||
{emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)}
|
||||
];
|
||||
overlay_vars_edition(ee) ->
|
||||
[ {emqx_schema_mod, emqx_enterprise_conf_schema}
|
||||
, {is_enterprise, "yes"}
|
||||
, {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)}
|
||||
[
|
||||
{emqx_schema_mod, emqx_enterprise_conf_schema},
|
||||
{is_enterprise, "yes"},
|
||||
{emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)}
|
||||
].
|
||||
|
||||
%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
|
||||
overlay_vars_pkg(bin) ->
|
||||
[ {platform_data_dir, "data"}
|
||||
, {platform_etc_dir, "etc"}
|
||||
, {platform_log_dir, "log"}
|
||||
, {platform_plugins_dir, "plugins"}
|
||||
, {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
|
||||
, {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"}
|
||||
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
|
||||
, {runner_log_dir, "$RUNNER_ROOT_DIR/log"}
|
||||
, {runner_user, ""}
|
||||
, {is_elixir, "no"}
|
||||
[
|
||||
{platform_data_dir, "data"},
|
||||
{platform_etc_dir, "etc"},
|
||||
{platform_log_dir, "log"},
|
||||
{platform_plugins_dir, "plugins"},
|
||||
{runner_bin_dir, "$RUNNER_ROOT_DIR/bin"},
|
||||
{emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"},
|
||||
{runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
|
||||
{runner_log_dir, "$RUNNER_ROOT_DIR/log"},
|
||||
{runner_user, ""},
|
||||
{is_elixir, "no"}
|
||||
];
|
||||
overlay_vars_pkg(pkg) ->
|
||||
[ {platform_data_dir, "/var/lib/emqx"}
|
||||
, {platform_etc_dir, "/etc/emqx"}
|
||||
, {platform_log_dir, "/var/log/emqx"}
|
||||
, {platform_plugins_dir, "/var/lib/emqx/plugins"}
|
||||
, {runner_bin_dir, "/usr/bin"}
|
||||
, {emqx_etc_dir, "/etc/emqx"}
|
||||
, {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
|
||||
, {runner_log_dir, "/var/log/emqx"}
|
||||
, {runner_user, "emqx"}
|
||||
, {is_elixir, "no"}
|
||||
[
|
||||
{platform_data_dir, "/var/lib/emqx"},
|
||||
{platform_etc_dir, "/etc/emqx"},
|
||||
{platform_log_dir, "/var/log/emqx"},
|
||||
{platform_plugins_dir, "/var/lib/emqx/plugins"},
|
||||
{runner_bin_dir, "/usr/bin"},
|
||||
{emqx_etc_dir, "/etc/emqx"},
|
||||
{runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
|
||||
{runner_log_dir, "/var/log/emqx"},
|
||||
{runner_user, "emqx"},
|
||||
{is_elixir, "no"}
|
||||
].
|
||||
|
||||
relx_apps(ReleaseType, Edition) ->
|
||||
[ kernel
|
||||
, sasl
|
||||
, crypto
|
||||
, public_key
|
||||
, asn1
|
||||
, syntax_tools
|
||||
, ssl
|
||||
, os_mon
|
||||
, inets
|
||||
, compiler
|
||||
, runtime_tools
|
||||
, redbug
|
||||
, {hocon, load}
|
||||
, {emqx, load} % started by emqx_machine
|
||||
, {emqx_conf, load}
|
||||
, emqx_machine
|
||||
, {mnesia, load}
|
||||
, {ekka, load}
|
||||
, {emqx_plugin_libs, load}
|
||||
, {esasl, load}
|
||||
, observer_cli
|
||||
, {system_monitor, load} % started by emqx_machine
|
||||
, emqx_http_lib
|
||||
, emqx_resource
|
||||
, emqx_connector
|
||||
, emqx_authn
|
||||
, emqx_authz
|
||||
, emqx_auto_subscribe
|
||||
, emqx_gateway
|
||||
, emqx_exhook
|
||||
, emqx_bridge
|
||||
, emqx_rule_engine
|
||||
, emqx_modules
|
||||
, emqx_management
|
||||
, emqx_dashboard
|
||||
, emqx_retainer
|
||||
, emqx_statsd
|
||||
, emqx_prometheus
|
||||
, emqx_psk
|
||||
, emqx_slow_subs
|
||||
, emqx_plugins
|
||||
]
|
||||
++ [quicer || is_quicer_supported()]
|
||||
++ [bcrypt || provide_bcrypt_release(ReleaseType)]
|
||||
++ relx_apps_per_rel(ReleaseType)
|
||||
++ relx_additional_apps(ReleaseType, Edition).
|
||||
[
|
||||
kernel,
|
||||
sasl,
|
||||
crypto,
|
||||
public_key,
|
||||
asn1,
|
||||
syntax_tools,
|
||||
ssl,
|
||||
os_mon,
|
||||
inets,
|
||||
compiler,
|
||||
runtime_tools,
|
||||
redbug,
|
||||
{hocon, load},
|
||||
% started by emqx_machine
|
||||
{emqx, load},
|
||||
{emqx_conf, load},
|
||||
emqx_machine,
|
||||
{mnesia, load},
|
||||
{ekka, load},
|
||||
{emqx_plugin_libs, load},
|
||||
{esasl, load},
|
||||
observer_cli,
|
||||
% started by emqx_machine
|
||||
{system_monitor, load},
|
||||
emqx_http_lib,
|
||||
emqx_resource,
|
||||
emqx_connector,
|
||||
emqx_authn,
|
||||
emqx_authz,
|
||||
emqx_auto_subscribe,
|
||||
emqx_gateway,
|
||||
emqx_exhook,
|
||||
emqx_bridge,
|
||||
emqx_rule_engine,
|
||||
emqx_modules,
|
||||
emqx_management,
|
||||
emqx_dashboard,
|
||||
emqx_retainer,
|
||||
emqx_statsd,
|
||||
emqx_prometheus,
|
||||
emqx_psk,
|
||||
emqx_slow_subs,
|
||||
emqx_plugins
|
||||
] ++
|
||||
[quicer || is_quicer_supported()] ++
|
||||
[bcrypt || provide_bcrypt_release(ReleaseType)] ++
|
||||
[jq || provide_jq()] ++
|
||||
relx_apps_per_rel(ReleaseType) ++
|
||||
relx_additional_apps(ReleaseType, Edition).
|
||||
|
||||
relx_apps_per_rel(cloud) ->
|
||||
[ xmerl
|
||||
[
|
||||
xmerl
|
||||
| [{observer, load} || is_app(observer)]
|
||||
];
|
||||
relx_apps_per_rel(edge) ->
|
||||
|
|
@ -354,8 +384,8 @@ is_app(Name) ->
|
|||
end.
|
||||
|
||||
relx_additional_apps(ReleaseType, Edition) ->
|
||||
relx_plugin_apps_per_rel(ReleaseType)
|
||||
++ relx_apps_per_edition(Edition).
|
||||
relx_plugin_apps_per_rel(ReleaseType) ++
|
||||
relx_apps_per_edition(Edition).
|
||||
|
||||
relx_plugin_apps_per_rel(cloud) ->
|
||||
[];
|
||||
|
|
@ -363,33 +393,34 @@ relx_plugin_apps_per_rel(edge) ->
|
|||
[].
|
||||
|
||||
relx_apps_per_edition(ee) ->
|
||||
[ emqx_license
|
||||
, {emqx_enterprise_conf, load}
|
||||
[
|
||||
emqx_license,
|
||||
{emqx_enterprise_conf, load}
|
||||
];
|
||||
|
||||
relx_apps_per_edition(ce) -> [].
|
||||
relx_apps_per_edition(ce) ->
|
||||
[].
|
||||
|
||||
emqx_machine_boot_apps(ce) ->
|
||||
[ emqx_prometheus
|
||||
, emqx_modules
|
||||
, emqx_dashboard
|
||||
, emqx_connector
|
||||
, emqx_gateway
|
||||
, emqx_statsd
|
||||
, emqx_resource
|
||||
, emqx_rule_engine
|
||||
, emqx_bridge
|
||||
, emqx_plugin_libs
|
||||
, emqx_management
|
||||
, emqx_retainer
|
||||
, emqx_exhook
|
||||
, emqx_authn
|
||||
, emqx_authz
|
||||
, emqx_slow_subs
|
||||
, emqx_auto_subscribe
|
||||
, emqx_plugins
|
||||
[
|
||||
emqx_prometheus,
|
||||
emqx_modules,
|
||||
emqx_dashboard,
|
||||
emqx_connector,
|
||||
emqx_gateway,
|
||||
emqx_statsd,
|
||||
emqx_resource,
|
||||
emqx_rule_engine,
|
||||
emqx_bridge,
|
||||
emqx_plugin_libs,
|
||||
emqx_management,
|
||||
emqx_retainer,
|
||||
emqx_exhook,
|
||||
emqx_authn,
|
||||
emqx_authz,
|
||||
emqx_slow_subs,
|
||||
emqx_auto_subscribe,
|
||||
emqx_plugins
|
||||
];
|
||||
|
||||
emqx_machine_boot_apps(ee) ->
|
||||
emqx_machine_boot_apps(ce) ++
|
||||
[].
|
||||
|
|
@ -397,67 +428,75 @@ emqx_machine_boot_apps(ee) ->
|
|||
emqx_machine_boot_app_list(Edition) ->
|
||||
string:join(
|
||||
[atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)],
|
||||
", ").
|
||||
", "
|
||||
).
|
||||
|
||||
relx_overlay(ReleaseType, Edition) ->
|
||||
[ {mkdir, "log/"}
|
||||
, {mkdir, "data/"}
|
||||
, {mkdir, "plugins"}
|
||||
, {mkdir, "data/mnesia"}
|
||||
, {mkdir, "data/configs"}
|
||||
, {mkdir, "data/patches"}
|
||||
, {mkdir, "data/scripts"}
|
||||
, {template, "rel/emqx_vars", "releases/emqx_vars"}
|
||||
, {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"}
|
||||
, {copy, "bin/emqx", "bin/emqx"}
|
||||
, {copy, "bin/emqx_ctl", "bin/emqx_ctl"}
|
||||
, {copy, "bin/node_dump", "bin/node_dump"}
|
||||
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}
|
||||
, {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup
|
||||
, {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup
|
||||
, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup
|
||||
, {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}
|
||||
, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}
|
||||
, {template, "bin/emqx.cmd", "bin/emqx.cmd"}
|
||||
, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
|
||||
, {copy, "bin/nodetool", "bin/nodetool"}
|
||||
, {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
|
||||
[
|
||||
{mkdir, "log/"},
|
||||
{mkdir, "data/"},
|
||||
{mkdir, "plugins"},
|
||||
{mkdir, "data/mnesia"},
|
||||
{mkdir, "data/configs"},
|
||||
{mkdir, "data/patches"},
|
||||
{mkdir, "data/scripts"},
|
||||
{template, "rel/emqx_vars", "releases/emqx_vars"},
|
||||
{template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"},
|
||||
{copy, "bin/emqx", "bin/emqx"},
|
||||
{copy, "bin/emqx_ctl", "bin/emqx_ctl"},
|
||||
{copy, "bin/node_dump", "bin/node_dump"},
|
||||
{copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"},
|
||||
%% for relup
|
||||
{copy, "bin/emqx", "bin/emqx-{{release_version}}"},
|
||||
%% for relup
|
||||
{copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"},
|
||||
%% for relup
|
||||
{copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"},
|
||||
{copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"},
|
||||
{copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"},
|
||||
{template, "bin/emqx.cmd", "bin/emqx.cmd"},
|
||||
{template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"},
|
||||
{copy, "bin/nodetool", "bin/nodetool"},
|
||||
{copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
|
||||
] ++ etc_overlay(ReleaseType, Edition).
|
||||
|
||||
etc_overlay(ReleaseType, Edition) ->
|
||||
Templates = emqx_etc_overlay(ReleaseType, Edition),
|
||||
[ {mkdir, "etc/"}
|
||||
, {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"}
|
||||
[
|
||||
{mkdir, "etc/"},
|
||||
{copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}
|
||||
] ++
|
||||
lists:map(
|
||||
fun({From, To}) -> {template, From, To};
|
||||
fun
|
||||
({From, To}) -> {template, From, To};
|
||||
(FromTo) -> {template, FromTo, FromTo}
|
||||
end, Templates).
|
||||
end,
|
||||
Templates
|
||||
).
|
||||
|
||||
emqx_etc_overlay(ReleaseType, Edition) ->
|
||||
emqx_etc_overlay_per_rel(ReleaseType)
|
||||
++ emqx_etc_overlay_per_edition(Edition)
|
||||
++ emqx_etc_overlay_common().
|
||||
emqx_etc_overlay_per_rel(ReleaseType) ++
|
||||
emqx_etc_overlay_per_edition(Edition) ++
|
||||
emqx_etc_overlay_common().
|
||||
|
||||
emqx_etc_overlay_per_rel(cloud) ->
|
||||
[ {"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args","etc/vm.args"}
|
||||
];
|
||||
[{"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args", "etc/vm.args"}];
|
||||
emqx_etc_overlay_per_rel(edge) ->
|
||||
[ {"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args","etc/vm.args"}
|
||||
].
|
||||
[{"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args", "etc/vm.args"}].
|
||||
|
||||
emqx_etc_overlay_common() ->
|
||||
[ {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}
|
||||
].
|
||||
[{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}].
|
||||
|
||||
emqx_etc_overlay_per_edition(ce) ->
|
||||
[ {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
|
||||
, {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
|
||||
[
|
||||
{"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
|
||||
{"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
|
||||
];
|
||||
emqx_etc_overlay_per_edition(ee) ->
|
||||
[ {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"}
|
||||
, {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
|
||||
, {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
|
||||
[
|
||||
{"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"},
|
||||
{"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
|
||||
{"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
|
||||
].
|
||||
|
||||
get_vsn(Profile) ->
|
||||
|
|
@ -471,7 +510,8 @@ os_cmd(Cmd) ->
|
|||
re:replace(Output, "\n", "", [{return, list}]).
|
||||
|
||||
maybe_dump(Config) ->
|
||||
is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
|
||||
is_debug() andalso
|
||||
file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
|
||||
Config.
|
||||
|
||||
is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC").
|
||||
|
|
@ -486,6 +526,9 @@ is_debug(VarName) ->
|
|||
provide_bcrypt_dep() ->
|
||||
not is_win32().
|
||||
|
||||
provide_jq() ->
|
||||
not is_win32().
|
||||
|
||||
provide_bcrypt_release(ReleaseType) ->
|
||||
provide_bcrypt_dep() andalso ReleaseType =:= cloud.
|
||||
|
||||
|
|
@ -497,7 +540,8 @@ erl_opts_i() ->
|
|||
dialyzer(Config) ->
|
||||
{dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
|
||||
|
||||
AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of
|
||||
AppsToAnalyse =
|
||||
case os:getenv("DIALYZER_ANALYSE_APP") of
|
||||
false ->
|
||||
[];
|
||||
Value ->
|
||||
|
|
@ -512,7 +556,12 @@ dialyzer(Config) ->
|
|||
|
||||
case length(AppsToAnalyse) > 0 of
|
||||
true ->
|
||||
lists:keystore(dialyzer, 1, Config, {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]});
|
||||
lists:keystore(
|
||||
dialyzer,
|
||||
1,
|
||||
Config,
|
||||
{dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}
|
||||
);
|
||||
false ->
|
||||
Config
|
||||
end.
|
||||
|
|
@ -520,13 +569,17 @@ dialyzer(Config) ->
|
|||
coveralls() ->
|
||||
case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of
|
||||
{"true", Token} when is_list(Token) ->
|
||||
Cfgs = [{coveralls_repo_token, Token},
|
||||
Cfgs = [
|
||||
{coveralls_repo_token, Token},
|
||||
{coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")},
|
||||
{coveralls_commit_sha, os:getenv("GITHUB_SHA")},
|
||||
{coveralls_coverdata, "_build/test/cover/*.coverdata"},
|
||||
{coveralls_service_name, "github"}],
|
||||
case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request"
|
||||
andalso string:tokens(os:getenv("GITHUB_REF"), "/") of
|
||||
{coveralls_service_name, "github"}
|
||||
],
|
||||
case
|
||||
os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso
|
||||
string:tokens(os:getenv("GITHUB_REF"), "/")
|
||||
of
|
||||
[_, "pull", PRNO, _] ->
|
||||
[{coveralls_service_pull_request, PRNO} | Cfgs];
|
||||
_ ->
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
## example:
|
||||
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
|
||||
## --builder ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10
|
||||
## --builder ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ help() {
|
|||
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
|
||||
echo "--src_dir <SRC_DIR>: EMQX source ode in this dir, default to PWD"
|
||||
echo "--builder <BUILDER>: Builder image to pull"
|
||||
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10"
|
||||
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10"
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
|
|
@ -120,13 +120,6 @@ fi
|
|||
HOST_SYSTEM="$(./scripts/get-distro.sh)"
|
||||
BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')"
|
||||
|
||||
# quick workaround before builder image is renamed
|
||||
if [ "$BUILDER_SYSTEM" = 'centos7' ]; then
|
||||
BUILDER_SYSTEM='el7'
|
||||
elif [ "$BUILDER_SYSTEM" = 'rockylinux8' ]; then
|
||||
BUILDER_SYSTEM='el8'
|
||||
fi
|
||||
|
||||
CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
|
||||
|
||||
IS_NATIVE_SYSTEM='no'
|
||||
|
|
|
|||
Loading…
Reference in New Issue