Merge remote-tracking branch 'origin/release-v43' into release-v44

This commit is contained in:
Zaiming (Stone) Shi 2022-11-07 10:48:38 +01:00
commit 0eebabd34c
18 changed files with 161 additions and 75 deletions

View File

@ -14,31 +14,37 @@ jobs:
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
outputs:
ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }}
fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
steps:
- uses: actions/checkout@v3
with:
path: source
fetch-depth: 0
- name: git credentials
if: endsWith(github.repository, 'emqx-enterprise')
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
fi
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: find_ct_apps
working-directory: source
id: run_find_apps
# emqx_plugin_libs doesn't have a test suite -> excluded from app list
# emqx ct is run independently -> exclude it from the app list
# emqx ct is run independently -> exclude it from the app list (regular ct)
# lib-ee/emqx_node_rebalance -> exclude it from app list (regular ct)
run: |
ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . == "emqx"))')"
echo "ct-apps: $ct_apps"
echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT
fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT
echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT
- name: get_all_deps
working-directory: source
run: |
make deps-all
# build the default profile for two purposes
# 1. download all dependencies (so the individual app runs do not depend on github credentials)
# 2. some of the files such as segmented config files are not created when compiling only the test profile
make
# compile test profile to speed up the individual runs
./rebar3 as test compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
@ -75,12 +81,24 @@ jobs:
path: source/_build/test/cover
if-no-files-found: warn
emqx_ct:
fast_ct:
needs: prepare
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
strategy:
fail-fast: false
matrix:
app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
runs-on:
- aws-amd64
- ubuntu-20.04
use-self-hosted:
- ${{ github.repository_owner == 'emqx' }}
exclude:
- runs-on: ubuntu-20.04
use-self-hosted: true
- runs-on: aws-amd64
use-self-hosted: false
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
@ -89,24 +107,26 @@ jobs:
path: .
- name: unzip source code
run: unzip -o -q source.zip
# produces emqx-emqx.coverdata
- name: emqx-ct-pipeline
# produces emqx-<appname>.coverdata
- name: make-app-ct-pipeline
working-directory: source
run: make emqx-ct-pipeline
run: |
export CT_READABLE=true
make ${{ matrix.app_name }}-ct-pipeline
- uses: actions/upload-artifact@v3
with:
name: cover
path: source/_build/test/cover
if-no-files-found: warn
ct:
docker_ct:
needs: prepare
runs-on: ${{ matrix.runs-on }}
strategy:
max-parallel: 12
fail-fast: false
matrix:
app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }}
app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
runs-on:
- aws-amd64
- ubuntu-20.04
@ -132,28 +152,6 @@ jobs:
docker network rm $(docker network ls -q) || true
- name: docker compose up
working-directory: source
if: endsWith(github.repository, 'emqx')
env:
MYSQL_TAG: 8
REDIS_TAG: 6
MONGO_TAG: 4
PGSQL_TAG: 13
LDAP_TAG: 2.4.50
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
up -d --build
docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
- name: docker compose up
working-directory: source
if: endsWith(github.repository, 'emqx-enterprise')
env:
MYSQL_TAG: 8
REDIS_TAG: 6
@ -169,17 +167,20 @@ jobs:
KAFKA_TAG: 2.5.0
PULSAR_TAG: 2.3.2
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
timeout-minutes: 20
timeout-minutes: 40
run: |
docker-compose \
ulimit -n
docker_compose_files="\
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml"
ee_docker_compose_files="\
-f .ci/docker-compose-file/docker-compose-enterprise.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
@ -189,16 +190,30 @@ jobs:
-f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
up -d --build
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
-f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml"
if [ -f EMQX_ENTERPRISE ]; then
docker_compose_files="${docker_compose_files} ${ee_docker_compose_files}"
fi
# only build ldap docker image when necessary, it takes 9 minutes
if [[ ${{ matrix.app_name }} == *ldap ]]; then
docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml"
fi
docker-compose $docker_compose_files up -d --build
if [ -f EMQX_ENTERPRISE ]; then
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
fi
docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
!= $(docker ps -a --filter name=client | wc -l) ]; do
sleep 5
clients="$(docker ps -q --filter name=client)"
for client in ${clients}; do
docker ps -a --filter name=client
echo "waiting for docker ${client} to exit"
if ! timeout 60 docker wait "${client}"; then
docker-compose $docker_compose_files logs | tee docker-compose.log
exit 1
fi
done
- name: run common test
run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline"
run: docker exec -i erlang bash -c "env CT_READABLE=true make ${{ matrix.app_name }}-ct-pipeline"
- name: cat rebar.crashdump
if: failure()
working-directory: source
@ -210,7 +225,9 @@ jobs:
if: failure()
with:
name: ${{ env.LOGFILENAME }}
path: source/_build/test/logs
path: |
docker-compose.log
source/_build/test/logs
if-no-files-found: warn
- uses: actions/upload-artifact@v3
with:
@ -221,8 +238,8 @@ jobs:
make_cover:
needs:
- eunit_and_proper
- emqx_ct
- ct
- fast_ct
- docker_ct
runs-on: ubuntu-20.04
container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
steps:

View File

@ -30,7 +30,7 @@ File format:
- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
Eliminate uncertainty that the buffer size is set by OS default.
- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
- Disable authorization for `api/v4/emqx_prometheus` endpoint. [#8955](https://github.com/emqx/emqx/pull/8955)
- Added a test to prevent a last will testament message to be
published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)

View File

@ -21,7 +21,9 @@ REL_PROFILES := emqx emqx-edge
PKG_PROFILES := emqx-pkg emqx-edge-pkg
PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
export REBAR_GIT_CLONE_OPTIONS += --depth=1 --quiet
CT_READABLE ?= false
export REBAR_GIT_CLONE_OPTIONS += --depth=1
.PHONY: default
default: $(REBAR) $(PROFILE)
@ -58,7 +60,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
.PHONY: $(APPS:%=%-ct)
define gen-app-ct-target
$1-ct: $(REBAR)
$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
$(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
@ -66,7 +68,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
.PHONY: $(APPS:%=%-ct-pipeline)
define gen-app-ct-target-pipeline
$1-ct-pipeline: $(REBAR)
$(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
$(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
@ -87,6 +89,7 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard
@$(REBAR) as $(@) do compile,release

View File

View File

View File

View File

View File

View File

View File

@ -508,7 +508,7 @@ delete_all_deactivated_alarms() ->
delete_all_deactivated_alarms(Node) when Node =:= node() ->
emqx_alarm:delete_all_deactivated_alarms();
delete_all_deactivated_alarms(Node) ->
rpc_call(Node, delete_deactivated_alarms, [Node]).
rpc_call(Node, emqx_alarm, delete_all_deactivated_alarms, []).
add_duration_field(Alarms) ->
Now = erlang:system_time(microsecond),
@ -581,7 +581,10 @@ item(route, {Topic, Node}) ->
%%--------------------------------------------------------------------
rpc_call(Node, Fun, Args) ->
case rpc:call(Node, ?MODULE, Fun, Args) of
rpc_call(Node, ?MODULE, Fun, Args).
rpc_call(Node, Mod, Fun, Args) ->
case rpc:call(Node, Mod, Fun, Args) of
{badrpc, Reason} -> {error, Reason};
Res -> Res
end.

View File

@ -125,6 +125,11 @@ handle_request(_Method, _Path, Req) ->
cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req).
authorize_appid(Req) ->
authorize_appid(cowboy_req:method(Req), cowboy_req:path(Req), Req).
authorize_appid(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) ->
true;
authorize_appid(_Method, _Path, Req) ->
try
{basic, AppId, AppSecret} = cowboy_req:parse_header(<<"authorization">>, Req),
emqx_mgmt_auth:is_authorized(AppId, AppSecret)

View File

@ -34,7 +34,7 @@ all() ->
{group, check_cli}] ++ OtherTCs.
manage_apps_tests() ->
[t_app].
[t_app, t_alarms].
check_cli_tests() ->
[t_cli,
@ -99,6 +99,13 @@ init_per_testcase(t_backup_file, Config) ->
file:delete(filename:join(BackupDir, F))
end, Files),
Config;
init_per_testcase(Case, Config) when Case =:= t_alarms ->
try
?MODULE:Case({'init', Config})
catch
error : function_clause ->
Config
end;
init_per_testcase(_Case, Config) ->
mock_print(),
Config.
@ -124,6 +131,13 @@ end_per_testcase(t_backup_file, _Config) ->
file:delete(filename:join(BackupDir, F))
end, Files),
ok;
end_per_testcase(Case, Config) when Case =:= t_alarms ->
try
?MODULE:Case({'end', Config})
catch
error : function_clause ->
ok
end;
end_per_testcase(_Case, _Config) ->
unmock_print().
@ -158,6 +172,16 @@ t_app(_Config) ->
emqx_mgmt_auth:del_app(<<"app_id">>),
ok.
t_alarms({'init', Config}) ->
meck:new(rpc, [unstick]),
meck:expect(rpc, call, 4, ok),
Config;
t_alarms({'end', _}) ->
meck:unload(rpc);
t_alarms(_) ->
ok = emqx_mgmt:delete_all_deactivated_alarms(remote_node),
?assert(meck:called(rpc, call, [remote_node, emqx_alarm, delete_all_deactivated_alarms, []])).
t_log_cmd(_) ->
lists:foreach(fun(Level) ->
emqx_mgmt_cli:log(["primary-level", Level]),

View File

View File

@ -70,4 +70,8 @@
Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`.
See docs [Create Rule](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources).
- Calling 'DELETE /alarms/deactivated' now deletes deactived alarms on all nodes, including remote nodes, not just the local node [#9280](https://github.com/emqx/emqx/pull/9280).
- When republishing messages or bridge messages to other brokers, check the validity of the topic and make sure it does not have topic wildcards [#9291](https://github.com/emqx/emqx/pull/9291).
- Disable authorization for `api/v4/emqx_prometheus` endpoint on management api listener (default 8081) [#9294](https://github.com/emqx/emqx/pull/9294).

View File

@ -64,4 +64,8 @@
注意在创建规则或资源时HTTP body 中的 `id` 字段仍为字面值,而不是编码之后的值。
详情请参考 [创建规则](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources)。
- 修复调用 'DELETE /alarms/deactivated' 只在单个节点上生效的问题,现在将会删除所有节点上的非活跃警告 [#9280](https://github.com/emqx/emqx/pull/9280)。
- 在进行消息重发布或桥接消息到其他 mqtt broker 时,检查 topic 合法性,确定其不带有主题通配符 [#9291](https://github.com/emqx/emqx/pull/9291)。
- 关闭管理端口默认为8081上对 HTTP API `api/v4/emqx_prometheus` 的认证Prometheus 对时序数据抓取不在需要配置认证 [#9294](https://github.com/emqx/emqx/pull/9294)。

View File

@ -107,13 +107,13 @@ http_handlers() ->
%%--------------------------------------------------------------------
is_authorized(Req) ->
is_authorized(binary_to_list(cowboy_req:path(Req)), Req).
is_authorized(cowboy_req:method(Req), cowboy_req:path(Req), Req).
is_authorized("/api/v4/emqx_prometheus", _Req) ->
is_authorized(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) ->
true;
is_authorized("/api/v4/auth", _Req) ->
is_authorized(<<"POST">>, <<"/api/v4/auth">>, _Req) ->
true;
is_authorized(_Path, Req) ->
is_authorized(_Method, _Path, Req) ->
try
{basic, Username, Password} = cowboy_req:parse_header(<<"authorization">>, Req),
case emqx_dashboard_admin:check(iolist_to_binary(Username), iolist_to_binary(Password)) of

View File

@ -7,11 +7,13 @@ cd -P -- "$(dirname -- "$0")/.."
help() {
echo
echo "-h|--help: To display this usage info"
echo "--json: Print apps in json"
echo "-h|--help: To display this usage info"
echo "--json: Print apps in json"
echo "-ct fast|docker: Only find apps for CT"
}
WANT_JSON='no'
CT='novalue'
while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
@ -22,12 +24,17 @@ while [ "$#" -gt 0 ]; do
WANT_JSON='yes'
shift 1
;;
--ct)
CT="$2"
shift 2
;;
*)
echo "unknown option $1"
exit 1
;;
esac
done
if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
# Otherwise windows may resolve to find.exe
FIND="/usr/bin/find"
@ -41,25 +48,44 @@ find_app() {
}
EM="emqx"
CE="$(find_app 'apps')"
SHARED="$(find_app 'apps')"
if [ -f 'EMQX_ENTERPRISE' ]; then
LIB="$(find_app 'lib-ee')"
else
LIB="$(find_app 'lib-ce')"
fi
## find directories in lib-extra
LIBE="$(find_app 'lib-extra')"
## find symlinks in lib-extra
LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)"
APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")"
APPS_ALL="$(echo -e "${EM}\n${SHARED}\n${LIB}\n${LIBE}\n${LIBES}")"
if [ "$WANT_JSON" = 'yes' ]; then
echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
if [ "$CT" = 'novalue' ]; then
RESULT="${APPS_ALL}"
else
echo "${APPS_ALL}"
APPS_NORMAL_CT=( )
APPS_DOCKER_CT=( )
for app in ${APPS_ALL}; do
if [ "$app" = 'apps/emqx_plugin_libs' ]; then
# This app has no test SUITE
continue
fi
if [ -f "${app}/docker-ct" ]; then
APPS_DOCKER_CT+=("$app")
else
APPS_NORMAL_CT+=("$app")
fi
done
if [ "$CT" = 'docker' ]; then
RESULT="${APPS_DOCKER_CT[*]}"
else
RESULT="${APPS_NORMAL_CT[*]}"
fi
fi
if [ "$WANT_JSON" = 'yes' ]; then
echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
else
echo "${RESULT}" | xargs
fi

View File

@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
case "${PKG_VSN}" in
4.3*)
EMQX_CE_DASHBOARD_VERSION='v4.3.11'
EMQX_EE_DASHBOARD_VERSION='v4.3.26'
EMQX_CE_DASHBOARD_VERSION='v4.3.12'
EMQX_EE_DASHBOARD_VERSION='v4.3.27'
;;
4.4*)
# keep the above 4.3 untouched, otherwise conflicts!