From 835d3a33eff71d9c2e9e7cbda828691be8d5c64e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 31 Oct 2022 15:07:45 +0100 Subject: [PATCH 1/7] fix(emqx_mgmt): call emqx_alarm:delete_all_deactivated_alarms for remote nodes Old call was calling a non-existant function on current module, instead we make call directly into `emqx_alarm` on remote node. --- apps/emqx_management/src/emqx_mgmt.erl | 7 +++-- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 31 ++++++++++++++++++- changes/v4.3.22-en.md | 2 ++ changes/v4.3.22-zh.md | 2 ++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index c5062d477..2d0ffb7a6 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -499,7 +499,7 @@ delete_all_deactivated_alarms() -> delete_all_deactivated_alarms(Node) when Node =:= node() -> emqx_alarm:delete_all_deactivated_alarms(); delete_all_deactivated_alarms(Node) -> - rpc_call(Node, delete_deactivated_alarms, [Node]). + rpc_call(Node, emqx_alarm, delete_all_deactivated_alarms, []). add_duration_field(Alarms) -> Now = erlang:system_time(microsecond), @@ -574,7 +574,10 @@ item(route, {Topic, Node}) -> %%-------------------------------------------------------------------- rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of + rpc_call(Node, ?MODULE, Fun, Args). + +rpc_call(Node, Mod, Fun, Args) -> + case rpc:call(Node, Mod, Fun, Args) of {badrpc, Reason} -> {error, Reason}; Res -> Res end. diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 66bd07405..48b2b4201 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -34,7 +34,8 @@ all() -> groups() -> [{manage_apps, [sequence], - [t_app + [t_app, + t_alarms ]}, {check_cli, [sequence], [t_cli, @@ -64,6 +65,24 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps(apps()). +init_per_testcase(Case, Config) when Case =:= t_alarms -> + try + ?MODULE:Case({'init', Config}) + catch + error : function_clause -> + Config + end; +init_per_testcase(_, Config) -> Config. + +end_per_testcase(Case, Config) when Case =:= t_alarms -> + try + ?MODULE:Case({'end', Config}) + catch + error : function_clause -> + ok + end; +end_per_testcase(_, Config) -> Config. + t_app(_Config) -> {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>), ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)), @@ -95,6 +114,16 @@ t_app(_Config) -> emqx_mgmt_auth:del_app(<<"app_id">>), ok. +t_alarms({'init', Config}) -> + meck:new(rpc, [unstick]), + meck:expect(rpc, call, 4, ok), + Config; +t_alarms({'end', _}) -> + meck:unload(rpc); +t_alarms(_) -> + ok = emqx_mgmt:delete_all_deactivated_alarms(remote_node), + ?assert(meck:called(rpc, call, [remote_node, emqx_alarm, delete_all_deactivated_alarms, []])). + t_log_cmd(_) -> mock_print(), lists:foreach(fun(Level) -> diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 214d996a7..4a355d520 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -41,3 +41,5 @@ - Make sure Rule-Engine API supports Percent-encoding `rule_id` and `resource_id` in HTTP request path [#9190](https://github.com/emqx/emqx/pull/9190). Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`. See docs [Create Rule](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources). + +- Calling 'DELETE /alarms/deactivated' now deletes deactived alarms on all nodes, including remote nodes, not just the local node. [#9280](https://github.com/emqx/emqx/pull/9280) diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 2323882dc..588244c3a 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -41,3 +41,5 @@ - 使规则引擎 API 在 HTTP 请求路径中支持百分号编码的 `rule_id` 及 `resource_id` [#9190](https://github.com/emqx/emqx/pull/9190)。 注意在创建规则或资源时，HTTP body 中的 `id` 字段仍为字面值，而不是编码之后的值。 详情请参考 [创建规则](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources)。 + +- 修复调用 'DELETE /alarms/deactivated' 只在单个节点上生效的问题，现在将会删除所有节点上的非活跃警告。 From 6ae2b06ba12599c054a6cc9b77af4f88c11c8223 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 4 Nov 2022 22:11:41 +0100 Subject: [PATCH 2/7] chore: upgrade dashboard version --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index 5a0ca5da0..c316e7054 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) - EMQX_CE_DASHBOARD_VERSION='v4.3.11' - EMQX_EE_DASHBOARD_VERSION='v4.3.26' + EMQX_CE_DASHBOARD_VERSION='v4.3.12' + EMQX_EE_DASHBOARD_VERSION='v4.3.27' ;; *) echo "Unsupported version $PKG_VSN" >&2 From 639006e3025d96a4437b66176ee2a42b306d7fcd Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 3 Nov 2022 18:13:23 +0800 Subject: [PATCH 3/7] fix(prometheus): disable auth for prometheus endpoint on mgmt listener --- CHANGES-4.3.md | 2 +- apps/emqx_management/src/emqx_mgmt_http.erl | 5 +++++ changes/v4.3.22-en.md | 2 +- changes/v4.3.22-zh.md | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 8 ++++---- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index c812713db..097801127 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -30,7 +30,7 @@ File format: - TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007) Eliminate uncertainty that the buffer size is set by OS default. -- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) +- Disable authorization for `api/v4/emqx_prometheus` endpoint. [#8955](https://github.com/emqx/emqx/pull/8955) - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 8cddc11b2..57c54ed3f 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -124,6 +124,11 @@ handle_request(_Method, _Path, Req) -> cowboy_req:reply(400, #{<<"content-type">> => <<"text/plain">>}, <<"Not found.">>, Req). authorize_appid(Req) -> + authorize_appid(cowboy_req:method(Req), cowboy_req:path(Req), Req). + +authorize_appid(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) -> + true; +authorize_appid(_Method, _Path, Req) -> try {basic, AppId, AppSecret} = cowboy_req:parse_header(<<"authorization">>, Req), emqx_mgmt_auth:is_authorized(AppId, AppSecret) diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index 386b9b0ff..1a83ae650 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -70,8 +70,8 @@ Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`. See docs [Create Rule](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources). - - Calling 'DELETE /alarms/deactivated' now deletes deactived alarms on all nodes, including remote nodes, not just the local node [#9280](https://github.com/emqx/emqx/pull/9280). - When republishing messages or bridge messages to other brokers, check the validity of the topic and make sure it does not have topic wildcards [#9291](https://github.com/emqx/emqx/pull/9291). +- Disable authorization for `api/v4/emqx_prometheus` endpoint on management api listener (default 8081) [#9294](https://github.com/emqx/emqx/pull/9294). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 9304d5c2e..38944d499 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -64,8 +64,8 @@ 注意在创建规则或资源时，HTTP body 中的 `id` 字段仍为字面值，而不是编码之后的值。 详情请参考 [创建规则](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources)。 - - 修复调用 'DELETE /alarms/deactivated' 只在单个节点上生效的问题，现在将会删除所有节点上的非活跃警告 [#9280](https://github.com/emqx/emqx/pull/9280)。 - 在进行消息重发布或桥接消息到其他 mqtt broker 时，检查 topic 合法性，确定其不带有主题通配符 [#9291](https://github.com/emqx/emqx/pull/9291)。 +- 关闭管理端口（默认为8081）上对 HTTP API `api/v4/emqx_prometheus` 的认证，Prometheus 对时序数据抓取不在需要配置认证 [#9294](https://github.com/emqx/emqx/pull/9294)。 diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index 517211607..3ad0694c4 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -107,13 +107,13 @@ http_handlers() -> %%-------------------------------------------------------------------- is_authorized(Req) -> - is_authorized(binary_to_list(cowboy_req:path(Req)), Req). + is_authorized(cowboy_req:method(Req), cowboy_req:path(Req), Req). -is_authorized("/api/v4/emqx_prometheus", _Req) -> +is_authorized(<<"GET">>, <<"/api/v4/emqx_prometheus">>, _Req) -> true; -is_authorized("/api/v4/auth", _Req) -> +is_authorized(<<"POST">>, <<"/api/v4/auth">>, _Req) -> true; -is_authorized(_Path, Req) -> +is_authorized(_Method, _Path, Req) -> try {basic, Username, Password} = cowboy_req:parse_header(<<"authorization">>, Req), case emqx_dashboard_admin:check(iolist_to_binary(Username), iolist_to_binary(Password)) of From 8ff8a8b55d45ecabe18cd4b7012368884bec5a71 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 5 Nov 2022 13:31:48 +0100 Subject: [PATCH 4/7] ci: split fast (regular) and docker-compose (docker) ct --- .github/workflows/run_test_cases.yaml | 121 +++++++++++++++----------- Makefile | 7 +- apps/emqx_auth_ldap/docker-ct | 0 apps/emqx_auth_mongo/docker-ct | 0 apps/emqx_auth_mysql/docker-ct | 0 apps/emqx_auth_pgsql/docker-ct | 0 apps/emqx_auth_redis/docker-ct | 0 scripts/find-apps.sh | 46 +++++++--- 8 files changed, 110 insertions(+), 64 deletions(-) create mode 100644 apps/emqx_auth_ldap/docker-ct create mode 100644 apps/emqx_auth_mongo/docker-ct create mode 100644 apps/emqx_auth_mysql/docker-ct create mode 100644 apps/emqx_auth_pgsql/docker-ct create mode 100644 apps/emqx_auth_redis/docker-ct diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d67ed1395..afc122751 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -14,31 +14,37 @@ jobs: runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: - ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }} + fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} + docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} steps: - uses: actions/checkout@v3 with: path: source fetch-depth: 0 - name: git credentials + if: endsWith(github.repository, 'emqx-enterprise') run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store - name: find_ct_apps working-directory: source id: run_find_apps # emqx_plugin_libs doesn't have a test suite -> excluded from app list - # emqx ct is run independently -> exclude it from the app list + # emqx ct is run independently -> exclude it from the app list (regular ct) + # lib-ee/emqx_node_rebalance -> exclude it from app list (regular ct) run: | - ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . == "emqx"))')" - echo "ct-apps: $ct_apps" - echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT + fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)" + docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)" + echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT + echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT - name: get_all_deps working-directory: source run: | - make deps-all + # build the default profile for two purposes + # 1. download all dependencies (so the individual app runs do not depend on github credentials) + # 2. some of the files such as segmented config files are not created when compiling only the test profile + make + # compile test profile to speed up the individual runs ./rebar3 as test compile cd .. zip -ryq source.zip source/* source/.[^.]* @@ -75,12 +81,24 @@ jobs: path: source/_build/test/cover if-no-files-found: warn - emqx_ct: + fast_ct: needs: prepare runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 strategy: fail-fast: false + matrix: + app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} + runs-on: + - aws-amd64 + - ubuntu-20.04 + use-self-hosted: + - ${{ github.repository_owner == 'emqx' }} + exclude: + - runs-on: ubuntu-20.04 + use-self-hosted: true + - runs-on: aws-amd64 + use-self-hosted: false steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 @@ -89,24 +107,26 @@ jobs: path: . - name: unzip source code run: unzip -o -q source.zip - # produces emqx-emqx.coverdata - - name: emqx-ct-pipeline + # produces emqx-.coverdata + - name: make-app-ct-pipeline working-directory: source - run: make emqx-ct-pipeline + run: | + export CT_READABLE=true + make ${{ matrix.app_name }}-ct-pipeline - uses: actions/upload-artifact@v3 with: name: cover path: source/_build/test/cover if-no-files-found: warn - ct: + docker_ct: needs: prepare runs-on: ${{ matrix.runs-on }} strategy: max-parallel: 12 fail-fast: false matrix: - app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }} + app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} runs-on: - aws-amd64 - ubuntu-20.04 @@ -132,28 +152,6 @@ jobs: docker network rm $(docker network ls -q) || true - name: docker compose up working-directory: source - if: endsWith(github.repository, 'emqx') - env: - MYSQL_TAG: 8 - REDIS_TAG: 6 - MONGO_TAG: 4 - PGSQL_TAG: 13 - LDAP_TAG: 2.4.50 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - up -d --build - docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - - name: docker compose up - working-directory: source - if: endsWith(github.repository, 'emqx-enterprise') env: MYSQL_TAG: 8 REDIS_TAG: 6 @@ -169,16 +167,19 @@ jobs: KAFKA_TAG: 2.5.0 PULSAR_TAG: 2.3.2 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 20 + timeout-minutes: 40 run: | - docker-compose \ + ulimit -n + docker_compose_files="\ -f .ci/docker-compose-file/docker-compose.yaml \ -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml" + ee_docker_compose_files="\ -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ @@ -189,16 +190,30 @@ jobs: -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \ - up -d --build - docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml" + if [ -f EMQX_ENTERPRISE ]; then + docker_compose_files="${docker_compose_files} ${ee_docker_compose_files}" + fi + # only build ldap docker image when necessary, it takes 9 minutes + if [[ ${{ matrix.app_name }} == *ldap ]]; then + docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml" + fi + docker-compose $docker_compose_files up -d --build + if [ -f EMQX_ENTERPRISE ]; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 5 + clients="$(docker ps -q --filter name=client)" + for client in ${clients}; do + docker ps -a --filter name=client + echo "waiting for docker ${client} to exit" + if ! timeout 60 docker wait "${client}"; then + docker-compose $docker_compose_files logs | tee docker-compose.log + exit 1 + fi done - name: run common test - run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline" + run: docker exec -i erlang bash -c "env CT_READABLE=true make ${{ matrix.app_name }}-ct-pipeline" - name: cat rebar.crashdump if: failure() working-directory: source @@ -210,7 +225,9 @@ jobs: if: failure() with: name: ${{ env.LOGFILENAME }} - path: source/_build/test/logs + path: | + docker-compose.log + source/_build/test/logs if-no-files-found: warn - uses: actions/upload-artifact@v3 with: @@ -221,8 +238,8 @@ jobs: make_cover: needs: - eunit_and_proper - - emqx_ct - - ct + - fast_ct + - docker_ct runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 steps: diff --git a/Makefile b/Makefile index dfe6ceaa0..64f278098 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ REL_PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default +CT_READABLE ?= false + export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default @@ -57,7 +59,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) - $(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) + $(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -65,7 +67,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) .PHONY: $(APPS:%=%-ct-pipeline) define gen-app-ct-target-pipeline $1-ct-pipeline: $(REBAR) - $(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) + $(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app)))) @@ -86,6 +88,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) + $(REL_PROFILES:%=%): $(REBAR) get-dashboard @$(REBAR) as $(@) do compile,release diff --git a/apps/emqx_auth_ldap/docker-ct b/apps/emqx_auth_ldap/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_mongo/docker-ct b/apps/emqx_auth_mongo/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_mysql/docker-ct b/apps/emqx_auth_mysql/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_pgsql/docker-ct b/apps/emqx_auth_pgsql/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_redis/docker-ct b/apps/emqx_auth_redis/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 9f188cf34..9ed84273a 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -7,11 +7,13 @@ cd -P -- "$(dirname -- "$0")/.." help() { echo - echo "-h|--help: To display this usage info" - echo "--json: Print apps in json" + echo "-h|--help: To display this usage info" + echo "--json: Print apps in json" + echo "-ct fast|docker: Only find apps for CT" } WANT_JSON='no' +CT='novalue' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -22,12 +24,17 @@ while [ "$#" -gt 0 ]; do WANT_JSON='yes' shift 1 ;; + --ct) + CT="$2" + shift 2 + ;; *) echo "unknown option $1" exit 1 ;; esac done + if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then # Otherwise windows may resolve to find.exe FIND="/usr/bin/find" @@ -41,25 +48,44 @@ find_app() { } EM="emqx" -CE="$(find_app 'apps')" - +SHARED="$(find_app 'apps')" if [ -f 'EMQX_ENTERPRISE' ]; then LIB="$(find_app 'lib-ee')" else LIB="$(find_app 'lib-ce')" fi - ## find directories in lib-extra LIBE="$(find_app 'lib-extra')" - ## find symlinks in lib-extra LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)" -APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")" +APPS_ALL="$(echo -e "${EM}\n${SHARED}\n${LIB}\n${LIBE}\n${LIBES}")" -if [ "$WANT_JSON" = 'yes' ]; then - echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")' +if [ "$CT" = 'novalue' ]; then + RESULT="${APPS_ALL}" else - echo "${APPS_ALL}" + APPS_NORMAL_CT=( ) + APPS_DOCKER_CT=( ) + for app in ${APPS_ALL}; do + if [ "$app" = 'apps/emqx_plugin_libs' ]; then + # This app has no test SUITE + continue + fi + if [ -f "${app}/docker-ct" ]; then + APPS_DOCKER_CT+=("$app") + else + APPS_NORMAL_CT+=("$app") + fi + done + if [ "$CT" = 'docker' ]; then + RESULT="${APPS_DOCKER_CT[*]}" + else + RESULT="${APPS_NORMAL_CT[*]}" + fi fi +if [ "$WANT_JSON" = 'yes' ]; then + echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")' +else + echo "${RESULT}" | xargs +fi From 90ff3a71f5626de7369703e47fea89b23577edba Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 6 Nov 2022 20:36:22 +0100 Subject: [PATCH 5/7] ci: mongo-replicaset and mongo-sharded are ee only --- .github/workflows/run_test_cases.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index afc122751..6f68469c3 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -174,13 +174,13 @@ jobs: -f .ci/docker-compose-file/docker-compose.yaml \ -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml" ee_docker_compose_files="\ -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \ From 568bd520a87b14ce98d26ea287abdf0fb965b235 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 7 Nov 2022 08:51:32 +0100 Subject: [PATCH 6/7] ci: ensure docker-ct for emqx_auth_http and emqx_web_hook --- apps/emqx_auth_http/docker-ct | 0 apps/emqx_web_hook/docker-ct | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 apps/emqx_auth_http/docker-ct create mode 100644 apps/emqx_web_hook/docker-ct diff --git a/apps/emqx_auth_http/docker-ct b/apps/emqx_auth_http/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_web_hook/docker-ct b/apps/emqx_web_hook/docker-ct new file mode 100644 index 000000000..e69de29bb From 4fb28783c5eacd5dcc151678e528d48369c4de88 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 7 Nov 2022 10:50:16 +0100 Subject: [PATCH 7/7] chore: bump 4.4 dashboard versions --- scripts/get-dashboard.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index d4fdd63d4..07bb60382 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -13,8 +13,8 @@ case "${PKG_VSN}" in ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! - EMQX_CE_DASHBOARD_VERSION='v4.4.6' - EMQX_EE_DASHBOARD_VERSION='v4.4.17' + EMQX_CE_DASHBOARD_VERSION='v4.4.7' + EMQX_EE_DASHBOARD_VERSION='v4.4.18' ;; *) echo "Unsupported version $PKG_VSN" >&2