From 2d0d578b4f9343bd03c4b47ac0aff8d3aa317fb1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 28 Feb 2023 17:38:35 +0800 Subject: [PATCH 1/3] fix: delete ssl dirs on resource destory --- .../src/emqx_bridge_mqtt_actions.erl | 1 + .../src/emqx_plugin_libs_ssl.erl | 18 ++++++++++++++++++ .../src/emqx_web_hook_actions.erl | 1 + 3 files changed, 20 insertions(+) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index b43d44eac..67b5a7a2e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -482,6 +482,7 @@ on_resource_destroy(ResId, #{<<"pool">> := PoolName}) -> ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]), case ecpool:stop_sup_pool(PoolName) of ok -> + emqx_plugin_libs_ssl:maybe_delete_dir("rules", ResId), ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]); {error, Reason} -> ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]), diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index d863d6981..4526dfb37 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -21,6 +21,10 @@ save_file/2 ]). +-export([maybe_delete_dir/1, + maybe_delete_dir/2 + ]). + -type file_input_key() :: binary(). %% <<"file">> | <<"filename">> -type file_input() :: #{file_input_key() => binary()}. @@ -141,6 +145,20 @@ maybe_save_file(FileName, Content, Dir) -> error({"failed_to_save_ssl_file", FullFilename, Reason}) end. +maybe_delete_dir(SubDir, ResId) -> + Dir = filename:join([emqx:get_env(data_dir), SubDir, ResId]), + maybe_delete_dir(Dir). + +maybe_delete_dir(Dir) -> + case file:del_dir_r(Dir) of + ok -> + ok; + {error, enoent} -> + ok; + {error, Reason} -> + logger:error("Delete Resource dir ~p failed for reason: ~p", [Dir, Reason]) + end. + ensure_str(L) when is_list(L) -> L; ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index 351a8ba2b..e8e96748e 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -238,6 +238,7 @@ on_resource_destroy(ResId, #{<<"pool">> := PoolName}) -> ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]), case ehttpc_pool:stop_pool(PoolName) of ok -> + emqx_plugin_libs_ssl:maybe_delete_dir("rules", ResId), ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]); {error, Reason} -> ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_WEBHOOK, ResId, Reason]), From 3a735d8f5d962e215e7e1cfa26a78d39c50b4332 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 28 Feb 2023 17:42:33 +0800 Subject: [PATCH 2/3] chore: bump app.src and appup.src. update changes.md --- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 33 ++++++++++--------- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- .../emqx_web_hook/src/emqx_web_hook.appup.src | 8 +++-- changes/v4.4.15-en.md | 2 ++ changes/v4.4.15-zh.md | 2 ++ 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 2b59d095b..39eb2f0fe 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.4.6"}, + {vsn, "4.4.7"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index b1042d289..cbc9eaed3 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -1,14 +1,16 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {<<"4\\.4\\.[3-5]">>, + [{"4.4.6", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {<<"4\\.4\\.[3-5]">>, [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, - {"4.4.2",[ - {load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {"4.4.2", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, @@ -20,17 +22,16 @@ {update,emqx_slow_subs,{advanced,["4.4.0"]}}, {load_module,emqx_slow_subs_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [ - {<<"4\\.4\\.[3-5]">>, + [{"4.4.6", + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, + {<<"4\\.4\\.[3-5]">>, [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, - {"4.4.3", - [{load_module,emqx_trace,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}]}, {"4.4.2", - [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace,brutal_purge,soft_purge,[]}, - {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace,brutal_purge,soft_purge,[]}, + {load_module,emqx_trace_api,brutal_purge,soft_purge,[]}]}, {"4.4.1", [{load_module,emqx_plugin_libs_ssl,brutal_purge,soft_purge,[]}, {load_module,emqx_trace,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 34050490b..b3ac93ef2 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.16"}, % strict semver, bump manually! + {vsn, "4.3.17"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index b66a1eb96..b9ed1f687 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,7 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {"4.3.14", @@ -32,7 +34,9 @@ [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.15", + [{"4.3.16", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.15", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {"4.3.14", diff --git a/changes/v4.4.15-en.md b/changes/v4.4.15-en.md index 881e73ea6..e97cdf586 100644 --- a/changes/v4.4.15-en.md +++ b/changes/v4.4.15-en.md @@ -51,3 +51,5 @@ - fix the `{badarg,[{ets,lookup,[gproc,{shared, ...` error logs during shutdown [#9919](https://github.com/emqx/emqx/pull/9919). - Fix crash when updating a client's keepalive via the HTTP API if it connects with keepalive disabled [#9933](https://github.com/emqx/emqx/pull/9933). + +- Delete the files directory when `resources` were deleted to avoid files leaking [#10039](https://github.com/emqx/emqx/pull/10039). diff --git a/changes/v4.4.15-zh.md b/changes/v4.4.15-zh.md index 73c681706..20c2645b6 100644 --- a/changes/v4.4.15-zh.md +++ b/changes/v4.4.15-zh.md @@ -52,3 +52,5 @@ - 修复停止 EMQX 的时候，日志出现 `{badarg,[{ets,lookup,[gproc,{shared, ...` 错误的问题 [#9919](https://github.com/emqx/emqx/pull/9919)。 - 修复当客户端连接禁用 keepalive 时, 通过 HTTP API 更新其 keepalive 会崩溃的问题 [#9933](https://github.com/emqx/emqx/pull/9933)。 + +- 在 `资源` 删除时清理其文件目录以防止文件泄露 [#10039](https://github.com/emqx/emqx/pull/10039)。 From b03557082fea269f3606ec5d78bab9d2270b7d2e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 28 Feb 2023 13:28:27 +0100 Subject: [PATCH 3/3] ci: use latest version of docker compose --- .github/workflows/run_automate_tests.yaml | 8 ++++---- .github/workflows/run_cts_tests.yaml | 18 +++++++++--------- .github/workflows/run_test_cases.yaml | 4 ++-- deploy/docker/README.md | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index 6168089f9..8b6e33223 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -79,7 +79,7 @@ jobs: TARGET: emqx/${{ needs.build.outputs.imgname }} EMQX_TAG: ${{ needs.build.outputs.version }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ up -d --build - uses: actions/checkout@v3 @@ -180,7 +180,7 @@ jobs: EMQX_TAG: ${{ needs.build.outputs.version }} MYSQL_TAG: ${{ matrix.mysql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ up -d --build @@ -287,7 +287,7 @@ jobs: EMQX_TAG: ${{ needs.build.outputs.version }} PGSQL_TAG: ${{ matrix.pgsql_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ up -d --build @@ -383,7 +383,7 @@ jobs: EMQX_TAG: ${{ needs.build.outputs.version }} MYSQL_TAG: 8 run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml \ diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index cd204c37e..ed5ab7603 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -33,7 +33,7 @@ jobs: env: LDAP_TAG: ${{ matrix.ldap_tag }} run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build @@ -88,9 +88,9 @@ jobs: run: | docker rm -f $(docker ps -qa) || true docker network rm $(docker network ls -q) || true - - name: docker-compose up + - name: docker compose up run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ -f .ci/docker-compose-file/docker-compose-mongo-${{ matrix.connect_type }}.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ @@ -164,10 +164,10 @@ jobs: run: | docker rm -f $(docker ps -qa) || true docker network rm $(docker network ls -q) || true - - name: docker-compose up + - name: docker compose up timeout-minutes: 5 run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-mysql-${{ matrix.connect_type }}.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build @@ -254,9 +254,9 @@ jobs: run: | docker rm -f $(docker ps -qa) || true docker network rm $(docker network ls -q) || true - - name: docker-compose up + - name: docker compose up run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-pgsql-${{ matrix.connect_type }}.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build @@ -339,9 +339,9 @@ jobs: run: | docker rm -f $(docker ps -qa) || true docker network rm $(docker network ls -q) || true - - name: docker-compose up + - name: docker compose up run: | - docker-compose \ + docker compose \ -f .ci/docker-compose-file/docker-compose-redis-${{ matrix.node_type }}-${{ matrix.connect_type }}.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 38cd2f9bb..9a0ea738d 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -201,7 +201,7 @@ jobs: if [[ ${{ matrix.app_name }} == *ldap ]]; then docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml" fi - docker-compose $docker_compose_files up -d --build + docker compose $docker_compose_files up -d --build if [ -f EMQX_ENTERPRISE ]; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi @@ -211,7 +211,7 @@ jobs: docker ps -a --filter name=client echo "waiting for docker ${client} to exit" if ! timeout 60 docker wait "${client}"; then - docker-compose $docker_compose_files logs | tee docker-compose.log + docker compose $docker_compose_files logs | tee docker-compose.log exit 1 fi done diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 35eb68e82..85e7715ad 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -243,7 +243,7 @@ Let's create a static node list cluster from docker-compose. + Start the docker-compose cluster ```bash - docker-compose -p my_emqx up -d + docker compose -p my_emqx up -d ``` + View cluster