diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index dfe2e03a3..88e1d9c9e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -98,7 +98,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }}-windows - path: source/_packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/ mac: needs: prepare @@ -133,7 +133,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: ${EMQX_NAME}-${{ matrix.otp }} - path: _packages/${EMQX_NAME}/. + path: _packages/${EMQX_NAME}/ linux: runs-on: ubuntu-20.04 @@ -209,7 +209,7 @@ jobs: - uses: actions/upload-artifact@v1 with: name: ${{ matrix.profile }}-${{ matrix.otp }} - path: source/_packages/${{ matrix.profile }}/. + path: source/_packages/${{ matrix.profile }}/ docker: runs-on: ubuntu-20.04 diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8bff3a33c..0a71b3c9c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -10,140 +10,255 @@ on: pull_request: jobs: - run_proper_test: - runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 + prepare: + runs-on: ubuntu-20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 + outputs: + ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }} + steps: + - uses: actions/checkout@v3 + with: + path: source + fetch-depth: 0 + - name: git credentials + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + fi + - name: find_ct_apps + working-directory: source + id: run_find_apps + # emqx_plugin_libs doesn't have a test suite -> excluded from app list + # emqx ct is run independently -> exclude it from the app list + run: | + ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . == "emqx"))')" + echo "ct-apps: $ct_apps" + echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT + - name: get_all_deps + working-directory: source + run: | + make deps-all + ./rebar3 as test compile + cd .. + zip -ryq source.zip source/* source/.[^.]* + - uses: actions/upload-artifact@v3 + with: + name: source + path: source.zip - steps: - - uses: actions/checkout@v2 - - name: set git credentials - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi - - name: proper - run: make proper + eunit_and_proper: + needs: prepare + runs-on: ubuntu-20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 + strategy: + fail-fast: false + matrix: + task: + - eunit + - proper + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: source + path: . + - name: unzip source code + run: unzip -o -q source.zip + # produces eunit.coverdata and proper.coverdata + - name: eunit and proper + working-directory: source + run: make ${{ matrix.task }} + - uses: actions/upload-artifact@v3 + with: + name: cover + path: source/_build/test/cover + if-no-files-found: warn - run_common_test: - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - matrix: - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false - steps: - - uses: actions/checkout@v2 - # to avoid dirty self-hosted runners - - name: stop containers - run: | - docker rm -f $(docker ps -qa) || true - docker network rm $(docker network ls -q) || true - - name: docker compose up - if: endsWith(github.repository, 'emqx') - env: - MYSQL_TAG: 8 - REDIS_TAG: 6 - MONGO_TAG: 4 - PGSQL_TAG: 13 - LDAP_TAG: 2.4.50 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - up -d --build - - name: docker compose up - if: endsWith(github.repository, 'emqx-enterprise') - env: - MYSQL_TAG: 8 - REDIS_TAG: 6 - MONGO_TAG: 4 - PGSQL_TAG: 13 - LDAP_TAG: 2.4.50 - OPENTSDB_TAG: latest - INFLUXDB_TAG: 1.7.6 - DYNAMODB_TAG: 1.11.477 - TIMESCALE_TAG: latest-pg11 - CASSANDRA_TAG: 3.11.6 - RABBITMQ_TAG: 3.7 - KAFKA_TAG: 2.5.0 - PULSAR_TAG: 2.3.2 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 20 - run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \ - up -d --build - docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" - docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 5 - done - - name: run eunit - run: | - docker exec -i erlang bash -c "make eunit" - - name: run common test - run: | - docker exec -i erlang bash -c "make ct" - - name: run cover - run: | - printenv > .env - docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - docker exec -i erlang bash -c "make cover" - docker exec --env-file .env -i erlang bash -c "make coveralls" - - name: cat rebar.crashdump - if: failure() - run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: _build/test/logs - - uses: actions/upload-artifact@v1 - with: - name: cover - path: _build/test/cover + emqx_ct: + needs: prepare + runs-on: ubuntu-20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 + strategy: + fail-fast: false + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: source + path: . + - name: unzip source code + run: unzip -o -q source.zip + # produces emqx-emqx.coverdata + - name: emqx-ct-pipeline + working-directory: source + run: make emqx-ct-pipeline + - uses: actions/upload-artifact@v3 + with: + name: cover + path: source/_build/test/cover + if-no-files-found: warn - finish: - needs: run_common_test - runs-on: ubuntu-20.04 - steps: - - name: Coveralls Finished - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - curl -v -k https://coveralls.io/webhook \ - --header "Content-Type: application/json" \ - --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true + ct: + needs: prepare + runs-on: ${{ matrix.runs-on }} + strategy: + max-parallel: 12 + fail-fast: false + matrix: + app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }} + runs-on: + - aws-amd64 + - ubuntu-20.04 + use-self-hosted: + - ${{ github.repository_owner == 'emqx' }} + exclude: + - runs-on: ubuntu-20.04 + use-self-hosted: true + - runs-on: aws-amd64 + use-self-hosted: false + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip + # to avoid dirty self-hosted runners + - name: stop containers + run: | + docker rm -f $(docker ps -qa) || true + docker network rm $(docker network ls -q) || true + - name: docker compose up + working-directory: source + if: endsWith(github.repository, 'emqx') + env: + MYSQL_TAG: 8 + REDIS_TAG: 6 + MONGO_TAG: 4 + PGSQL_TAG: 13 + LDAP_TAG: 2.4.50 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + docker-compose \ + -f .ci/docker-compose-file/docker-compose.yaml \ + -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ + -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ + up -d --build + docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" + - name: docker compose up + working-directory: source + if: endsWith(github.repository, 'emqx-enterprise') + env: + MYSQL_TAG: 8 + REDIS_TAG: 6 + MONGO_TAG: 4 + PGSQL_TAG: 13 + LDAP_TAG: 2.4.50 + OPENTSDB_TAG: latest + INFLUXDB_TAG: 1.7.6 + DYNAMODB_TAG: 1.11.477 + TIMESCALE_TAG: latest-pg11 + CASSANDRA_TAG: 3.11.6 + RABBITMQ_TAG: 3.7 + KAFKA_TAG: 2.5.0 + PULSAR_TAG: 2.3.2 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 20 + run: | + docker-compose \ + -f .ci/docker-compose-file/docker-compose.yaml \ + -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ + -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \ + -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \ + up -d --build + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" + while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ + != $(docker ps -a --filter name=client | wc -l) ]; do + sleep 5 + done + - name: run common test + run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline" + - name: cat rebar.crashdump + if: failure() + working-directory: source + run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi + - name: set log file name + if: failure() + run: echo "LOGFILENAME=logs-$(echo ${{ matrix.app_name }} | tr '/' '_')" >> $GITHUB_ENV + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: ${{ env.LOGFILENAME }} + path: source/_build/test/logs + if-no-files-found: warn + - uses: actions/upload-artifact@v3 + with: + name: cover + path: source/_build/test/cover + if-no-files-found: warn + + make_cover: + needs: + - eunit_and_proper + - emqx_ct + - ct + runs-on: ubuntu-20.04 + container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04 + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: source + path: . + - name: unzip source code + run: unzip -q source.zip + - uses: actions/download-artifact@v3 + name: download cover data + with: + name: cover + path: source/_build/test/cover + - name: make cover + working-directory: source + run: make cover + - name: send to coveralls + working-directory: source + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: make coveralls + - name: get coveralls logs + working-directory: source + if: failure() + run: cat rebar3.crashdump + + finish: + needs: make_cover + runs-on: ubuntu-20.04 + steps: + - name: Coveralls Finished + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + curl -v -k https://coveralls.io/webhook \ + --header "Content-Type: application/json" \ + --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true diff --git a/Makefile b/Makefile index 0d02f1b51..491e81031 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,14 @@ $1-ct: $(REBAR) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) +## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app +.PHONY: $(APPS:%=%-ct-pipeline) +define gen-app-ct-target-pipeline +$1-ct-pipeline: $(REBAR) + $(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) +endef +$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app)))) + ## apps/name-prop targets .PHONY: $(APPS:%=%-prop) define gen-app-prop-target diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 1739bd47a..e45903113 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -40,6 +40,7 @@ -define(RESOURCE_TYPE_MQTT, 'bridge_mqtt'). -define(RESOURCE_TYPE_RPC, 'bridge_rpc'). +-define(BAD_TOPIC_WITH_WILDCARD, wildcard_topic_not_allowed_for_publish). -define(RESOURCE_CONFIG_SPEC_MQTT, #{ address => #{ @@ -494,7 +495,7 @@ on_action_create_data_to_mqtt_broker(ActId, Opts = #{<<"pool">> := PoolName, PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl), TopicTks = case ForwardTopic == <<"">> of true -> undefined; - false -> emqx_rule_utils:preproc_tmpl(ForwardTopic) + false -> emqx_rule_utils:preproc_tmpl(assert_topic_valid(ForwardTopic)) end, Opts. @@ -515,7 +516,7 @@ on_action_data_to_mqtt_broker(Msg, _Env = qos = QoS, from = From, flags = Flags, - topic = Topic1, + topic = assert_topic_valid(Topic1), payload = format_data(PayloadTks, Msg), timestamp = TimeStamp}, ecpool:with_client(PoolName, @@ -583,7 +584,7 @@ options(Options, PoolName, ResId) -> Get = fun(Key) -> GetD(Key, undefined) end, Address = Get(<<"address">>), [{max_inflight_batches, 32}, - {forward_mountpoint, str(Get(<<"mountpoint">>))}, + {forward_mountpoint, str(assert_topic_valid(Get(<<"mountpoint">>)))}, {disk_cache, cuttlefish_flag:parse(str(GetD(<<"disk_cache">>, "off")))}, {start_type, auto}, {reconnect_delay_ms, cuttlefish_duration:parse(str(Get(<<"reconnect_interval">>)), ms)}, @@ -610,6 +611,12 @@ options(Options, PoolName, ResId) -> | maybe_ssl(Options, Get(<<"ssl">>), ResId)] end. +assert_topic_valid(T) -> + case emqx_topic:wildcard(T) of + true -> throw({?BAD_TOPIC_WITH_WILDCARD, T}); + false -> T + end. + maybe_ssl(_Options, false, _ResId) -> []; maybe_ssl(Options, true, ResId) -> diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index 1dbfc1583..ef6491b25 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -22,7 +22,7 @@ management.default_application.secret = public ## Initialize apps file ## Is used to add administrative app/secrets when EMQX is launched for the first time. -## This config will not take any effect once EMQX database is populated with the provided apps. +## This config will not take any effect once EMQX database has one or more apps. ## The file content format is as below: ## ``` ##819e5db182cf:l9C5suZClIF3FvdzWqmINrVU61WNfIjcglxw9CVM7y1VI diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index c3921140a..7d7a7245c 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -32,8 +32,11 @@ init_per_suite(Cfg) -> ok = emqx_dashboard_admin:mnesia(boot), application:load(emqx_modules), application:load(emqx_bridge_mqtt), + ekka_mnesia:start(), + emqx_dashboard_admin:mnesia(boot), emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]), application:ensure_all_started(emqx_dashboard), + ok = emqx_rule_engine:load_providers(), Cfg. end_per_suite(Cfg) -> diff --git a/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl index 7748c0224..e3894beb1 100644 --- a/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl @@ -41,6 +41,8 @@ init_per_suite(Config) -> Config. end_per_suite(_) -> + ok = application:unset_env(emqx_management, bootstrap_apps_file), + _ = mnesia:clear_table(mqtt_app), emqx_ct_helpers:stop_apps([]), ok. diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 557ddf423..67f68c99b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -22,6 +22,8 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-define(BAD_TOPIC_WITH_WILDCARD, wildcard_topic_not_allowed_for_publish). + -define(REPUBLISH_PARAMS_SPEC, #{ target_topic => #{ order => 1, @@ -163,7 +165,7 @@ on_action_create_republish(Id, Params = #{ }) -> TargetRetain = to_retain(maps:get(<<"target_retain">>, Params, <<"false">>)), TargetQoS = to_qos(TargetQoS0), - TopicTks = emqx_rule_utils:preproc_tmpl(TargetTopic), + TopicTks = emqx_rule_utils:preproc_tmpl(assert_topic_valid(TargetTopic)), PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl), Params. @@ -201,7 +203,7 @@ on_action_republish(Selected, _Envs = #{ from = ActId, flags = Flags#{retain => get_retain(TargetRetain, Selected)}, headers = #{republish_by => ActId}, - topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), + topic = assert_topic_valid(emqx_rule_utils:proc_tmpl(TopicTks, Selected)), payload = format_msg(PayloadTks, Selected), timestamp = Timestamp }, @@ -226,7 +228,7 @@ on_action_republish(Selected, _Envs = #{ from = ActId, flags = #{dup => false, retain => get_retain(TargetRetain, Selected)}, headers = #{republish_by => ActId}, - topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), + topic = assert_topic_valid(emqx_rule_utils:proc_tmpl(TopicTks, Selected)), payload = format_msg(PayloadTks, Selected), timestamp = erlang:system_time(millisecond) }, @@ -270,6 +272,12 @@ get_qos(-1, _Data, Default) -> Default; get_qos(TargetQoS, Data, _Default) -> qos(emqx_rule_utils:replace_var(TargetQoS, Data)). +assert_topic_valid(T) -> + case emqx_topic:wildcard(T) of + true -> throw({?BAD_TOPIC_WITH_WILDCARD, T}); + false -> T + end. + qos(<<"0">>) -> 0; qos(<<"1">>) -> 1; qos(<<"2">>) -> 2; diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index ba4b7ddbd..0004ce400 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -69,3 +69,5 @@ - Make sure Rule-Engine API supports Percent-encoding `rule_id` and `resource_id` in HTTP request path [#9190](https://github.com/emqx/emqx/pull/9190). Note that the `id` in `POST /api/v4/rules` should be literals (not encoded) when creating a `rule` or `resource`. See docs [Create Rule](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) [Create Resource](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources). + +- When republishing messages or bridge messages to other brokers, check the validity of the topic and make sure it does not have topic wildcards [#9291](https://github.com/emqx/emqx/pull/9291). diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index 0b245d698..cb1bf8e67 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -63,3 +63,5 @@ - 使规则引擎 API 在 HTTP 请求路径中支持百分号编码的 `rule_id` 及 `resource_id` [#9190](https://github.com/emqx/emqx/pull/9190)。 注意在创建规则或资源时,HTTP body 中的 `id` 字段仍为字面值,而不是编码之后的值。 详情请参考 [创建规则](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-rules) 和 [创建资源](https://www.emqx.io/docs/zh/v4.3/advanced/http-api.html#post-api-v4-resources)。 + +- 在进行消息重发布或桥接消息到其他 mqtt broker 时,检查 topic 合法性,确定其不带有主题通配符 [#9291](https://github.com/emqx/emqx/pull/9291)。 diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl index d2453bb2e..72b8e8602 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl @@ -23,6 +23,18 @@ all() -> emqx_ct:all(?MODULE). +init_per_suite(Config) -> + %% do not start the application + %% only testing the root supervisor in this suite + application:stop(emqx_modules), + {ok, Pid} = emqx_mod_sup:start_link(), + unlink(Pid), + Config. + +end_per_suite(_Config) -> + exit(whereis(emqx_mod_sup), kill), + ok. + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index c4a0eec62..9f188cf34 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -5,6 +5,29 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." +help() { + echo + echo "-h|--help: To display this usage info" + echo "--json: Print apps in json" +} + +WANT_JSON='no' +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --json) + WANT_JSON='yes' + shift 1 + ;; + *) + echo "unknown option $1" + exit 1 + ;; + esac +done if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then # Otherwise windows may resolve to find.exe FIND="/usr/bin/find" @@ -17,17 +40,26 @@ find_app() { "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } -# append emqx application first -echo 'emqx' +EM="emqx" +CE="$(find_app 'apps')" -find_app 'apps' if [ -f 'EMQX_ENTERPRISE' ]; then - find_app 'lib-ee' + LIB="$(find_app 'lib-ee')" else - find_app 'lib-ce' + LIB="$(find_app 'lib-ce')" fi ## find directories in lib-extra -find_app 'lib-extra' +LIBE="$(find_app 'lib-extra')" + ## find symlinks in lib-extra -"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print +LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)" + +APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")" + +if [ "$WANT_JSON" = 'yes' ]; then + echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")' +else + echo "${APPS_ALL}" +fi + diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index e887d35f4..0f0e7d33f 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -825,15 +825,18 @@ t_enrich_connack_caps(_) -> wildcard_subscription => true } end), - AckProps = emqx_channel:enrich_connack_caps(#{}, channel()), - ?assertMatch(#{'Retain-Available' := 1, - 'Maximum-Packet-Size' := 1024, - 'Topic-Alias-Maximum' := 10, - 'Wildcard-Subscription-Available' := 1, - 'Subscription-Identifier-Available' := 1, - 'Shared-Subscription-Available' := 1 - }, AckProps), - ok = meck:unload(emqx_mqtt_caps). + try + AckProps = emqx_channel:enrich_connack_caps(#{}, channel()), + ?assertMatch(#{'Retain-Available' := 1, + 'Maximum-Packet-Size' := 1024, + 'Topic-Alias-Maximum' := 10, + 'Wildcard-Subscription-Available' := 1, + 'Subscription-Identifier-Available' := 1, + 'Shared-Subscription-Available' := 1 + }, AckProps) + after + ok = meck:unload(emqx_mqtt_caps) + end. %%-------------------------------------------------------------------- %% Test cases for terminate diff --git a/test/emqx_mqtt_caps_SUITE.erl b/test/emqx_mqtt_caps_SUITE.erl index c2b1a477b..8afd6a3be 100644 --- a/test/emqx_mqtt_caps_SUITE.erl +++ b/test/emqx_mqtt_caps_SUITE.erl @@ -52,13 +52,15 @@ t_check_sub(_) -> wildcard_subscription => false }, emqx_zone:set_env(zone, '$mqtt_sub_caps', SubCaps), - timer:sleep(50), - ClientInfo = #{zone => zone}, - ok = emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts), - ?assertEqual({error, ?RC_TOPIC_FILTER_INVALID}, - emqx_mqtt_caps:check_sub(ClientInfo, <<"a/b/c/d">>, SubOpts)), - ?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(ClientInfo, <<"+/#">>, SubOpts)), - ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})), - emqx_zone:unset_env(zone, '$mqtt_pub_caps'). + try + ClientInfo = #{zone => zone}, + ok = emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts), + ?assertEqual({error, ?RC_TOPIC_FILTER_INVALID}, + emqx_mqtt_caps:check_sub(ClientInfo, <<"a/b/c/d">>, SubOpts)), + ?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED}, + emqx_mqtt_caps:check_sub(ClientInfo, <<"+/#">>, SubOpts)), + ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, + emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})) + after + emqx_zone:unset_env(zone, '$mqtt_pub_caps') + end. diff --git a/test/emqx_mqtt_protocol_v5_SUITE.erl b/test/emqx_mqtt_protocol_v5_SUITE.erl index 05424d642..33592b849 100644 --- a/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -468,8 +468,8 @@ t_connack_max_qos_allowed(_) -> %% max_qos_allowed = 0 emqx_zone:set_env(external, max_qos_allowed, 0), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_zone:unset_env(external, '$mqtt_caps'), + emqx_zone:unset_env(external, '$mqtt_pub_caps'), {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), {ok, Connack1} = emqtt:connect(Client1), @@ -496,8 +496,8 @@ t_connack_max_qos_allowed(_) -> %% max_qos_allowed = 1 emqx_zone:set_env(external, max_qos_allowed, 1), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_zone:unset_env(external, '$mqtt_caps'), + emqx_zone:unset_env(external, '$mqtt_pub_caps'), {ok, Client3} = emqtt:start_link([{proto_ver, v5}]), {ok, Connack3} = emqtt:connect(Client3), @@ -524,8 +524,8 @@ t_connack_max_qos_allowed(_) -> %% max_qos_allowed = 2 emqx_zone:set_env(external, max_qos_allowed, 2), - persistent_term:erase({emqx_zone, external, '$mqtt_caps'}), - persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}), + emqx_zone:unset_env(external, '$mqtt_caps'), + emqx_zone:unset_env(external, '$mqtt_pub_caps'), {ok, Client5} = emqtt:start_link([{proto_ver, v5}]), {ok, Connack5} = emqtt:connect(Client5), diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl index 5e15f858e..f787d70c1 100644 --- a/test/emqx_router_SUITE.erl +++ b/test/emqx_router_SUITE.erl @@ -32,6 +32,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> + emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:stop_apps([]). init_per_testcase(_TestCase, Config) -> diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 2112f0b8c..897a59cb6 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -51,7 +51,7 @@ init_per_suite(Config) -> PortDiscovery = application:get_env(gen_rpc, port_discovery), application:set_env(gen_rpc, port_discovery, stateless), application:ensure_all_started(gen_rpc), - %% ensure emqx_moduels' app modules are loaded + %% ensure emqx_modules app modules are loaded %% so the mnesia tables are created ok = load_app(emqx_modules), emqx_ct_helpers:start_apps([]), diff --git a/test/emqx_takeover_SUITE.erl b/test/emqx_takeover_SUITE.erl index 3317317c5..c7e3c9703 100644 --- a/test/emqx_takeover_SUITE.erl +++ b/test/emqx_takeover_SUITE.erl @@ -32,25 +32,44 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), emqx_ct_helpers:start_apps([]), Config. end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]), ok. + +init_per_testcase(Case, Config) -> + ?MODULE:Case({'init', Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + %%-------------------------------------------------------------------- %% Testcases -t_takeover(_) -> - process_flag(trap_exit, true), +t_takeover({init, Config}) when is_list(Config) -> + Config; +t_takeover({'end', Config}) when is_list(Config) -> + ok; +t_takeover(Config) when is_list(Config) -> AllMsgs = messages(?CNT), Pos = rand:uniform(?CNT), - - ClientId = <<"clientid">>, - {ok, C1} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]), - {ok, _} = emqtt:connect(C1), + ClientId = random_clientid(), + ClientOpts = [{clientid, ClientId}, + {clean_start, false}, + {host, "127.0.0.1"}, + {port, 1883} + ], + C1 = + with_retry( + fun() -> + {ok, C} = emqtt:start_link(ClientOpts), + {ok, _} = emqtt:connect(C), + C + end, 5), emqtt:subscribe(C1, <<"t">>, 1), - spawn(fun() -> [begin emqx:publish(lists:nth(I, AllMsgs)), @@ -59,31 +78,65 @@ t_takeover(_) -> end), emqtt:pause(C1), timer:sleep(?CNT*10), - load_meck(ClientId), - spawn(fun() -> - [begin - emqx:publish(lists:nth(I, AllMsgs)), - timer:sleep(rand:uniform(10)) - end || I <- lists:seq(Pos+1, ?CNT)] - end), - {ok, C2} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]), - {ok, _} = emqtt:connect(C2), - - Received = all_received_publishs(), - ct:pal("middle: ~p, received: ~p", [Pos, [P || {publish, #{payload := P}} <- Received]]), - assert_messages_missed(AllMsgs, Received), - assert_messages_order(AllMsgs, Received), - - emqtt:disconnect(C2), - unload_meck(ClientId). - -t_takover_in_cluster(_) -> - todo. + try + spawn(fun() -> + [begin + emqx:publish(lists:nth(I, AllMsgs)), + timer:sleep(rand:uniform(10)) + end || I <- lists:seq(Pos+1, ?CNT)] + end), + {ok, C2} = emqtt:start_link(ClientOpts), + %% C1 is going down, unlink it so the test can continue to run + _ = monitor(process, C1), + ?assert(erlang:is_process_alive(C1)), + unlink(C1), + {ok, _} = emqtt:connect(C2), + receive + {'DOWN', _, process, C1, _} -> + ok + after 1000 -> + ct:fail("timedout_waiting_for_old_connection_shutdown") + end, + Received = all_received_publishs(), + ct:pal("middle: ~p, received: ~p", [Pos, [P || {publish, #{payload := P}} <- Received]]), + assert_messages_missed(AllMsgs, Received), + assert_messages_order(AllMsgs, Received), + kill_process(C2, fun emqtt:stop/1) + after + unload_meck(ClientId) + end. %%-------------------------------------------------------------------- %% Helpers +random_clientid() -> + iolist_to_binary(["clientid", "-", integer_to_list(erlang:system_time())]). + +kill_process(Pid, WithFun) -> + _ = unlink(Pid), + _ = monitor(process, Pid), + try WithFun(Pid) + catch _:_ -> ok + end, + receive + {'DOWN', _, process, Pid, _} -> + ok + after 10_000 -> + exit(Pid, kill), + error(timeout) + end. + +with_retry(Fun, 1) -> Fun(); +with_retry(Fun, N) when N > 1 -> + try + Fun() + catch + _ : _ -> + ct:sleep(1000), + with_retry(Fun, N - 1) + end. + load_meck(ClientId) -> meck:new(fake_conn_mod, [non_strict]), HookTakeover = fun(Pid, Msg = {takeover, 'begin'}) ->