diff --git a/.github/workflows/.zipignore b/.github/workflows/.zipignore new file mode 100644 index 000000000..d72c7ba23 --- /dev/null +++ b/.github/workflows/.zipignore @@ -0,0 +1,3 @@ +.git/* +*/.git/* +*/.github/* diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 1172b437d..4a98f0808 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -137,9 +137,10 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make ensure-rebar3 - make ${PROFILE} - make test-compile - zip -ryq $PROFILE.zip . + make ${PROFILE}-compile test-compile + echo "PROFILE=${PROFILE}" | tee -a .env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env + zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 1132f2e1d..e2eff6dc7 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -146,7 +146,9 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make $PROFILE - zip -ryq $PROFILE.zip . + echo "PROFILE=${PROFILE}" | tee -a .env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env + zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - uses: actions/upload-artifact@v3 with: name: ${{ matrix.profile }} diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 2d7d98fba..ba24be6c2 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -68,6 +68,9 @@ on: type: string default: '5.2-3' +permissions: + contents: read + jobs: docker: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index d339bdf1c..a4bc58da2 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -20,6 +20,9 @@ on: required: true type: string +permissions: + contents: read + jobs: docker: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index e9f81994f..5b83ab063 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -7,6 +7,9 @@ on: required: true type: string +permissions: + contents: read + jobs: check_deps_integrity: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 94ca34f8c..3aad025db 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -8,6 +8,9 @@ on: ref: required: false +permissions: + contents: read + jobs: analyze: name: Analyze @@ -15,7 +18,6 @@ jobs: timeout-minutes: 360 permissions: actions: read - contents: read security-events: write container: image: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04 diff --git a/.github/workflows/green_master.yaml b/.github/workflows/green_master.yaml index 679062aea..0d938f6cd 100644 --- a/.github/workflows/green_master.yaml +++ b/.github/workflows/green_master.yaml @@ -7,6 +7,12 @@ on: # run hourly - cron: "0 * * * *" workflow_dispatch: + inputs: + ref: + required: false + +permissions: + contents: read jobs: rerun-failed-jobs: @@ -17,10 +23,16 @@ jobs: actions: write steps: - uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.ref || 'master' }} - name: run script shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - python3 scripts/rerun-failed-checks.py + gh api --method GET -f head_sha=$(git rev-parse HEAD) -f status=completed -f exclude_pull_requests=true /repos/emqx/emqx/actions/runs > runs.json + for id in $(jq -r '.workflow_runs[] | select((."conclusion" != "success") and .run_attempt < 3) | .id' runs.json); do + echo "rerun https://github.com/emqx/emqx/actions/runs/$id" + gh api --method POST /repos/emqx/emqx/actions/runs/$id/rerun-failed-jobs + done diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index ace84fd5f..1b6101f06 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -19,6 +19,9 @@ env: TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }} SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }} +permissions: + contents: read + jobs: prepare: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ab145a764..b23f91128 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,9 +13,14 @@ on: required: true default: false +permissions: + contents: read + jobs: upload: runs-on: ubuntu-22.04 + permissions: + packages: write strategy: fail-fast: false steps: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index 2b2b47d7e..fc12787a8 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -11,12 +11,13 @@ on: required: true type: string +permissions: + contents: read + jobs: run_conf_tests: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} container: ${{ inputs.builder }} - env: - PROFILE: ${{ matrix.profile }} strategy: fail-fast: false matrix: @@ -31,6 +32,8 @@ jobs: run: | unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" + - run: cat .env | tee -a $GITHUB_ENV + - run: make ${{ matrix.profile }} - run: ./scripts/test/check-example-configs.sh - run: ./scripts/conf-test/run.sh - name: print erlang log diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 8b8996dfb..a36806e9e 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -14,6 +14,9 @@ on: required: true type: string +permissions: + contents: read + jobs: basic-tests: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 8672b9bdb..88e8e951a 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -23,6 +23,9 @@ on: env: IS_CI: "yes" +permissions: + contents: read + jobs: run_emqx_app_tests: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/run_helm_tests.yaml b/.github/workflows/run_helm_tests.yaml index beae8ddc8..e191100c4 100644 --- a/.github/workflows/run_helm_tests.yaml +++ b/.github/workflows/run_helm_tests.yaml @@ -14,6 +14,9 @@ on: required: true type: string +permissions: + contents: read + jobs: helm_test: runs-on: ${{ github.repository_owner == 'emqx' && fromJSON('["self-hosted","ephemeral","linux","x64"]') || 'ubuntu-22.04' }} diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 54b0f7625..381e95753 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -11,6 +11,9 @@ on: required: true type: string +permissions: + contents: read + jobs: relup_test_plan: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 000000000..29f6610ce --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,51 @@ +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + - cron: '25 21 * * 6' + push: + branches: [ "master" ] + workflow_dispatch: + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 + with: + results_file: results.sarif + results_format: sarif + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + - name: "Upload artifact" + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@8e0b1c74b1d5a0077b04d064c76ee714d3da7637 # v2.22.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml index 0b6f0dad1..4fecadd31 100644 --- a/.github/workflows/spellcheck.yaml +++ b/.github/workflows/spellcheck.yaml @@ -7,6 +7,9 @@ concurrency: on: workflow_call: +permissions: + contents: read + jobs: spellcheck: strategy: diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 69ac3d95c..5dcb4a5fa 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -8,6 +8,9 @@ on: - cron: "0 * * * *" workflow_dispatch: +permissions: + contents: read + jobs: stale: if: github.repository_owner == 'emqx' diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index a61719559..29c8384a0 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -17,6 +17,9 @@ on: env: IS_CI: "yes" +permissions: + contents: read + jobs: static_checks: runs-on: ${{ github.repository_owner == 'emqx' && fromJSON('["self-hosted","ephemeral","linux","x64"]') || 'ubuntu-22.04' }} @@ -37,10 +40,9 @@ jobs: - uses: actions/cache@v3 with: path: "emqx_dialyzer_${{ matrix.otp }}_plt" - key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*', 'lib-ee/*/rebar.*') }} + key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }} restore-keys: | rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}- + - run: cat .env | tee -a $GITHUB_ENV - name: run static checks - env: - PROFILE: ${{ matrix.profile }} run: make static_checks diff --git a/Makefile b/Makefile index d0d9127b8..0112776bb 100644 --- a/Makefile +++ b/Makefile @@ -315,8 +315,10 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) .PHONY: fmt fmt: $(REBAR) - @$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}' + @$(SCRIPTS)/erlfmt -w 'apps/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}' @$(SCRIPTS)/erlfmt -w 'rebar.config.erl' + @$(SCRIPTS)/erlfmt -w '$(SCRIPTS)/**/*.escript' + @$(SCRIPTS)/erlfmt -w 'bin/**/*.escript' @mix format .PHONY: clean-test-cluster-config diff --git a/apps/emqx/include/bpapi.hrl b/apps/emqx/include/bpapi.hrl index 1373e0381..ed7693e78 100644 --- a/apps/emqx/include/bpapi.hrl +++ b/apps/emqx/include/bpapi.hrl @@ -14,9 +14,4 @@ %% limitations under the License. %%-------------------------------------------------------------------- --ifndef(EMQX_BPAPI_HRL). --define(EMQX_BPAPI_HRL, true). - --compile({parse_transform, emqx_bpapi_trans}). - --endif. +-include_lib("emqx_utils/include/bpapi.hrl"). diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 664ec5803..86a64d8bb 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -55,29 +55,7 @@ -record(subscription, {topic, subid, subopts}). -%% See 'Application Message' in MQTT Version 5.0 --record(message, { - %% Global unique message ID - id :: binary(), - %% Message QoS - qos = 0, - %% Message from - from :: atom() | binary(), - %% Message flags - flags = #{} :: emqx_types:flags(), - %% Message headers. May contain any metadata. e.g. the - %% protocol version number, username, peerhost or - %% the PUBLISH properties (MQTT 5.0). - headers = #{} :: emqx_types:headers(), - %% Topic that the message is published to - topic :: emqx_types:topic(), - %% Message Payload - payload :: emqx_types:payload(), - %% Timestamp (Unit: millisecond) - timestamp :: integer(), - %% not used so far, for future extension - extra = [] :: term() -}). +-include_lib("emqx_utils/include/emqx_message.hrl"). -record(delivery, { %% Sender of the delivery diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index 7b2ce6c6b..1db80c72d 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -19,67 +19,79 @@ -define(PH_VAR_THIS, <<"$_THIS_">>). --define(PH(Type), <<"${", Type/binary, "}">>). +-define(PH(Var), <<"${" Var "}">>). %% action: publish/subscribe --define(PH_ACTION, <<"${action}">>). +-define(VAR_ACTION, "action"). +-define(PH_ACTION, ?PH(?VAR_ACTION)). %% cert --define(PH_CERT_SUBJECT, <<"${cert_subject}">>). --define(PH_CERT_CN_NAME, <<"${cert_common_name}">>). +-define(VAR_CERT_SUBJECT, "cert_subject"). +-define(VAR_CERT_CN_NAME, "cert_common_name"). +-define(PH_CERT_SUBJECT, ?PH(?VAR_CERT_SUBJECT)). +-define(PH_CERT_CN_NAME, ?PH(?VAR_CERT_CN_NAME)). %% MQTT --define(PH_PASSWORD, <<"${password}">>). --define(PH_CLIENTID, <<"${clientid}">>). --define(PH_FROM_CLIENTID, <<"${from_clientid}">>). --define(PH_USERNAME, <<"${username}">>). --define(PH_FROM_USERNAME, <<"${from_username}">>). --define(PH_TOPIC, <<"${topic}">>). +-define(VAR_PASSWORD, "password"). +-define(VAR_CLIENTID, "clientid"). +-define(VAR_USERNAME, "username"). +-define(VAR_TOPIC, "topic"). +-define(PH_PASSWORD, ?PH(?VAR_PASSWORD)). +-define(PH_CLIENTID, ?PH(?VAR_CLIENTID)). +-define(PH_FROM_CLIENTID, ?PH("from_clientid")). +-define(PH_USERNAME, ?PH(?VAR_USERNAME)). +-define(PH_FROM_USERNAME, ?PH("from_username")). +-define(PH_TOPIC, ?PH(?VAR_TOPIC)). %% MQTT payload --define(PH_PAYLOAD, <<"${payload}">>). +-define(PH_PAYLOAD, ?PH("payload")). %% client IPAddress --define(PH_PEERHOST, <<"${peerhost}">>). +-define(VAR_PEERHOST, "peerhost"). +-define(PH_PEERHOST, ?PH(?VAR_PEERHOST)). %% ip & port --define(PH_HOST, <<"${host}">>). --define(PH_PORT, <<"${port}">>). +-define(PH_HOST, ?PH("host")). +-define(PH_PORT, ?PH("port")). %% Enumeration of message QoS 0,1,2 --define(PH_QOS, <<"${qos}">>). --define(PH_FLAGS, <<"${flags}">>). +-define(VAR_QOS, "qos"). +-define(PH_QOS, ?PH(?VAR_QOS)). +-define(PH_FLAGS, ?PH("flags")). %% Additional data related to process within the MQTT message --define(PH_HEADERS, <<"${headers}">>). +-define(PH_HEADERS, ?PH("headers")). %% protocol name --define(PH_PROTONAME, <<"${proto_name}">>). +-define(VAR_PROTONAME, "proto_name"). +-define(PH_PROTONAME, ?PH(?VAR_PROTONAME)). %% protocol version --define(PH_PROTOVER, <<"${proto_ver}">>). +-define(PH_PROTOVER, ?PH("proto_ver")). %% MQTT keepalive interval --define(PH_KEEPALIVE, <<"${keepalive}">>). +-define(PH_KEEPALIVE, ?PH("keepalive")). %% MQTT clean_start --define(PH_CLEAR_START, <<"${clean_start}">>). +-define(PH_CLEAR_START, ?PH("clean_start")). %% MQTT Session Expiration time --define(PH_EXPIRY_INTERVAL, <<"${expiry_interval}">>). +-define(PH_EXPIRY_INTERVAL, ?PH("expiry_interval")). %% Time when PUBLISH message reaches Broker (ms) --define(PH_PUBLISH_RECEIVED_AT, <<"${publish_received_at}">>). +-define(PH_PUBLISH_RECEIVED_AT, ?PH("publish_received_at")). %% Mountpoint for bridging messages --define(PH_MOUNTPOINT, <<"${mountpoint}">>). +-define(VAR_MOUNTPOINT, "mountpoint"). +-define(PH_MOUNTPOINT, ?PH(?VAR_MOUNTPOINT)). %% IPAddress and Port of terminal --define(PH_PEERNAME, <<"${peername}">>). +-define(PH_PEERNAME, ?PH("peername")). %% IPAddress and Port listened by emqx --define(PH_SOCKNAME, <<"${sockname}">>). +-define(PH_SOCKNAME, ?PH("sockname")). %% whether it is MQTT bridge connection --define(PH_IS_BRIDGE, <<"${is_bridge}">>). +-define(PH_IS_BRIDGE, ?PH("is_bridge")). %% Terminal connection completion time (s) --define(PH_CONNECTED_AT, <<"${connected_at}">>). +-define(PH_CONNECTED_AT, ?PH("connected_at")). %% Event trigger time(millisecond) --define(PH_TIMESTAMP, <<"${timestamp}">>). +-define(PH_TIMESTAMP, ?PH("timestamp")). %% Terminal disconnection completion time (s) --define(PH_DISCONNECTED_AT, <<"${disconnected_at}">>). +-define(PH_DISCONNECTED_AT, ?PH("disconnected_at")). --define(PH_NODE, <<"${node}">>). --define(PH_REASON, <<"${reason}">>). +-define(PH_NODE, ?PH("node")). +-define(PH_REASON, ?PH("reason")). --define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>). --define(PH_RETAIN, <<"${retain}">>). +-define(PH_ENDPOINT_NAME, ?PH("endpoint_name")). +-define(VAR_RETAIN, "retain"). +-define(PH_RETAIN, ?PH(?VAR_RETAIN)). %% sync change these place holder with binary def. -define(PH_S_ACTION, "${action}"). diff --git a/apps/emqx/integration_test/emqx_ds_SUITE.erl b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl similarity index 61% rename from apps/emqx/integration_test/emqx_ds_SUITE.erl rename to apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl index 34c15b505..f22a4f97e 100644 --- a/apps/emqx/integration_test/emqx_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ds_SUITE). +-module(emqx_persistent_session_ds_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -11,10 +11,11 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/src/emqx_persistent_session_ds.hrl"). + -define(DEFAULT_KEYSPACE, default). -define(DS_SHARD_ID, <<"local">>). -define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). --define(ITERATOR_REF_TAB, emqx_ds_iterator_ref). -import(emqx_common_test_helpers, [on_exit/1]). @@ -91,9 +92,6 @@ get_mqtt_port(Node, Type) -> {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), Port. -get_all_iterator_refs(Node) -> - erpc:call(Node, mnesia, dirty_all_keys, [?ITERATOR_REF_TAB]). - get_all_iterator_ids(Node) -> Fn = fun(K, _V, Acc) -> [K | Acc] end, erpc:call(Node, fun() -> @@ -122,10 +120,40 @@ start_client(Opts0 = #{}) -> properties => #{'Session-Expiry-Interval' => 300} }, Opts = maps:to_list(emqx_utils_maps:deep_merge(Defaults, Opts0)), + ct:pal("starting client with opts:\n ~p", [Opts]), {ok, Client} = emqtt:start_link(Opts), on_exit(fun() -> catch emqtt:stop(Client) end), Client. +restart_node(Node, NodeSpec) -> + ?tp(will_restart_node, #{}), + ?tp(notice, "restarting node", #{node => Node}), + true = monitor_node(Node, true), + ok = erpc:call(Node, init, restart, []), + receive + {nodedown, Node} -> + ok + after 10_000 -> + ct:fail("node ~p didn't stop", [Node]) + end, + ?tp(notice, "waiting for nodeup", #{node => Node}), + wait_nodeup(Node), + wait_gen_rpc_down(NodeSpec), + ?tp(notice, "restarting apps", #{node => Node}), + Apps = maps:get(apps, NodeSpec), + ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]), + _ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]), + %% have to re-inject this so that we may stop the node succesfully at the + %% end.... + ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec), + ok = snabbkaffe:forward_trace(Node), + ?tp(notice, "node restarted", #{node => Node}), + ?tp(restarted_node, #{}), + ok. + +is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}}) -> + EI > 0. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -143,24 +171,14 @@ t_non_persistent_session_subscription(_Config) -> {ok, _} = emqtt:connect(Client), ?tp(notice, "subscribing", #{}), {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client, SubTopicFilter, qos2), - IteratorRefs = get_all_iterator_refs(node()), - IteratorIds = get_all_iterator_ids(node()), ok = emqtt:stop(Client), - #{ - iterator_refs => IteratorRefs, - iterator_ids => IteratorIds - } + ok end, - fun(Res, Trace) -> + fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), - #{ - iterator_refs := IteratorRefs, - iterator_ids := IteratorIds - } = Res, - ?assertEqual([], IteratorRefs), - ?assertEqual({ok, []}, IteratorIds), + ?assertEqual([], ?of_kind(ds_session_subscription_added, Trace)), ok end ), @@ -175,7 +193,7 @@ t_session_subscription_idempotency(Config) -> ?check_trace( begin ?force_ordering( - #{?snk_kind := persistent_session_ds_iterator_added}, + #{?snk_kind := persistent_session_ds_subscription_added}, _NEvents0 = 1, #{?snk_kind := will_restart_node}, _Guard0 = true @@ -187,32 +205,7 @@ t_session_subscription_idempotency(Config) -> _Guard1 = true ), - spawn_link(fun() -> - ?tp(will_restart_node, #{}), - ?tp(notice, "restarting node", #{node => Node1}), - true = monitor_node(Node1, true), - ok = erpc:call(Node1, init, restart, []), - receive - {nodedown, Node1} -> - ok - after 10_000 -> - ct:fail("node ~p didn't stop", [Node1]) - end, - ?tp(notice, "waiting for nodeup", #{node => Node1}), - wait_nodeup(Node1), - wait_gen_rpc_down(Node1Spec), - ?tp(notice, "restarting apps", #{node => Node1}), - Apps = maps:get(apps, Node1Spec), - ok = erpc:call(Node1, emqx_cth_suite, load_apps, [Apps]), - _ = erpc:call(Node1, emqx_cth_suite, start_apps, [Apps, Node1Spec]), - %% have to re-inject this so that we may stop the node succesfully at the - %% end.... - ok = emqx_cth_cluster:set_node_opts(Node1, Node1Spec), - ok = snabbkaffe:forward_trace(Node1), - ?tp(notice, "node restarted", #{node => Node1}), - ?tp(restarted_node, #{}), - ok - end), + spawn_link(fun() -> restart_node(Node1, Node1Spec) end), ?tp(notice, "starting 1", #{}), Client0 = start_client(#{port => Port, clientid => ClientId}), @@ -223,7 +216,7 @@ t_session_subscription_idempotency(Config) -> receive {'EXIT', {shutdown, _}} -> ok - after 0 -> ok + after 100 -> ok end, process_flag(trap_exit, false), @@ -240,10 +233,7 @@ t_session_subscription_idempotency(Config) -> end, fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), - %% Exactly one iterator should have been opened. SubTopicFilterWords = emqx_topic:words(SubTopicFilter), - ?assertEqual([{ClientId, SubTopicFilterWords}], get_all_iterator_refs(Node1)), - ?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)), ?assertMatch( {ok, #{}, #{SubTopicFilterWords := #{}}}, erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId]) @@ -262,7 +252,10 @@ t_session_unsubscription_idempotency(Config) -> ?check_trace( begin ?force_ordering( - #{?snk_kind := persistent_session_ds_close_iterators, ?snk_span := {complete, _}}, + #{ + ?snk_kind := persistent_session_ds_subscription_delete, + ?snk_span := {complete, _} + }, _NEvents0 = 1, #{?snk_kind := will_restart_node}, _Guard0 = true @@ -270,36 +263,11 @@ t_session_unsubscription_idempotency(Config) -> ?force_ordering( #{?snk_kind := restarted_node}, _NEvents1 = 1, - #{?snk_kind := persistent_session_ds_iterator_delete, ?snk_span := start}, + #{?snk_kind := persistent_session_ds_subscription_route_delete, ?snk_span := start}, _Guard1 = true ), - spawn_link(fun() -> - ?tp(will_restart_node, #{}), - ?tp(notice, "restarting node", #{node => Node1}), - true = monitor_node(Node1, true), - ok = erpc:call(Node1, init, restart, []), - receive - {nodedown, Node1} -> - ok - after 10_000 -> - ct:fail("node ~p didn't stop", [Node1]) - end, - ?tp(notice, "waiting for nodeup", #{node => Node1}), - wait_nodeup(Node1), - wait_gen_rpc_down(Node1Spec), - ?tp(notice, "restarting apps", #{node => Node1}), - Apps = maps:get(apps, Node1Spec), - ok = erpc:call(Node1, emqx_cth_suite, load_apps, [Apps]), - _ = erpc:call(Node1, emqx_cth_suite, start_apps, [Apps, Node1Spec]), - %% have to re-inject this so that we may stop the node succesfully at the - %% end.... - ok = emqx_cth_cluster:set_node_opts(Node1, Node1Spec), - ok = snabbkaffe:forward_trace(Node1), - ?tp(notice, "node restarted", #{node => Node1}), - ?tp(restarted_node, #{}), - ok - end), + spawn_link(fun() -> restart_node(Node1, Node1Spec) end), ?tp(notice, "starting 1", #{}), Client0 = start_client(#{port => Port, clientid => ClientId}), @@ -312,7 +280,7 @@ t_session_unsubscription_idempotency(Config) -> receive {'EXIT', {shutdown, _}} -> ok - after 0 -> ok + after 100 -> ok end, process_flag(trap_exit, false), @@ -327,7 +295,7 @@ t_session_unsubscription_idempotency(Config) -> ?wait_async_action( emqtt:unsubscribe(Client1, SubTopicFilter), #{ - ?snk_kind := persistent_session_ds_iterator_delete, + ?snk_kind := persistent_session_ds_subscription_route_delete, ?snk_span := {complete, _} }, 15_000 @@ -339,9 +307,101 @@ t_session_unsubscription_idempotency(Config) -> end, fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), - %% No iterators remaining - ?assertEqual([], get_all_iterator_refs(Node1)), - ?assertEqual({ok, []}, get_all_iterator_ids(Node1)), + ?assertMatch( + {ok, #{}, Subs = #{}} when map_size(Subs) =:= 0, + erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId]) + ), + ok + end + ), + ok. + +t_session_discard_persistent_to_non_persistent(_Config) -> + ClientId = atom_to_binary(?FUNCTION_NAME), + Params = #{ + client_id => ClientId, + reconnect_opts => + #{ + clean_start => true, + %% we set it to zero so that a new session is not created. + properties => #{'Session-Expiry-Interval' => 0}, + proto_ver => v5 + } + }, + do_t_session_discard(Params). + +t_session_discard_persistent_to_persistent(_Config) -> + ClientId = atom_to_binary(?FUNCTION_NAME), + Params = #{ + client_id => ClientId, + reconnect_opts => + #{ + clean_start => true, + properties => #{'Session-Expiry-Interval' => 30}, + proto_ver => v5 + } + }, + do_t_session_discard(Params). + +do_t_session_discard(Params) -> + #{ + client_id := ClientId, + reconnect_opts := ReconnectOpts0 + } = Params, + ReconnectOpts = ReconnectOpts0#{clientid => ClientId}, + SubTopicFilter = <<"t/+">>, + ?check_trace( + begin + ?tp(notice, "starting", #{}), + Client0 = start_client(#{ + clientid => ClientId, + clean_start => false, + properties => #{'Session-Expiry-Interval' => 30}, + proto_ver => v5 + }), + {ok, _} = emqtt:connect(Client0), + ?tp(notice, "subscribing", #{}), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, SubTopicFilter, qos2), + %% Store some matching messages so that streams and iterators are created. + ok = emqtt:publish(Client0, <<"t/1">>, <<"1">>), + ok = emqtt:publish(Client0, <<"t/2">>, <<"2">>), + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = map_size(emqx_persistent_session_ds:list_all_streams()) > 0 + ), + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = map_size(emqx_persistent_session_ds:list_all_iterators()) > 0 + ), + ok = emqtt:stop(Client0), + ?tp(notice, "disconnected", #{}), + + ?tp(notice, "reconnecting", #{}), + %% we still have iterators and streams + ?assert(map_size(emqx_persistent_session_ds:list_all_streams()) > 0), + ?assert(map_size(emqx_persistent_session_ds:list_all_iterators()) > 0), + Client1 = start_client(ReconnectOpts), + {ok, _} = emqtt:connect(Client1), + ?assertEqual([], emqtt:subscriptions(Client1)), + case is_persistent_connect_opts(ReconnectOpts) of + true -> + ?assertMatch(#{ClientId := _}, emqx_persistent_session_ds:list_all_sessions()); + false -> + ?assertEqual(#{}, emqx_persistent_session_ds:list_all_sessions()) + end, + ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()), + ?assertEqual([], emqx_persistent_session_ds_router:topics()), + ?assertEqual(#{}, emqx_persistent_session_ds:list_all_streams()), + ?assertEqual(#{}, emqx_persistent_session_ds:list_all_iterators()), + ok = emqtt:stop(Client1), + ?tp(notice, "disconnected", #{}), + + ok + end, + fun(Trace) -> + ct:pal("trace:\n ~p", [Trace]), ok end ), diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 47967cb1e..7042f5186 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -14,10 +14,11 @@ {emqx_conf,1}. {emqx_conf,2}. {emqx_conf,3}. -{emqx_connector, 1}. +{emqx_connector,1}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_delayed,2}. +{emqx_ds,1}. {emqx_eviction_agent,1}. {emqx_eviction_agent,2}. {emqx_exhook,1}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9f67caf5d..71f581267 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.19"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index f8a02db2e..0545f36a5 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.13"}, + {vsn, "5.1.14"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 1e4940965..537c60876 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -258,21 +258,21 @@ set_chan_stats(ClientId, ChanPid, Stats) -> end. %% @doc Open a session. --spec open_session(boolean(), emqx_types:clientinfo(), emqx_types:conninfo()) -> +-spec open_session(_CleanStart :: boolean(), emqx_types:clientinfo(), emqx_types:conninfo()) -> {ok, #{ session := emqx_session:t(), present := boolean(), replay => _ReplayContext }} | {error, Reason :: term()}. -open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> +open_session(_CleanStart = true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), emqx_cm_locker:trans(ClientId, fun(_) -> ok = discard_session(ClientId), ok = emqx_session:destroy(ClientInfo, ConnInfo), create_register_session(ClientInfo, ConnInfo, Self) end); -open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> +open_session(_CleanStart = false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), emqx_cm_locker:trans(ClientId, fun(_) -> case emqx_session:open(ClientInfo, ConnInfo) of diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index c3e9c2230..efe2c0de8 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -66,8 +66,9 @@ %% - Callbacks with greater priority values will be run before %% the ones with lower priority values. e.g. A Callback with %% priority = 2 precedes the callback with priority = 1. -%% - The execution order is the adding order of callbacks if they have -%% equal priority values. +%% - If the priorities of the hooks are equal then their execution +%% order is determined by the lexicographic of hook function +%% names. -type hookpoint() :: atom() | binary(). -type action() :: {module(), atom(), [term()] | undefined}. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 802b29837..09ab6099c 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -33,7 +33,8 @@ desc/1, types/0, short_paths/0, - short_paths_fields/1 + short_paths_fields/0, + rate_type/0 ]). -define(KILOBYTE, 1024). @@ -103,11 +104,11 @@ roots() -> ]. fields(limiter) -> - short_paths_fields(?MODULE, ?IMPORTANCE_HIDDEN) ++ + short_paths_fields(?IMPORTANCE_HIDDEN) ++ [ {Type, ?HOCON(?R_REF(node_opts), #{ - desc => ?DESC(Type), + desc => deprecated_desc(Type), importance => ?IMPORTANCE_HIDDEN, required => {false, recursively}, aliases => alias_of_type(Type) @@ -120,7 +121,7 @@ fields(limiter) -> ?HOCON( ?R_REF(client_fields), #{ - desc => ?DESC(client), + desc => deprecated_desc(client), importance => ?IMPORTANCE_HIDDEN, required => {false, recursively}, deprecated => {since, "5.0.25"} @@ -129,10 +130,10 @@ fields(limiter) -> ]; fields(node_opts) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, + {rate, ?HOCON(rate_type(), #{desc => deprecated_desc(rate), default => <<"infinity">>})}, {burst, - ?HOCON(burst_rate(), #{ - desc => ?DESC(burst), + ?HOCON(burst_rate_type(), #{ + desc => deprecated_desc(burst), default => <<"0">> })} ]; @@ -142,11 +143,12 @@ fields(bucket_opts) -> fields_of_bucket(<<"infinity">>); fields(client_opts) -> [ - {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, + {rate, ?HOCON(rate_type(), #{default => <<"infinity">>, desc => deprecated_desc(rate)})}, {initial, ?HOCON(initial(), #{ default => <<"0">>, - desc => ?DESC(initial), + + desc => deprecated_desc(initial), importance => ?IMPORTANCE_HIDDEN })}, %% low_watermark add for emqx_channel and emqx_session @@ -157,14 +159,14 @@ fields(client_opts) -> ?HOCON( initial(), #{ - desc => ?DESC(low_watermark), + desc => deprecated_desc(low_watermark), default => <<"0">>, importance => ?IMPORTANCE_HIDDEN } )}, {burst, - ?HOCON(burst(), #{ - desc => ?DESC(burst), + ?HOCON(burst_type(), #{ + desc => deprecated_desc(burst), default => <<"0">>, importance => ?IMPORTANCE_HIDDEN, aliases => [capacity] @@ -173,7 +175,7 @@ fields(client_opts) -> ?HOCON( boolean(), #{ - desc => ?DESC(divisible), + desc => deprecated_desc(divisible), default => true, importance => ?IMPORTANCE_HIDDEN } @@ -182,7 +184,7 @@ fields(client_opts) -> ?HOCON( emqx_schema:timeout_duration(), #{ - desc => ?DESC(max_retry_time), + desc => deprecated_desc(max_retry_time), default => <<"1h">>, importance => ?IMPORTANCE_HIDDEN } @@ -191,7 +193,7 @@ fields(client_opts) -> ?HOCON( failure_strategy(), #{ - desc => ?DESC(failure_strategy), + desc => deprecated_desc(failure_strategy), default => force, importance => ?IMPORTANCE_HIDDEN } @@ -204,14 +206,14 @@ fields(listener_client_fields) -> fields(Type) -> simple_bucket_field(Type). -short_paths_fields(DesModule) -> - short_paths_fields(DesModule, ?DEFAULT_IMPORTANCE). +short_paths_fields() -> + short_paths_fields(?DEFAULT_IMPORTANCE). -short_paths_fields(DesModule, Importance) -> +short_paths_fields(Importance) -> [ {Name, - ?HOCON(rate(), #{ - desc => ?DESC(DesModule, Name), + ?HOCON(rate_type(), #{ + desc => ?DESC(Name), required => false, importance => Importance, example => Example @@ -381,7 +383,7 @@ simple_bucket_field(Type) when is_atom(Type) -> ?HOCON( ?R_REF(?MODULE, client_opts), #{ - desc => ?DESC(client), + desc => deprecated_desc(client), required => {false, recursively}, importance => importance_of_type(Type), aliases => alias_of_type(Type) @@ -394,7 +396,7 @@ composite_bucket_fields(Types, ClientRef) -> [ {Type, ?HOCON(?R_REF(?MODULE, bucket_opts), #{ - desc => ?DESC(?MODULE, Type), + desc => deprecated_desc(Type), required => {false, recursively}, importance => importance_of_type(Type), aliases => alias_of_type(Type) @@ -406,7 +408,7 @@ composite_bucket_fields(Types, ClientRef) -> ?HOCON( ?R_REF(?MODULE, ClientRef), #{ - desc => ?DESC(client), + desc => deprecated_desc(client), required => {false, recursively} } )} @@ -414,10 +416,10 @@ composite_bucket_fields(Types, ClientRef) -> fields_of_bucket(Default) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => Default})}, + {rate, ?HOCON(rate_type(), #{desc => deprecated_desc(rate), default => Default})}, {burst, ?HOCON(burst(), #{ - desc => ?DESC(burst), + desc => deprecated_desc(burst), default => <<"0">>, importance => ?IMPORTANCE_HIDDEN, aliases => [capacity] @@ -425,7 +427,7 @@ fields_of_bucket(Default) -> {initial, ?HOCON(initial(), #{ default => <<"0">>, - desc => ?DESC(initial), + desc => deprecated_desc(initial), importance => ?IMPORTANCE_HIDDEN })} ]. @@ -434,7 +436,7 @@ client_fields(Types) -> [ {Type, ?HOCON(?R_REF(client_opts), #{ - desc => ?DESC(Type), + desc => deprecated_desc(Type), required => false, importance => importance_of_type(Type), aliases => alias_of_type(Type) @@ -457,3 +459,15 @@ alias_of_type(bytes) -> [bytes_in]; alias_of_type(_) -> []. + +deprecated_desc(_Field) -> + <<"Deprecated since v5.0.25">>. + +rate_type() -> + typerefl:alias("string", rate()). + +burst_type() -> + typerefl:alias("string", burst()). + +burst_rate_type() -> + typerefl:alias("string", burst_rate()). diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index 509d4c90d..4ff36504d 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -66,7 +66,8 @@ -export([ is_expired/1, - update_expiry/1 + update_expiry/1, + timestamp_now/0 ]). -export([ @@ -113,14 +114,13 @@ make(From, Topic, Payload) -> emqx_types:payload() ) -> emqx_types:message(). make(From, QoS, Topic, Payload) when ?QOS_0 =< QoS, QoS =< ?QOS_2 -> - Now = erlang:system_time(millisecond), #message{ id = emqx_guid:gen(), qos = QoS, from = From, topic = Topic, payload = Payload, - timestamp = Now + timestamp = timestamp_now() }. -spec make( @@ -137,7 +137,6 @@ make(From, QoS, Topic, Payload, Flags, Headers) when is_map(Flags), is_map(Headers) -> - Now = erlang:system_time(millisecond), #message{ id = emqx_guid:gen(), qos = QoS, @@ -146,7 +145,7 @@ make(From, QoS, Topic, Payload, Flags, Headers) when headers = Headers, topic = Topic, payload = Payload, - timestamp = Now + timestamp = timestamp_now() }. -spec make( @@ -164,7 +163,6 @@ make(MsgId, From, QoS, Topic, Payload, Flags, Headers) when is_map(Flags), is_map(Headers) -> - Now = erlang:system_time(millisecond), #message{ id = MsgId, qos = QoS, @@ -173,7 +171,7 @@ make(MsgId, From, QoS, Topic, Payload, Flags, Headers) when headers = Headers, topic = Topic, payload = Payload, - timestamp = Now + timestamp = timestamp_now() }. %% optimistic esitmation of a message size after serialization @@ -403,6 +401,11 @@ from_map(#{ extra = Extra }. +%% @doc Get current timestamp in milliseconds. +-spec timestamp_now() -> integer(). +timestamp_now() -> + erlang:system_time(millisecond). + %% MilliSeconds elapsed(Since) -> - max(0, erlang:system_time(millisecond) - Since). + max(0, timestamp_now() - Since). diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index c68a146ed..1232dfcb4 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -83,7 +83,7 @@ do_check_pass({_SimpleHash, _Salt, _SaltPosition} = HashParams, PasswordHash, Pa compare_secure(Hash, PasswordHash). -spec hash(hash_params(), password()) -> password_hash(). -hash({pbkdf2, MacFun, Salt, Iterations, DKLength}, Password) -> +hash({pbkdf2, MacFun, Salt, Iterations, DKLength}, Password) when Iterations > 0 -> case pbkdf2(MacFun, Password, Salt, Iterations, DKLength) of {ok, HashPasswd} -> hex(HashPasswd); diff --git a/apps/emqx/src/emqx_persistent_message.erl b/apps/emqx/src/emqx_persistent_message.erl index 609b0139d..632ff2a27 100644 --- a/apps/emqx/src/emqx_persistent_message.erl +++ b/apps/emqx/src/emqx_persistent_message.erl @@ -23,16 +23,12 @@ %% Message persistence -export([ - persist/1, - serialize/1, - deserialize/1 + persist/1 ]). -%% FIXME --define(DS_SHARD_ID, <<"local">>). --define(DEFAULT_KEYSPACE, default). --define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). +-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message). +%% FIXME -define(WHEN_ENABLED(DO), case is_store_enabled() of true -> DO; @@ -44,18 +40,10 @@ init() -> ?WHEN_ENABLED(begin - ok = emqx_ds:ensure_shard( - ?DS_SHARD, - #{ - dir => filename:join([ - emqx:data_dir(), - ds, - messages, - ?DEFAULT_KEYSPACE, - ?DS_SHARD_ID - ]) - } - ), + ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{ + backend => builtin, + storage => {emqx_ds_storage_bitfield_lts, #{}} + }), ok = emqx_persistent_session_ds_router:init_tables(), ok = emqx_persistent_session_ds:create_tables(), ok @@ -82,19 +70,11 @@ persist(Msg) -> needs_persistence(Msg) -> not (emqx_message:get_flag(dup, Msg) orelse emqx_message:is_sys(Msg)). +-spec store_message(emqx_types:message()) -> emqx_ds:store_batch_result(). store_message(Msg) -> - ID = emqx_message:id(Msg), - Timestamp = emqx_guid:timestamp(ID), - Topic = emqx_topic:words(emqx_message:topic(Msg)), - emqx_ds_storage_layer:store(?DS_SHARD, ID, Timestamp, Topic, serialize(Msg)). + emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg]). has_subscribers(#message{topic = Topic}) -> emqx_persistent_session_ds_router:has_any_route(Topic). %% - -serialize(Msg) -> - term_to_binary(emqx_message:to_map(Msg)). - -deserialize(Bin) -> - emqx_message:from_map(binary_to_term(Bin)). diff --git a/apps/emqx/src/emqx_persistent_message_ds_replayer.erl b/apps/emqx/src/emqx_persistent_message_ds_replayer.erl new file mode 100644 index 000000000..69b6675d8 --- /dev/null +++ b/apps/emqx/src/emqx_persistent_message_ds_replayer.erl @@ -0,0 +1,314 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This module implements the routines for replaying streams of +%% messages. +-module(emqx_persistent_message_ds_replayer). + +%% API: +-export([new/0, next_packet_id/1, replay/2, commit_offset/3, poll/3, n_inflight/1]). + +%% internal exports: +-export([]). + +-export_type([inflight/0]). + +-include_lib("emqx/include/logger.hrl"). +-include("emqx_persistent_session_ds.hrl"). + +-ifdef(TEST). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%%================================================================================ +%% Type declarations +%%================================================================================ + +%% Note: sequence numbers are monotonic; they don't wrap around: +-type seqno() :: non_neg_integer(). + +-record(range, { + stream :: emqx_ds:stream(), + first :: seqno(), + last :: seqno(), + iterator_next :: emqx_ds:iterator() | undefined +}). + +-type range() :: #range{}. + +-record(inflight, { + next_seqno = 0 :: seqno(), + acked_seqno = 0 :: seqno(), + offset_ranges = [] :: [range()] +}). + +-opaque inflight() :: #inflight{}. + +%%================================================================================ +%% API funcions +%%================================================================================ + +-spec new() -> inflight(). +new() -> + #inflight{}. + +-spec next_packet_id(inflight()) -> {emqx_types:packet_id(), inflight()}. +next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqNo}) -> + Inflight = Inflight0#inflight{next_seqno = LastSeqNo + 1}, + case LastSeqNo rem 16#10000 of + 0 -> + %% We skip sequence numbers that lead to PacketId = 0 to + %% simplify math. Note: it leads to occasional gaps in the + %% sequence numbers. + next_packet_id(Inflight); + PacketId -> + {PacketId, Inflight} + end. + +-spec n_inflight(inflight()) -> non_neg_integer(). +n_inflight(#inflight{next_seqno = NextSeqNo, acked_seqno = AckedSeqno}) -> + %% NOTE: this function assumes that gaps in the sequence ID occur + %% _only_ when the packet ID wraps: + case AckedSeqno >= ((NextSeqNo bsr 16) bsl 16) of + true -> + NextSeqNo - AckedSeqno; + false -> + NextSeqNo - AckedSeqno - 1 + end. + +-spec replay(emqx_persistent_session_ds:id(), inflight()) -> + emqx_session:replies(). +replay(_SessionId, _Inflight = #inflight{offset_ranges = _Ranges}) -> + []. + +-spec commit_offset(emqx_persistent_session_ds:id(), emqx_types:packet_id(), inflight()) -> + {_IsValidOffset :: boolean(), inflight()}. +commit_offset( + SessionId, + PacketId, + Inflight0 = #inflight{ + acked_seqno = AckedSeqno0, next_seqno = NextSeqNo, offset_ranges = Ranges0 + } +) -> + AckedSeqno = + case packet_id_to_seqno(NextSeqNo, PacketId) of + N when N > AckedSeqno0; AckedSeqno0 =:= 0 -> + N; + OutOfRange -> + ?SLOG(warning, #{ + msg => "out-of-order_ack", + prev_seqno => AckedSeqno0, + acked_seqno => OutOfRange, + next_seqno => NextSeqNo, + packet_id => PacketId + }), + AckedSeqno0 + end, + Ranges = lists:filter( + fun(#range{stream = Stream, last = LastSeqno, iterator_next = ItNext}) -> + case LastSeqno =< AckedSeqno of + true -> + %% This range has been fully + %% acked. Remove it and replace saved + %% iterator with the trailing iterator. + update_iterator(SessionId, Stream, ItNext), + false; + false -> + %% This range still has unacked + %% messages: + true + end + end, + Ranges0 + ), + Inflight = Inflight0#inflight{acked_seqno = AckedSeqno, offset_ranges = Ranges}, + {true, Inflight}. + +-spec poll(emqx_persistent_session_ds:id(), inflight(), pos_integer()) -> + {emqx_session:replies(), inflight()}. +poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff -> + #inflight{next_seqno = NextSeqNo0, acked_seqno = AckedSeqno} = + Inflight0, + FetchThreshold = max(1, WindowSize div 2), + FreeSpace = AckedSeqno + WindowSize - NextSeqNo0, + case FreeSpace >= FetchThreshold of + false -> + %% TODO: this branch is meant to avoid fetching data from + %% the DB in chunks that are too small. However, this + %% logic is not exactly good for the latency. Can the + %% client get stuck even? + {[], Inflight0}; + true -> + Streams = shuffle(get_streams(SessionId)), + fetch(SessionId, Inflight0, Streams, FreeSpace, []) + end. + +%%================================================================================ +%% Internal exports +%%================================================================================ + +%%================================================================================ +%% Internal functions +%%================================================================================ + +fetch(_SessionId, Inflight, _Streams = [], _N, Acc) -> + {lists:reverse(Acc), Inflight}; +fetch(_SessionId, Inflight, _Streams, 0, Acc) -> + {lists:reverse(Acc), Inflight}; +fetch(SessionId, Inflight0, [Stream | Streams], N, Publishes0) -> + #inflight{next_seqno = FirstSeqNo, offset_ranges = Ranges0} = Inflight0, + ItBegin = get_last_iterator(SessionId, Stream, Ranges0), + {ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N), + {NMessages, Publishes, Inflight1} = + lists:foldl( + fun(Msg, {N0, PubAcc0, InflightAcc0}) -> + {PacketId, InflightAcc} = next_packet_id(InflightAcc0), + PubAcc = [{PacketId, Msg} | PubAcc0], + {N0 + 1, PubAcc, InflightAcc} + end, + {0, Publishes0, Inflight0}, + Messages + ), + #inflight{next_seqno = LastSeqNo} = Inflight1, + case NMessages > 0 of + true -> + Range = #range{ + first = FirstSeqNo, + last = LastSeqNo - 1, + stream = Stream, + iterator_next = ItEnd + }, + Inflight = Inflight1#inflight{offset_ranges = Ranges0 ++ [Range]}, + fetch(SessionId, Inflight, Streams, N - NMessages, Publishes); + false -> + fetch(SessionId, Inflight1, Streams, N, Publishes) + end. + +-spec update_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream(), emqx_ds:iterator()) -> ok. +update_iterator(DSSessionId, Stream, Iterator) -> + %% Workaround: we convert `Stream' to a binary before attempting to store it in + %% mnesia(rocksdb) because of a bug in `mnesia_rocksdb' when trying to do + %% `mnesia:dirty_all_keys' later. + StreamBin = term_to_binary(Stream), + mria:dirty_write(?SESSION_ITER_TAB, #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator}). + +get_last_iterator(SessionId, Stream, Ranges) -> + case lists:keyfind(Stream, #range.stream, lists:reverse(Ranges)) of + false -> + get_iterator(SessionId, Stream); + #range{iterator_next = Next} -> + Next + end. + +-spec get_iterator(emqx_persistent_session_ds:id(), emqx_ds:stream()) -> emqx_ds:iterator(). +get_iterator(DSSessionId, Stream) -> + %% See comment in `update_iterator'. + StreamBin = term_to_binary(Stream), + Id = {DSSessionId, StreamBin}, + [#ds_iter{iter = It}] = mnesia:dirty_read(?SESSION_ITER_TAB, Id), + It. + +-spec get_streams(emqx_persistent_session_ds:id()) -> [emqx_ds:stream()]. +get_streams(SessionId) -> + lists:map( + fun(#ds_stream{stream = Stream}) -> + Stream + end, + mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId) + ). + +%% Reconstruct session counter by adding most significant bits from +%% the current counter to the packet id. +-spec packet_id_to_seqno(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer(). +packet_id_to_seqno(NextSeqNo, PacketId) -> + Epoch = NextSeqNo bsr 16, + case packet_id_to_seqno_(Epoch, PacketId) of + N when N =< NextSeqNo -> + N; + _ -> + packet_id_to_seqno_(Epoch - 1, PacketId) + end. + +-spec packet_id_to_seqno_(non_neg_integer(), emqx_types:packet_id()) -> non_neg_integer(). +packet_id_to_seqno_(Epoch, PacketId) -> + (Epoch bsl 16) + PacketId. + +-spec shuffle([A]) -> [A]. +shuffle(L0) -> + L1 = lists:map( + fun(A) -> + {rand:uniform(), A} + end, + L0 + ), + L2 = lists:sort(L1), + {_, L} = lists:unzip(L2), + L. + +-ifdef(TEST). + +%% This test only tests boundary conditions (to make sure property-based test didn't skip them): +packet_id_to_seqno_test() -> + %% Packet ID = 1; first epoch: + ?assertEqual(1, packet_id_to_seqno(1, 1)), + ?assertEqual(1, packet_id_to_seqno(10, 1)), + ?assertEqual(1, packet_id_to_seqno(1 bsl 16 - 1, 1)), + ?assertEqual(1, packet_id_to_seqno(1 bsl 16, 1)), + %% Packet ID = 1; second and 3rd epochs: + ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(1 bsl 16 + 1, 1)), + ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(2 bsl 16, 1)), + ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno(2 bsl 16 + 1, 1)), + %% Packet ID = 16#ffff: + PID = 1 bsl 16 - 1, + ?assertEqual(PID, packet_id_to_seqno(PID, PID)), + ?assertEqual(PID, packet_id_to_seqno(1 bsl 16, PID)), + ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno(2 bsl 16, PID)), + ok. + +packet_id_to_seqno_test_() -> + Opts = [{numtests, 1000}, {to_file, user}], + {timeout, 30, fun() -> ?assert(proper:quickcheck(packet_id_to_seqno_prop(), Opts)) end}. + +packet_id_to_seqno_prop() -> + ?FORALL( + NextSeqNo, + next_seqno_gen(), + ?FORALL( + SeqNo, + seqno_gen(NextSeqNo), + begin + PacketId = SeqNo rem 16#10000, + ?assertEqual(SeqNo, packet_id_to_seqno(NextSeqNo, PacketId)), + true + end + ) + ). + +next_seqno_gen() -> + ?LET( + {Epoch, Offset}, + {non_neg_integer(), non_neg_integer()}, + Epoch bsl 16 + Offset + ). + +seqno_gen(NextSeqNo) -> + WindowSize = 1 bsl 16 - 1, + Min = max(0, NextSeqNo - WindowSize), + Max = max(0, NextSeqNo - 1), + range(Min, Max). + +-endif. diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index e456211fc..6c0fc2dcc 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -16,11 +16,16 @@ -module(emqx_persistent_session_ds). +-behaviour(emqx_session). + -include("emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_persistent_session_ds.hrl"). + %% Session API -export([ create/3, @@ -50,7 +55,7 @@ -export([ deliver/3, replay/3, - % handle_timeout/3, + handle_timeout/3, disconnect/1, terminate/2 ]). @@ -58,33 +63,33 @@ %% session table operations -export([create_tables/0]). --ifdef(TEST). --export([session_open/1]). --endif. - -%% RPC --export([ - ensure_iterator_closed_on_all_shards/1, - ensure_all_iterators_closed/1 -]). +%% Remove me later (satisfy checks for an unused BPAPI) -export([ do_open_iterator/3, do_ensure_iterator_closed/1, do_ensure_all_iterators_closed/1 ]). -%% FIXME --define(DS_SHARD_ID, <<"local">>). --define(DEFAULT_KEYSPACE, default). --define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). +-ifdef(TEST). +-export([ + session_open/1, + list_all_sessions/0, + list_all_subscriptions/0, + list_all_streams/0, + list_all_iterators/0 +]). +-endif. %% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be %% an atom, in theory (?). -type id() :: binary(). --type iterator() :: emqx_ds:iterator(). --type iterator_id() :: emqx_ds:iterator_id(). -type topic_filter() :: emqx_ds:topic_filter(). --type iterators() :: #{topic_filter() => iterator()}. +-type subscription_id() :: {id(), topic_filter()}. +-type subscription() :: #{ + start_time := emqx_ds:time(), + propts := map(), + extra := map() +}. -type session() :: #{ %% Client ID id := id(), @@ -93,7 +98,11 @@ %% When the session should expire expires_at := timestamp() | never, %% Client’s Subscriptions. - iterators := #{topic() => iterator()}, + iterators := #{topic() => subscription()}, + %% Inflight messages + inflight := emqx_persistent_message_ds_replayer:inflight(), + %% Receive maximum + receive_maximum := pos_integer(), %% props := map() }. @@ -104,19 +113,28 @@ -type conninfo() :: emqx_session:conninfo(). -type replies() :: emqx_session:replies(). +-define(STATS_KEYS, [ + subscriptions_cnt, + subscriptions_max, + inflight_cnt, + inflight_max, + next_pkt_id +]). + -export_type([id/0]). %% -spec create(clientinfo(), conninfo(), emqx_session:conf()) -> session(). -create(#{clientid := ClientID}, _ConnInfo, Conf) -> +create(#{clientid := ClientID}, ConnInfo, Conf) -> % TODO: expiration - ensure_session(ClientID, Conf). + ensure_timers(), + ensure_session(ClientID, ConnInfo, Conf). -spec open(clientinfo(), conninfo()) -> {_IsPresent :: true, session(), []} | false. -open(#{clientid := ClientID}, _ConnInfo) -> +open(#{clientid := ClientID} = _ClientInfo, ConnInfo) -> %% NOTE %% The fact that we need to concern about discarding all live channels here %% is essentially a consequence of the in-memory session design, where we @@ -125,29 +143,33 @@ open(#{clientid := ClientID}, _ConnInfo) -> %% space, and move this call back into `emqx_cm` where it belongs. ok = emqx_cm:discard_session(ClientID), case open_session(ClientID) of - Session = #{} -> + Session0 = #{} -> + ensure_timers(), + ReceiveMaximum = receive_maximum(ConnInfo), + Session = Session0#{receive_maximum => ReceiveMaximum}, {true, Session, []}; false -> false end. -ensure_session(ClientID, Conf) -> +ensure_session(ClientID, ConnInfo, Conf) -> {ok, Session, #{}} = session_ensure_new(ClientID, Conf), - Session#{iterators => #{}}. + ReceiveMaximum = receive_maximum(ConnInfo), + Session#{iterators => #{}, receive_maximum => ReceiveMaximum}. open_session(ClientID) -> case session_open(ClientID) of - {ok, Session, Iterators} -> - Session#{iterators => prep_iterators(Iterators)}; + {ok, Session, Subscriptions} -> + Session#{iterators => prep_subscriptions(Subscriptions)}; false -> false end. -prep_iterators(Iterators) -> +prep_subscriptions(Subscriptions) -> maps:fold( - fun(Topic, Iterator, Acc) -> Acc#{emqx_topic:join(Topic) => Iterator} end, + fun(Topic, Subscription, Acc) -> Acc#{emqx_topic:join(Topic) => Subscription} end, #{}, - Iterators + Subscriptions ). -spec destroy(session() | clientinfo()) -> ok. @@ -157,7 +179,6 @@ destroy(#{clientid := ClientID}) -> destroy_session(ClientID). destroy_session(ClientID) -> - _ = ensure_all_iterators_closed(ClientID), session_drop(ClientID). %%-------------------------------------------------------------------- @@ -184,10 +205,10 @@ info(upgrade_qos, #{props := Conf}) -> maps:get(upgrade_qos, Conf); % info(inflight, #sessmem{inflight = Inflight}) -> % Inflight; -% info(inflight_cnt, #sessmem{inflight = Inflight}) -> -% emqx_inflight:size(Inflight); -% info(inflight_max, #sessmem{inflight = Inflight}) -> -% emqx_inflight:max_size(Inflight); +info(inflight_cnt, #{inflight := Inflight}) -> + emqx_persistent_message_ds_replayer:n_inflight(Inflight); +info(inflight_max, #{receive_maximum := ReceiveMaximum}) -> + ReceiveMaximum; info(retry_interval, #{props := Conf}) -> maps:get(retry_interval, Conf); % info(mqueue, #sessmem{mqueue = MQueue}) -> @@ -198,8 +219,9 @@ info(retry_interval, #{props := Conf}) -> % emqx_mqueue:max_len(MQueue); % info(mqueue_dropped, #sessmem{mqueue = MQueue}) -> % emqx_mqueue:dropped(MQueue); -info(next_pkt_id, #{}) -> - _PacketId = 'TODO'; +info(next_pkt_id, #{inflight := Inflight}) -> + {PacketId, _} = emqx_persistent_message_ds_replayer:next_packet_id(Inflight), + PacketId; % info(awaiting_rel, #sessmem{awaiting_rel = AwaitingRel}) -> % AwaitingRel; % info(awaiting_rel_cnt, #sessmem{awaiting_rel = AwaitingRel}) -> @@ -211,8 +233,7 @@ info(await_rel_timeout, #{props := Conf}) -> -spec stats(session()) -> emqx_types:stats(). stats(Session) -> - % TODO: stub - info([], Session). + info(?STATS_KEYS, Session). %%-------------------------------------------------------------------- %% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE @@ -245,7 +266,7 @@ unsubscribe( ) when is_map_key(TopicFilter, Iters) -> Iterator = maps:get(TopicFilter, Iters), SubOpts = maps:get(props, Iterator), - ok = del_subscription(TopicFilter, Iterator, ID), + ok = del_subscription(TopicFilter, ID), {ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts}; unsubscribe( _TopicFilter, @@ -271,19 +292,29 @@ get_subscription(TopicFilter, #{iterators := Iters}) -> {ok, emqx_types:publish_result(), replies(), session()} | {error, emqx_types:reason_code()}. publish(_PacketId, Msg, Session) -> - % TODO: stub - {ok, emqx_broker:publish(Msg), [], Session}. + %% TODO: + Result = emqx_broker:publish(Msg), + {ok, Result, [], Session}. %%-------------------------------------------------------------------- %% Client -> Broker: PUBACK %%-------------------------------------------------------------------- +%% FIXME: parts of the commit offset function are mocked +-dialyzer({nowarn_function, puback/3}). + -spec puback(clientinfo(), emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}. -puback(_ClientInfo, _PacketId, _Session = #{}) -> - % TODO: stub - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. +puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) -> + case emqx_persistent_message_ds_replayer:commit_offset(Id, PacketId, Inflight0) of + {true, Inflight} -> + %% TODO + Msg = #message{}, + {ok, Msg, [], Session#{inflight => Inflight}}; + {false, _} -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. %%-------------------------------------------------------------------- %% Client -> Broker: PUBREC @@ -320,10 +351,33 @@ pubcomp(_ClientInfo, _PacketId, _Session = #{}) -> %%-------------------------------------------------------------------- -spec deliver(clientinfo(), [emqx_types:deliver()], session()) -> - no_return(). -deliver(_ClientInfo, _Delivers, _Session = #{}) -> - % TODO: ensure it's unreachable somehow - error(unexpected). + {ok, replies(), session()}. +deliver(_ClientInfo, _Delivers, Session) -> + %% TODO: QoS0 and system messages end up here. + {ok, [], Session}. + +-spec handle_timeout(clientinfo(), _Timeout, session()) -> + {ok, replies(), session()} | {ok, replies(), timeout(), session()}. +handle_timeout( + _ClientInfo, + pull, + Session = #{id := Id, inflight := Inflight0, receive_maximum := ReceiveMaximum} +) -> + {Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(Id, Inflight0, ReceiveMaximum), + %% TODO: make these values configurable: + Timeout = + case Publishes of + [] -> + 100; + [_ | _] -> + 0 + end, + ensure_timer(pull, Timeout), + {ok, Publishes, Session#{inflight => Inflight}}; +handle_timeout(_ClientInfo, get_streams, Session = #{id := Id}) -> + renew_streams(Id), + ensure_timer(get_streams), + {ok, [], Session}. -spec replay(clientinfo(), [], session()) -> {ok, replies(), session()}. @@ -344,151 +398,69 @@ terminate(_Reason, _Session = #{}) -> %%-------------------------------------------------------------------- -spec add_subscription(topic(), emqx_types:subopts(), id()) -> - emqx_ds:iterator(). + subscription(). add_subscription(TopicFilterBin, SubOpts, DSSessionID) -> - % N.B.: we chose to update the router before adding the subscription to the - % session/iterator table. The reasoning for this is as follows: - % - % Messages matching this topic filter should start to be persisted as soon as - % possible to avoid missing messages. If this is the first such persistent - % session subscription, it's important to do so early on. - % - % This could, in turn, lead to some inconsistency: if such a route gets - % created but the session/iterator data fails to be updated accordingly, we - % have a dangling route. To remove such dangling routes, we may have a - % periodic GC process that removes routes that do not have a matching - % persistent subscription. Also, route operations use dirty mnesia - % operations, which inherently have room for inconsistencies. - % - % In practice, we use the iterator reference table as a source of truth, - % since it is guarded by a transaction context: we consider a subscription - % operation to be successful if it ended up changing this table. Both router - % and iterator information can be reconstructed from this table, if needed. + %% N.B.: we chose to update the router before adding the subscription to the + %% session/iterator table. The reasoning for this is as follows: + %% + %% Messages matching this topic filter should start to be persisted as soon as + %% possible to avoid missing messages. If this is the first such persistent + %% session subscription, it's important to do so early on. + %% + %% This could, in turn, lead to some inconsistency: if such a route gets + %% created but the session/iterator data fails to be updated accordingly, we + %% have a dangling route. To remove such dangling routes, we may have a + %% periodic GC process that removes routes that do not have a matching + %% persistent subscription. Also, route operations use dirty mnesia + %% operations, which inherently have room for inconsistencies. + %% + %% In practice, we use the iterator reference table as a source of truth, + %% since it is guarded by a transaction context: we consider a subscription + %% operation to be successful if it ended up changing this table. Both router + %% and iterator information can be reconstructed from this table, if needed. ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID), TopicFilter = emqx_topic:words(TopicFilterBin), - {ok, Iterator, IsNew} = session_add_iterator( + {ok, DSSubExt, IsNew} = session_add_subscription( DSSessionID, TopicFilter, SubOpts ), - Ctx = #{iterator => Iterator, is_new => IsNew}, - ?tp(persistent_session_ds_iterator_added, Ctx), - ?tp_span( - persistent_session_ds_open_iterators, - Ctx, - ok = open_iterator_on_all_shards(TopicFilter, Iterator) - ), - Iterator. + ?tp(persistent_session_ds_subscription_added, #{sub => DSSubExt, is_new => IsNew}), + %% we'll list streams and open iterators when implementing message replay. + DSSubExt. --spec update_subscription(topic(), iterator(), emqx_types:subopts(), id()) -> - iterator(). -update_subscription(TopicFilterBin, Iterator, SubOpts, DSSessionID) -> +-spec update_subscription(topic(), subscription(), emqx_types:subopts(), id()) -> + subscription(). +update_subscription(TopicFilterBin, DSSubExt, SubOpts, DSSessionID) -> TopicFilter = emqx_topic:words(TopicFilterBin), - {ok, NIterator, false} = session_add_iterator( + {ok, NDSSubExt, false} = session_add_subscription( DSSessionID, TopicFilter, SubOpts ), - ok = ?tp(persistent_session_ds_iterator_updated, #{iterator => Iterator}), - NIterator. + ok = ?tp(persistent_session_ds_iterator_updated, #{sub => DSSubExt}), + NDSSubExt. --spec open_iterator_on_all_shards(emqx_types:words(), emqx_ds:iterator()) -> ok. -open_iterator_on_all_shards(TopicFilter, Iterator) -> - ?tp(persistent_session_ds_will_open_iterators, #{iterator => Iterator}), - %% Note: currently, shards map 1:1 to nodes, but this will change in the future. - Nodes = emqx:running_nodes(), - Results = emqx_persistent_session_ds_proto_v1:open_iterator( - Nodes, - TopicFilter, - maps:get(start_time, Iterator), - maps:get(id, Iterator) - ), - %% TODO - %% 1. Handle errors. - %% 2. Iterator handles are rocksdb resources, it's doubtful they survive RPC. - %% Even if they do, we throw them away here anyway. All in all, we probably should - %% hold each of them in a process on the respective node. - true = lists:all(fun(Res) -> element(1, Res) =:= ok end, Results), +-spec del_subscription(topic(), id()) -> ok. - -%% RPC target. --spec do_open_iterator(emqx_types:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> - {ok, emqx_ds_storage_layer:iterator()} | {error, _Reason}. -do_open_iterator(TopicFilter, StartMS, IteratorID) -> - Replay = {TopicFilter, StartMS}, - emqx_ds_storage_layer:ensure_iterator(?DS_SHARD, IteratorID, Replay). - --spec del_subscription(topic(), iterator(), id()) -> - ok. -del_subscription(TopicFilterBin, #{id := IteratorID}, DSSessionID) -> - % N.B.: see comments in `?MODULE:add_subscription' for a discussion about the - % order of operations here. +del_subscription(TopicFilterBin, DSSessionId) -> TopicFilter = emqx_topic:words(TopicFilterBin), - Ctx = #{iterator_id => IteratorID}, ?tp_span( - persistent_session_ds_close_iterators, - Ctx, - ok = ensure_iterator_closed_on_all_shards(IteratorID) + persistent_session_ds_subscription_delete, + #{session_id => DSSessionId}, + ok = session_del_subscription(DSSessionId, TopicFilter) ), ?tp_span( - persistent_session_ds_iterator_delete, - Ctx, - session_del_iterator(DSSessionID, TopicFilter) - ), - ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionID). - --spec ensure_iterator_closed_on_all_shards(emqx_ds:iterator_id()) -> ok. -ensure_iterator_closed_on_all_shards(IteratorID) -> - %% Note: currently, shards map 1:1 to nodes, but this will change in the future. - Nodes = emqx:running_nodes(), - Results = emqx_persistent_session_ds_proto_v1:close_iterator(Nodes, IteratorID), - %% TODO: handle errors - true = lists:all(fun(Res) -> Res =:= {ok, ok} end, Results), - ok. - -%% RPC target. --spec do_ensure_iterator_closed(emqx_ds:iterator_id()) -> ok. -do_ensure_iterator_closed(IteratorID) -> - ok = emqx_ds_storage_layer:discard_iterator(?DS_SHARD, IteratorID), - ok. - --spec ensure_all_iterators_closed(id()) -> ok. -ensure_all_iterators_closed(DSSessionID) -> - %% Note: currently, shards map 1:1 to nodes, but this will change in the future. - Nodes = emqx:running_nodes(), - Results = emqx_persistent_session_ds_proto_v1:close_all_iterators(Nodes, DSSessionID), - %% TODO: handle errors - true = lists:all(fun(Res) -> Res =:= {ok, ok} end, Results), - ok. - -%% RPC target. --spec do_ensure_all_iterators_closed(id()) -> ok. -do_ensure_all_iterators_closed(DSSessionID) -> - ok = emqx_ds_storage_layer:discard_iterator_prefix(?DS_SHARD, DSSessionID), - ok. + persistent_session_ds_subscription_route_delete, + #{session_id => DSSessionId}, + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId) + ). %%-------------------------------------------------------------------- %% Session tables operations %%-------------------------------------------------------------------- --define(SESSION_TAB, emqx_ds_session). --define(ITERATOR_REF_TAB, emqx_ds_iterator_ref). --define(DS_MRIA_SHARD, emqx_ds_shard). - --record(session, { - %% same as clientid - id :: id(), - %% creation time - created_at :: _Millisecond :: non_neg_integer(), - expires_at = never :: _Millisecond :: non_neg_integer() | never, - %% for future usage - props = #{} :: map() -}). - --record(iterator_ref, { - ref_id :: {id(), emqx_ds:topic_filter()}, - it_id :: emqx_ds:iterator_id(), - start_time :: emqx_ds:time(), - props = #{} :: map() -}). - create_tables() -> + ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, #{ + backend => builtin, + storage => {emqx_ds_storage_bitfield_lts, #{}} + }), ok = mria:create_table( ?SESSION_TAB, [ @@ -500,15 +472,38 @@ create_tables() -> ] ), ok = mria:create_table( - ?ITERATOR_REF_TAB, + ?SESSION_SUBSCRIPTIONS_TAB, [ {rlog_shard, ?DS_MRIA_SHARD}, {type, ordered_set}, {storage, storage()}, - {record_name, iterator_ref}, - {attributes, record_info(fields, iterator_ref)} + {record_name, ds_sub}, + {attributes, record_info(fields, ds_sub)} ] ), + ok = mria:create_table( + ?SESSION_STREAM_TAB, + [ + {rlog_shard, ?DS_MRIA_SHARD}, + {type, bag}, + {storage, storage()}, + {record_name, ds_stream}, + {attributes, record_info(fields, ds_stream)} + ] + ), + ok = mria:create_table( + ?SESSION_ITER_TAB, + [ + {rlog_shard, ?DS_MRIA_SHARD}, + {type, set}, + {storage, storage()}, + {record_name, ds_iter}, + {attributes, record_info(fields, ds_iter)} + ] + ), + ok = mria:wait_for_tables([ + ?SESSION_TAB, ?SESSION_SUBSCRIPTIONS_TAB, ?SESSION_STREAM_TAB, ?SESSION_ITER_TAB + ]), ok. -dialyzer({nowarn_function, storage/0}). @@ -524,31 +519,29 @@ storage() -> %% @doc Called when a client connects. This function looks up a %% session or returns `false` if previous one couldn't be found. %% -%% This function also spawns replay agents for each iterator. -%% %% Note: session API doesn't handle session takeovers, it's the job of %% the broker. -spec session_open(id()) -> - {ok, session(), iterators()} | false. + {ok, session(), #{topic() => subscription()}} | false. session_open(SessionId) -> transaction(fun() -> case mnesia:read(?SESSION_TAB, SessionId, write) of [Record = #session{}] -> - Session = export_record(Record), - IteratorRefs = session_read_iterators(SessionId), - Iterators = export_iterators(IteratorRefs), - {ok, Session, Iterators}; + Session = export_session(Record), + DSSubs = session_read_subscriptions(SessionId), + Subscriptions = export_subscriptions(DSSubs), + {ok, Session, Subscriptions}; [] -> false end end). -spec session_ensure_new(id(), _Props :: map()) -> - {ok, session(), iterators()}. + {ok, session(), #{topic() => subscription()}}. session_ensure_new(SessionId, Props) -> transaction(fun() -> - ok = session_drop_iterators(SessionId), - Session = export_record(session_create(SessionId, Props)), + ok = session_drop_subscriptions(SessionId), + Session = export_session(session_create(SessionId, Props)), {ok, Session, #{}} end). @@ -557,7 +550,8 @@ session_create(SessionId, Props) -> id = SessionId, created_at = erlang:system_time(millisecond), expires_at = never, - props = Props + props = Props, + inflight = emqx_persistent_message_ds_replayer:new() }, ok = mnesia:write(?SESSION_TAB, Session, write), Session. @@ -567,81 +561,194 @@ session_create(SessionId, Props) -> -spec session_drop(id()) -> ok. session_drop(DSSessionId) -> transaction(fun() -> - %% TODO: ensure all iterators from this clientid are closed? + ok = session_drop_subscriptions(DSSessionId), ok = session_drop_iterators(DSSessionId), + ok = session_drop_streams(DSSessionId), ok = mnesia:delete(?SESSION_TAB, DSSessionId, write) end). -session_drop_iterators(DSSessionId) -> - IteratorRefs = session_read_iterators(DSSessionId), - ok = lists:foreach(fun session_del_iterator/1, IteratorRefs). +-spec session_drop_subscriptions(id()) -> ok. +session_drop_subscriptions(DSSessionId) -> + Subscriptions = session_read_subscriptions(DSSessionId), + lists:foreach( + fun(#ds_sub{id = DSSubId} = DSSub) -> + TopicFilter = subscription_id_to_topic_filter(DSSubId), + TopicFilterBin = emqx_topic:join(TopicFilter), + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionId), + ok = session_del_subscription(DSSub) + end, + Subscriptions + ). %% @doc Called when a client subscribes to a topic. Idempotent. --spec session_add_iterator(id(), topic_filter(), _Props :: map()) -> - {ok, iterator(), _IsNew :: boolean()}. -session_add_iterator(DSSessionId, TopicFilter, Props) -> - IteratorRefId = {DSSessionId, TopicFilter}, +-spec session_add_subscription(id(), topic_filter(), _Props :: map()) -> + {ok, subscription(), _IsNew :: boolean()}. +session_add_subscription(DSSessionId, TopicFilter, Props) -> + DSSubId = {DSSessionId, TopicFilter}, transaction(fun() -> - case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of + case mnesia:read(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write) of [] -> - IteratorRef = session_insert_iterator(DSSessionId, TopicFilter, Props), - Iterator = export_record(IteratorRef), + DSSub = session_insert_subscription(DSSessionId, TopicFilter, Props), + DSSubExt = export_subscription(DSSub), ?tp( ds_session_subscription_added, - #{iterator => Iterator, session_id => DSSessionId} + #{sub => DSSubExt, session_id => DSSessionId} ), - {ok, Iterator, _IsNew = true}; - [#iterator_ref{} = IteratorRef] -> - NIteratorRef = session_update_iterator(IteratorRef, Props), - NIterator = export_record(NIteratorRef), + {ok, DSSubExt, _IsNew = true}; + [#ds_sub{} = DSSub] -> + NDSSub = session_update_subscription(DSSub, Props), + NDSSubExt = export_subscription(NDSSub), ?tp( ds_session_subscription_present, - #{iterator => NIterator, session_id => DSSessionId} + #{sub => NDSSubExt, session_id => DSSessionId} ), - {ok, NIterator, _IsNew = false} + {ok, NDSSubExt, _IsNew = false} end end). -session_insert_iterator(DSSessionId, TopicFilter, Props) -> - {IteratorId, StartMS} = new_iterator_id(DSSessionId), - IteratorRef = #iterator_ref{ - ref_id = {DSSessionId, TopicFilter}, - it_id = IteratorId, +-spec session_insert_subscription(id(), topic_filter(), map()) -> ds_sub(). +session_insert_subscription(DSSessionId, TopicFilter, Props) -> + {DSSubId, StartMS} = new_subscription_id(DSSessionId, TopicFilter), + DSSub = #ds_sub{ + id = DSSubId, start_time = StartMS, - props = Props + props = Props, + extra = #{} }, - ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write), - IteratorRef. + ok = mnesia:write(?SESSION_SUBSCRIPTIONS_TAB, DSSub, write), + DSSub. -session_update_iterator(IteratorRef, Props) -> - NIteratorRef = IteratorRef#iterator_ref{props = Props}, - ok = mnesia:write(?ITERATOR_REF_TAB, NIteratorRef, write), - NIteratorRef. +-spec session_update_subscription(ds_sub(), map()) -> ds_sub(). +session_update_subscription(DSSub, Props) -> + NDSSub = DSSub#ds_sub{props = Props}, + ok = mnesia:write(?SESSION_SUBSCRIPTIONS_TAB, NDSSub, write), + NDSSub. -%% @doc Called when a client unsubscribes from a topic. --spec session_del_iterator(id(), topic_filter()) -> ok. -session_del_iterator(DSSessionId, TopicFilter) -> - IteratorRefId = {DSSessionId, TopicFilter}, +session_del_subscription(DSSessionId, TopicFilter) -> + DSSubId = {DSSessionId, TopicFilter}, transaction(fun() -> - mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write) + mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write) end). -session_del_iterator(#iterator_ref{ref_id = IteratorRefId}) -> - mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write). +session_del_subscription(#ds_sub{id = DSSubId}) -> + mnesia:delete(?SESSION_SUBSCRIPTIONS_TAB, DSSubId, write). -session_read_iterators(DSSessionId) -> - % NOTE: somewhat convoluted way to trick dialyzer - Pat = erlang:make_tuple(record_info(size, iterator_ref), '_', [ - {1, iterator_ref}, - {#iterator_ref.ref_id, {DSSessionId, '_'}} - ]), - mnesia:match_object(?ITERATOR_REF_TAB, Pat, read). +session_read_subscriptions(DSSessionId) -> + MS = ets:fun2ms( + fun(Sub = #ds_sub{id = {Sess, _}}) when Sess =:= DSSessionId -> + Sub + end + ), + mnesia:select(?SESSION_SUBSCRIPTIONS_TAB, MS, read). --spec new_iterator_id(id()) -> {iterator_id(), emqx_ds:time()}. -new_iterator_id(DSSessionId) -> - NowMS = erlang:system_time(microsecond), - IteratorId = <>, - {IteratorId, NowMS}. +-spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), integer()}. +new_subscription_id(DSSessionId, TopicFilter) -> + %% Note: here we use _milliseconds_ to match with the timestamp + %% field of `#message' record. + NowMS = erlang:system_time(millisecond), + DSSubId = {DSSessionId, TopicFilter}, + {DSSubId, NowMS}. + +-spec subscription_id_to_topic_filter(subscription_id()) -> topic_filter(). +subscription_id_to_topic_filter({_DSSessionId, TopicFilter}) -> + TopicFilter. + +%%-------------------------------------------------------------------- +%% RPC targets (v1) +%%-------------------------------------------------------------------- + +%% RPC target. +-spec do_open_iterator(emqx_types:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _Reason}. +do_open_iterator(_TopicFilter, _StartMS, _IteratorID) -> + {error, not_implemented}. + +%% RPC target. +-spec do_ensure_iterator_closed(emqx_ds:iterator_id()) -> ok. +do_ensure_iterator_closed(_IteratorID) -> + ok. + +%% RPC target. +-spec do_ensure_all_iterators_closed(id()) -> ok. +do_ensure_all_iterators_closed(_DSSessionID) -> + ok. + +%%-------------------------------------------------------------------- +%% Reading batches +%%-------------------------------------------------------------------- + +-spec renew_streams(id()) -> ok. +renew_streams(DSSessionId) -> + Subscriptions = ro_transaction(fun() -> session_read_subscriptions(DSSessionId) end), + ExistingStreams = ro_transaction(fun() -> mnesia:read(?SESSION_STREAM_TAB, DSSessionId) end), + lists:foreach( + fun(#ds_sub{id = {_, TopicFilter}, start_time = StartTime}) -> + renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime) + end, + Subscriptions + ). + +-spec renew_streams(id(), [ds_stream()], emqx_ds:topic_filter(), emqx_ds:time()) -> ok. +renew_streams(DSSessionId, ExistingStreams, TopicFilter, StartTime) -> + AllStreams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime), + transaction( + fun() -> + lists:foreach( + fun({Rank, Stream}) -> + Rec = #ds_stream{ + session = DSSessionId, + topic_filter = TopicFilter, + stream = Stream, + rank = Rank + }, + case lists:member(Rec, ExistingStreams) of + true -> + ok; + false -> + mnesia:write(?SESSION_STREAM_TAB, Rec, write), + {ok, Iterator} = emqx_ds:make_iterator( + ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime + ), + %% Workaround: we convert `Stream' to a binary before + %% attempting to store it in mnesia(rocksdb) because of a bug + %% in `mnesia_rocksdb' when trying to do + %% `mnesia:dirty_all_keys' later. + StreamBin = term_to_binary(Stream), + IterRec = #ds_iter{id = {DSSessionId, StreamBin}, iter = Iterator}, + mnesia:write(?SESSION_ITER_TAB, IterRec, write) + end + end, + AllStreams + ) + end + ). + +%% must be called inside a transaction +-spec session_drop_streams(id()) -> ok. +session_drop_streams(DSSessionId) -> + MS = ets:fun2ms( + fun(#ds_stream{session = DSSessionId0}) when DSSessionId0 =:= DSSessionId -> + DSSessionId0 + end + ), + StreamIDs = mnesia:select(?SESSION_STREAM_TAB, MS, write), + lists:foreach(fun(Key) -> mnesia:delete(?SESSION_STREAM_TAB, Key, write) end, StreamIDs). + +%% must be called inside a transaction +-spec session_drop_iterators(id()) -> ok. +session_drop_iterators(DSSessionId) -> + MS = ets:fun2ms( + fun(#ds_iter{id = {DSSessionId0, StreamBin}}) when DSSessionId0 =:= DSSessionId -> + StreamBin + end + ), + StreamBins = mnesia:select(?SESSION_ITER_TAB, MS, write), + lists:foreach( + fun(StreamBin) -> + mnesia:delete(?SESSION_ITER_TAB, {DSSessionId, StreamBin}, write) + end, + StreamBins + ). %%-------------------------------------------------------------------------------- @@ -649,23 +756,110 @@ transaction(Fun) -> {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun), Res. +ro_transaction(Fun) -> + {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun), + Res. + %%-------------------------------------------------------------------------------- -export_iterators(IteratorRefs) -> +export_subscriptions(DSSubs) -> lists:foldl( - fun(IteratorRef = #iterator_ref{ref_id = {_DSSessionId, TopicFilter}}, Acc) -> - Acc#{TopicFilter => export_record(IteratorRef)} + fun(DSSub = #ds_sub{id = {_DSSessionId, TopicFilter}}, Acc) -> + Acc#{TopicFilter => export_subscription(DSSub)} end, #{}, - IteratorRefs + DSSubs ). -export_record(#session{} = Record) -> - export_record(Record, #session.id, [id, created_at, expires_at, props], #{}); -export_record(#iterator_ref{} = Record) -> - export_record(Record, #iterator_ref.it_id, [id, start_time, props], #{}). +export_session(#session{} = Record) -> + export_record(Record, #session.id, [id, created_at, expires_at, inflight, props], #{}). + +export_subscription(#ds_sub{} = Record) -> + export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}). export_record(Record, I, [Field | Rest], Acc) -> export_record(Record, I + 1, Rest, Acc#{Field => element(I, Record)}); export_record(_, _, [], Acc) -> Acc. + +%% TODO: find a more reliable way to perform actions that have side +%% effects. Add `CBM:init' callback to the session behavior? +ensure_timers() -> + ensure_timer(pull), + ensure_timer(get_streams). + +-spec ensure_timer(pull | get_streams) -> ok. +ensure_timer(Type) -> + ensure_timer(Type, 100). + +-spec ensure_timer(pull | get_streams, non_neg_integer()) -> ok. +ensure_timer(Type, Timeout) -> + _ = emqx_utils:start_timer(Timeout, {emqx_session, Type}), + ok. + +-spec receive_maximum(conninfo()) -> pos_integer(). +receive_maximum(ConnInfo) -> + %% Note: the default value should be always set by the channel + %% with respect to the zone configuration, but the type spec + %% indicates that it's optional. + maps:get(receive_maximum, ConnInfo, 65_535). + +-ifdef(TEST). +list_all_sessions() -> + DSSessionIds = mnesia:dirty_all_keys(?SESSION_TAB), + Sessions = lists:map( + fun(SessionID) -> + {ok, Session, Subscriptions} = session_open(SessionID), + {SessionID, #{session => Session, subscriptions => Subscriptions}} + end, + DSSessionIds + ), + maps:from_list(Sessions). + +list_all_subscriptions() -> + DSSubIds = mnesia:dirty_all_keys(?SESSION_SUBSCRIPTIONS_TAB), + Subscriptions = lists:map( + fun(DSSubId) -> + [DSSub] = mnesia:dirty_read(?SESSION_SUBSCRIPTIONS_TAB, DSSubId), + {DSSubId, export_subscription(DSSub)} + end, + DSSubIds + ), + maps:from_list(Subscriptions). + +list_all_streams() -> + DSStreamIds = mnesia:dirty_all_keys(?SESSION_STREAM_TAB), + DSStreams = lists:map( + fun(DSStreamId) -> + Records = mnesia:dirty_read(?SESSION_STREAM_TAB, DSStreamId), + ExtDSStreams = + lists:map( + fun(Record) -> + export_record( + Record, + #ds_stream.session, + [session, topic_filter, stream, rank], + #{} + ) + end, + Records + ), + {DSStreamId, ExtDSStreams} + end, + DSStreamIds + ), + maps:from_list(DSStreams). + +list_all_iterators() -> + DSIterIds = mnesia:dirty_all_keys(?SESSION_ITER_TAB), + DSIters = lists:map( + fun(DSIterId) -> + [Record] = mnesia:dirty_read(?SESSION_ITER_TAB, DSIterId), + {DSIterId, export_record(Record, #ds_iter.id, [id, iter], #{})} + end, + DSIterIds + ), + maps:from_list(DSIters). + +%% ifdef(TEST) +-endif. diff --git a/apps/emqx/src/emqx_persistent_session_ds.hrl b/apps/emqx/src/emqx_persistent_session_ds.hrl new file mode 100644 index 000000000..cc995ce66 --- /dev/null +++ b/apps/emqx/src/emqx_persistent_session_ds.hrl @@ -0,0 +1,60 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_PERSISTENT_SESSION_DS_HRL_HRL). +-define(EMQX_PERSISTENT_SESSION_DS_HRL_HRL, true). + +-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message). + +-define(SESSION_TAB, emqx_ds_session). +-define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions). +-define(SESSION_STREAM_TAB, emqx_ds_stream_tab). +-define(SESSION_ITER_TAB, emqx_ds_iter_tab). +-define(DS_MRIA_SHARD, emqx_ds_session_shard). + +-record(ds_sub, { + id :: emqx_persistent_session_ds:subscription_id(), + start_time :: emqx_ds:time(), + props = #{} :: map(), + extra = #{} :: map() +}). +-type ds_sub() :: #ds_sub{}. + +-record(ds_stream, { + session :: emqx_persistent_session_ds:id(), + topic_filter :: emqx_ds:topic_filter(), + stream :: emqx_ds:stream(), + rank :: emqx_ds:stream_rank() +}). +-type ds_stream() :: #ds_stream{}. +-type ds_stream_bin() :: binary(). + +-record(ds_iter, { + id :: {emqx_persistent_session_ds:id(), ds_stream_bin()}, + iter :: emqx_ds:iterator() +}). + +-record(session, { + %% same as clientid + id :: emqx_persistent_session_ds:id(), + %% creation time + created_at :: _Millisecond :: non_neg_integer(), + expires_at = never :: _Millisecond :: non_neg_integer() | never, + inflight :: emqx_persistent_message_ds_replayer:inflight(), + %% for future usage + props = #{} :: map() +}). + +-endif. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 804a3a04c..3ad03c4d4 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -47,11 +47,9 @@ -type bytesize() :: integer(). -type wordsize() :: bytesize(). -type percent() :: float(). --type file() :: string(). --type comma_separated_list() :: list(). +-type comma_separated_list() :: list(string()). -type comma_separated_binary() :: [binary()]. -type comma_separated_atoms() :: [atom()]. --type bar_separated_list() :: list(). -type ip_port() :: tuple() | integer(). -type cipher() :: map(). -type port_number() :: 1..65535. @@ -75,7 +73,6 @@ -typerefl_from_string({percent/0, emqx_schema, to_percent}). -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). -typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}). --typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). @@ -118,7 +115,6 @@ to_percent/1, to_comma_separated_list/1, to_comma_separated_binary/1, - to_bar_separated_list/1, to_ip_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1, @@ -154,10 +150,8 @@ bytesize/0, wordsize/0, percent/0, - file/0, comma_separated_list/0, comma_separated_binary/0, - bar_separated_list/0, ip_port/0, cipher/0, comma_separated_atoms/0, @@ -1849,7 +1843,7 @@ base_listener(Bind) -> default => true } )} - ] ++ emqx_limiter_schema:short_paths_fields(?MODULE). + ] ++ emqx_limiter_schema:short_paths_fields(). desc("persistent_session_store") -> "Settings for message persistence."; @@ -2564,9 +2558,6 @@ to_json_binary(Str) -> Error end. -to_bar_separated_list(Str) -> - {ok, string:tokens(Str, "| ")}. - %% @doc support the following format: %% - 127.0.0.1:1883 %% - ::1:1883 @@ -3316,7 +3307,7 @@ get_tombstone_map_value_type(Schema) -> %% hoconsc:map_value_type(Schema) ?MAP(_Name, Union) = hocon_schema:field_schema(Schema, type), %% TODO: violation of abstraction, fix hoconsc:union_members/1 - ?UNION(Members) = Union, + ?UNION(Members, _) = Union, Tombstone = tombstone(), [Type, Tombstone] = hoconsc:union_members(Members), Type. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 8bdd47392..52342d7ee 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -176,6 +176,7 @@ t(). -callback open(clientinfo(), conninfo()) -> {_IsPresent :: true, t(), _ReplayContext} | false. +-callback destroy(t() | clientinfo()) -> ok. %%-------------------------------------------------------------------- %% Create a Session @@ -247,7 +248,14 @@ get_mqtt_conf(Zone, Key) -> -spec destroy(clientinfo(), conninfo()) -> ok. destroy(ClientInfo, ConnInfo) -> - (choose_impl_mod(ConnInfo)):destroy(ClientInfo). + %% When destroying/discarding a session, the current `ClientInfo' might suggest an + %% implementation which does not correspond to the one previously used by this client. + %% An example of this is a client that first connects with `Session-Expiry-Interval' > + %% 0, and later reconnects with `Session-Expiry-Interval' = 0 and `clean_start' = + %% true. So we may simply destroy sessions from all implementations, since the key + %% (ClientID) is the same. + Mods = choose_impl_candidates(ConnInfo), + lists:foreach(fun(Mod) -> Mod:destroy(ClientInfo) end, Mods). -spec destroy(t()) -> ok. destroy(Session) -> diff --git a/apps/emqx/src/emqx_session_mem.erl b/apps/emqx/src/emqx_session_mem.erl index e72feffd5..3ea4f9f3b 100644 --- a/apps/emqx/src/emqx_session_mem.erl +++ b/apps/emqx/src/emqx_session_mem.erl @@ -44,6 +44,8 @@ %% State is stored in-memory in the process heap. -module(emqx_session_mem). +-behaviour(emqx_session). + -include("emqx.hrl"). -include("emqx_mqtt.hrl"). -include("emqx_session_mem.hrl"). diff --git a/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl b/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl index d35ccd963..e879b495c 100644 --- a/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, open_iterator/4, close_iterator/2, @@ -31,9 +32,11 @@ -define(TIMEOUT, 30_000). introduced_in() -> - %% FIXME "5.3.0". +deprecated_since() -> + "5.4.0". + -spec open_iterator( [node()], emqx_types:words(), diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 6766912c0..657776317 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -244,19 +244,28 @@ get_param_types(Signatures, {M, F, A}) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% dump() -> - case - { - filelib:wildcard(project_root_dir() ++ "/*_plt"), - filelib:wildcard(project_root_dir() ++ "/_build/check/lib") - } - of + RootDir = project_root_dir(), + TryRelDir = RootDir ++ "/_build/check/lib", + case {filelib:wildcard(RootDir ++ "/*_plt"), filelib:wildcard(TryRelDir)} of {[PLT | _], [RelDir | _]} -> dump(#{ plt => PLT, reldir => RelDir }); - _ -> - error("failed to guess run options") + {[], _} -> + logger:error( + "No usable PLT files found in \"~s\", abort ~n" + "Try running `rebar3 as check dialyzer` at least once first", + [RootDir] + ), + error(run_failed); + {_, []} -> + logger:error( + "No built applications found in \"~s\", abort ~n" + "Try running `rebar3 as check compile` at least once first", + [TryRelDir] + ), + error(run_failed) end. %% Collect the local BPAPI modules to a dump file @@ -411,10 +420,19 @@ setnok() -> put(bpapi_ok, false). dumps_dir() -> - filename:join(project_root_dir(), "apps/emqx/test/emqx_static_checks_data"). - -project_root_dir() -> - string:trim(os:cmd("git rev-parse --show-toplevel")). + filename:join(emqx_app_dir(), "test/emqx_static_checks_data"). versions_file() -> - filename:join(project_root_dir(), "apps/emqx/priv/bpapi.versions"). + filename:join(emqx_app_dir(), "priv/bpapi.versions"). + +emqx_app_dir() -> + Info = ?MODULE:module_info(compile), + case proplists:get_value(source, Info) of + Source when is_list(Source) -> + filename:dirname(filename:dirname(Source)); + undefined -> + "apps/emqx" + end. + +project_root_dir() -> + filename:dirname(filename:dirname(emqx_app_dir())). diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 6afdfa478..9bfb4d5e7 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -306,7 +306,7 @@ test_stepdown_session(Action, Reason) -> ok = emqx_cm:register_channel(ClientId, Pid1, ConnInfo), ok = emqx_cm:register_channel(ClientId, Pid1, ConnInfo), ok = emqx_cm:register_channel(ClientId, Pid2, ConnInfo), - ?assertEqual([Pid1, Pid2], lists:sort(emqx_cm:lookup_channels(ClientId))), + ?assertEqual(lists:sort([Pid1, Pid2]), lists:sort(emqx_cm:lookup_channels(ClientId))), case Reason of noproc -> exit(Pid1, kill), diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index 751b7e4b8..45cf85a05 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -26,9 +26,7 @@ -import(emqx_common_test_helpers, [on_exit/1]). --define(DEFAULT_KEYSPACE, default). --define(DS_SHARD_ID, <<"local">>). --define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). +-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message). all() -> emqx_common_test_helpers:all(?MODULE). @@ -48,6 +46,7 @@ init_per_testcase(t_session_subscription_iterators = TestCase, Config) -> Nodes = emqx_cth_cluster:start(Cluster, #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}), [{nodes, Nodes} | Config]; init_per_testcase(TestCase, Config) -> + ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB), Apps = emqx_cth_suite:start( app_specs(), #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} @@ -58,10 +57,11 @@ end_per_testcase(t_session_subscription_iterators, Config) -> Nodes = ?config(nodes, Config), emqx_common_test_helpers:call_janitor(60_000), ok = emqx_cth_cluster:stop(Nodes), - ok; + end_per_testcase(common, Config); end_per_testcase(_TestCase, Config) -> - Apps = ?config(apps, Config), + Apps = proplists:get_value(apps, Config, []), emqx_common_test_helpers:call_janitor(60_000), + clear_db(), emqx_cth_suite:stop(Apps), ok. @@ -95,14 +95,15 @@ t_messages_persisted(_Config) -> Results = [emqtt:publish(CP, Topic, Payload, 1) || {Topic, Payload} <- Messages], ct:pal("Results = ~p", [Results]), + timer:sleep(2000), - Persisted = consume(?DS_SHARD, {['#'], 0}), + Persisted = consume(['#'], 0), ct:pal("Persisted = ~p", [Persisted]), ?assertEqual( - [M1, M2, M5, M7, M9, M10], - [{emqx_message:topic(M), emqx_message:payload(M)} || M <- Persisted] + lists:sort([M1, M2, M5, M7, M9, M10]), + lists:sort([{emqx_message:topic(M), emqx_message:payload(M)} || M <- Persisted]) ), ok. @@ -139,23 +140,25 @@ t_messages_persisted_2(_Config) -> {ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}} = emqtt:publish(CP, T(<<"client/2/topic">>), <<"8">>, 1), - Persisted = consume(?DS_SHARD, {['#'], 0}), + timer:sleep(2000), + + Persisted = consume(['#'], 0), ct:pal("Persisted = ~p", [Persisted]), ?assertEqual( - [ + lists:sort([ {T(<<"client/1/topic">>), <<"4">>}, {T(<<"client/2/topic">>), <<"5">>} - ], - [{emqx_message:topic(M), emqx_message:payload(M)} || M <- Persisted] + ]), + lists:sort([{emqx_message:topic(M), emqx_message:payload(M)} || M <- Persisted]) ), ok. %% TODO: test quic and ws too t_session_subscription_iterators(Config) -> - [Node1, Node2] = ?config(nodes, Config), + [Node1, _Node2] = ?config(nodes, Config), Port = get_mqtt_port(Node1, tcp), Topic = <<"t/topic">>, SubTopicFilter = <<"t/+">>, @@ -202,11 +205,8 @@ t_session_subscription_iterators(Config) -> messages => [Message1, Message2, Message3, Message4] } end, - fun(Results, Trace) -> + fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), - #{ - messages := [_Message1, Message2, Message3 | _] - } = Results, case ?of_kind(ds_session_subscription_added, Trace) of [] -> %% Since `emqx_durable_storage' is a dependency of `emqx', it gets @@ -228,17 +228,6 @@ t_session_subscription_iterators(Config) -> ), ok end, - ?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)), - {ok, [IteratorId]} = get_all_iterator_ids(Node1), - ?assertMatch({ok, [IteratorId]}, get_all_iterator_ids(Node2)), - ReplayMessages1 = erpc:call(Node1, fun() -> consume(?DS_SHARD, IteratorId) end), - ExpectedMessages = [Message2, Message3], - %% Note: it is expected that this will break after replayers are in place. - %% They might have consumed all the messages by this time. - ?assertEqual(ExpectedMessages, ReplayMessages1), - %% Different DS shard - ReplayMessages2 = erpc:call(Node2, fun() -> consume(?DS_SHARD, IteratorId) end), - ?assertEqual([], ReplayMessages2), ok end ), @@ -263,33 +252,26 @@ connect(Opts0 = #{}) -> {ok, _} = emqtt:connect(Client), Client. -consume(Shard, Replay = {_TopicFiler, _StartMS}) -> - {ok, It} = emqx_ds_storage_layer:make_iterator(Shard, Replay), - consume(It); -consume(Shard, IteratorId) when is_binary(IteratorId) -> - {ok, It} = emqx_ds_storage_layer:restore_iterator(Shard, IteratorId), - consume(It). +consume(TopicFilter, StartMS) -> + Streams = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartMS), + lists:flatmap( + fun({_Rank, Stream}) -> + {ok, It} = emqx_ds:make_iterator(?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartMS), + consume(It) + end, + Streams + ). consume(It) -> - case emqx_ds_storage_layer:next(It) of - {value, Msg, NIt} -> - [emqx_persistent_message:deserialize(Msg) | consume(NIt)]; - none -> + case emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, 100) of + {ok, _NIt, _Msgs = []} -> + []; + {ok, NIt, Msgs} -> + Msgs ++ consume(NIt); + {ok, end_of_stream} -> [] end. -delete_all_messages() -> - Persisted = consume(?DS_SHARD, {['#'], 0}), - lists:foreach( - fun(Msg) -> - GUID = emqx_message:id(Msg), - Topic = emqx_topic:words(emqx_message:topic(Msg)), - Timestamp = emqx_guid:timestamp(GUID), - ok = emqx_ds_storage_layer:delete(?DS_SHARD, GUID, Timestamp, Topic) - end, - Persisted - ). - receive_messages(Count) -> receive_messages(Count, []). @@ -306,13 +288,6 @@ receive_messages(Count, Msgs) -> publish(Node, Message) -> erpc:call(Node, emqx, publish, [Message]). -get_iterator_ids(Node, ClientId) -> - Channel = erpc:call(Node, fun() -> - [ConnPid] = emqx_cm:lookup_channels(ClientId), - sys:get_state(ConnPid) - end), - emqx_connection:info({channel, {session, iterators}}, Channel). - app_specs() -> [ emqx_durable_storage, @@ -330,5 +305,6 @@ get_mqtt_port(Node, Type) -> {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]), Port. -get_all_iterator_ids(Node) -> - erpc:call(Node, emqx_ds_storage_layer, list_iterator_prefix, [?DS_SHARD, <<>>]). +clear_db() -> + ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB), + ok. diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index be3bf6e6a..bd7ca1c46 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -24,6 +24,8 @@ -compile(export_all). -compile(nowarn_export_all). +-define(PERSISTENT_MESSAGE_DB, emqx_persistent_message). + %%-------------------------------------------------------------------- %% SUITE boilerplate %%-------------------------------------------------------------------- @@ -131,6 +133,7 @@ get_listener_port(Type, Name) -> end_per_group(Group, Config) when Group == tcp; Group == ws; Group == quic -> ok = emqx_cth_suite:stop(?config(group_apps, Config)); end_per_group(_, _Config) -> + catch emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB), ok. init_per_testcase(TestCase, Config) -> @@ -188,7 +191,7 @@ receive_messages(Count, Msgs) -> receive_messages(Count - 1, [Msg | Msgs]); _Other -> receive_messages(Count, Msgs) - after 5000 -> + after 15000 -> Msgs end. @@ -227,11 +230,11 @@ wait_for_cm_unregister(ClientId, N) -> end. publish(Topic, Payloads) -> - publish(Topic, Payloads, false). + publish(Topic, Payloads, false, 2). -publish(Topic, Payloads, WaitForUnregister) -> +publish(Topic, Payloads, WaitForUnregister, QoS) -> Fun = fun(Client, Payload) -> - {ok, _} = emqtt:publish(Client, Topic, Payload, 2) + {ok, _} = emqtt:publish(Client, Topic, Payload, QoS) end, do_publish(Payloads, Fun, WaitForUnregister). @@ -510,6 +513,48 @@ t_process_dies_session_expires(Config) -> emqtt:disconnect(Client2). +t_publish_while_client_is_gone_qos1(Config) -> + %% A persistent session should receive messages in its + %% subscription even if the process owning the session dies. + ConnFun = ?config(conn_fun, Config), + Topic = ?config(topic, Config), + STopic = ?config(stopic, Config), + Payload1 = <<"hello1">>, + Payload2 = <<"hello2">>, + ClientId = ?config(client_id, Config), + {ok, Client1} = emqtt:start_link([ + {proto_ver, v5}, + {clientid, ClientId}, + {properties, #{'Session-Expiry-Interval' => 30}}, + {clean_start, true} + | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), + {ok, _, [1]} = emqtt:subscribe(Client1, STopic, qos1), + + ok = emqtt:disconnect(Client1), + maybe_kill_connection_process(ClientId, Config), + + ok = publish(Topic, [Payload1, Payload2], false, 1), + + {ok, Client2} = emqtt:start_link([ + {proto_ver, v5}, + {clientid, ClientId}, + {properties, #{'Session-Expiry-Interval' => 30}}, + {clean_start, false} + | Config + ]), + {ok, _} = emqtt:ConnFun(Client2), + Msgs = receive_messages(2), + ?assertMatch([_, _], Msgs), + [Msg2, Msg1] = Msgs, + ?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)), + ?assertEqual({ok, 1}, maps:find(qos, Msg1)), + ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg2)), + ?assertEqual({ok, 1}, maps:find(qos, Msg2)), + + ok = emqtt:disconnect(Client2). + t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config); t_publish_while_client_is_gone('end', _Config) -> ok. t_publish_while_client_is_gone(Config) -> @@ -554,6 +599,7 @@ t_publish_while_client_is_gone(Config) -> ok = emqtt:disconnect(Client2). +%% TODO: don't skip after QoS2 support is added to DS. t_clean_start_drops_subscriptions(init, Config) -> skip_ds_tc(Config); t_clean_start_drops_subscriptions('end', _Config) -> ok. t_clean_start_drops_subscriptions(Config) -> diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index cfd2aa447..3d9109fd1 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth, [ {description, "EMQX Authentication and authorization"}, - {vsn, "0.1.27"}, + {vsn, "0.1.28"}, {modules, []}, {registered, [emqx_auth_sup]}, {applications, [ diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl index 9938a3018..f30f7f473 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl @@ -147,7 +147,7 @@ schema("/authentication") -> description => ?DESC(authentication_get), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - hoconsc:array(emqx_authn_schema:authenticator_type()), + hoconsc:array(authenticator_type(config)), authenticator_array_example() ) } @@ -156,12 +156,12 @@ schema("/authentication") -> tags => ?API_TAGS_GLOBAL, description => ?DESC(authentication_post), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -178,7 +178,7 @@ schema("/authentication/:id") -> parameters => [param_auth_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) @@ -189,7 +189,7 @@ schema("/authentication/:id") -> description => ?DESC(authentication_id_put), parameters => [param_auth_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ @@ -236,7 +236,7 @@ schema("/listeners/:listener_id/authentication") -> parameters => [param_listener_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - hoconsc:array(emqx_authn_schema:authenticator_type()), + hoconsc:array(authenticator_type(config)), authenticator_array_example() ) } @@ -247,12 +247,12 @@ schema("/listeners/:listener_id/authentication") -> description => ?DESC(listeners_listener_id_authentication_post), parameters => [param_listener_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -270,7 +270,7 @@ schema("/listeners/:listener_id/authentication/:id") -> parameters => [param_listener_id(), param_auth_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) @@ -282,7 +282,7 @@ schema("/listeners/:listener_id/authentication/:id") -> description => ?DESC(listeners_listener_id_authentication_id_put), parameters => [param_listener_id(), param_auth_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ @@ -1278,6 +1278,9 @@ paginated_list_type(Type) -> {meta, ref(emqx_dashboard_swagger, meta)} ]. +authenticator_type(Kind) -> + emqx_authn_schema:authenticator_type(Kind). + authenticator_array_example() -> [Config || #{value := Config} <- maps:values(authenticator_examples())]. diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl index 66bc6bfc6..756f39d06 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl @@ -53,7 +53,8 @@ -export([ type_ro/1, - type_rw/1 + type_rw/1, + type_rw_api/1 ]). -export([ @@ -67,21 +68,17 @@ -define(SALT_ROUNDS_MAX, 10). namespace() -> "authn-hash". -roots() -> [pbkdf2, bcrypt, bcrypt_rw, simple]. +roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple]. fields(bcrypt_rw) -> fields(bcrypt) ++ [ - {salt_rounds, - sc( - range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX), - #{ - default => ?SALT_ROUNDS_MAX, - example => ?SALT_ROUNDS_MAX, - desc => "Work factor for BCRYPT password generation.", - converter => fun salt_rounds_converter/2 - } - )} + {salt_rounds, fun bcrypt_salt_rounds/1} + ]; +fields(bcrypt_rw_api) -> + fields(bcrypt) ++ + [ + {salt_rounds, fun bcrypt_salt_rounds_api/1} ]; fields(bcrypt) -> [{name, sc(bcrypt, #{required => true, desc => "BCRYPT password hashing."})}]; @@ -95,7 +92,7 @@ fields(pbkdf2) -> )}, {iterations, sc( - integer(), + pos_integer(), #{required => true, desc => "Iteration count for PBKDF2 hashing algorithm."} )}, {dk_length, fun dk_length/1} @@ -110,6 +107,15 @@ fields(simple) -> {salt_position, fun salt_position/1} ]. +bcrypt_salt_rounds(converter) -> fun salt_rounds_converter/2; +bcrypt_salt_rounds(Option) -> bcrypt_salt_rounds_api(Option). + +bcrypt_salt_rounds_api(type) -> range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX); +bcrypt_salt_rounds_api(default) -> ?SALT_ROUNDS_MAX; +bcrypt_salt_rounds_api(example) -> ?SALT_ROUNDS_MAX; +bcrypt_salt_rounds_api(desc) -> "Work factor for BCRYPT password generation."; +bcrypt_salt_rounds_api(_) -> undefined. + salt_rounds_converter(undefined, _) -> undefined; salt_rounds_converter(I, _) when is_integer(I) -> @@ -119,6 +125,8 @@ salt_rounds_converter(X, _) -> desc(bcrypt_rw) -> "Settings for bcrypt password hashing algorithm (for DB backends with write capability)."; +desc(bcrypt_rw_api) -> + desc(bcrypt_rw); desc(bcrypt) -> "Settings for bcrypt password hashing algorithm."; desc(pbkdf2) -> @@ -143,14 +151,20 @@ dk_length(desc) -> dk_length(_) -> undefined. -%% for simple_authn/emqx_authn_mnesia +%% for emqx_authn_mnesia type_rw(type) -> hoconsc:union(rw_refs()); -type_rw(default) -> - #{<<"name">> => sha256, <<"salt_position">> => prefix}; type_rw(desc) -> "Options for password hash creation and verification."; -type_rw(_) -> +type_rw(Option) -> + type_ro(Option). + +%% for emqx_authn_mnesia API +type_rw_api(type) -> + hoconsc:union(api_refs()); +type_rw_api(desc) -> + "Options for password hash creation and verification through API."; +type_rw_api(_) -> undefined. %% for other authn resources @@ -242,31 +256,41 @@ check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHa %%------------------------------------------------------------------------------ rw_refs() -> - All = [ - hoconsc:ref(?MODULE, bcrypt_rw), - hoconsc:ref(?MODULE, pbkdf2), - hoconsc:ref(?MODULE, simple) - ], - fun - (all_union_members) -> All; - ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt_rw)]; - ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)]; - ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)]; - ({value, _}) -> throw(#{reason => "algorithm_name_missing"}) - end. + union_selector(rw). ro_refs() -> - All = [ - hoconsc:ref(?MODULE, bcrypt), - hoconsc:ref(?MODULE, pbkdf2), - hoconsc:ref(?MODULE, simple) - ], + union_selector(ro). + +api_refs() -> + union_selector(api). + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). + +union_selector(Kind) -> fun - (all_union_members) -> All; - ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt)]; - ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)]; - ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)]; + (all_union_members) -> refs(Kind); + ({value, #{<<"name">> := <<"bcrypt">>}}) -> [bcrypt_ref(Kind)]; + ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [pbkdf2_ref(Kind)]; + ({value, #{<<"name">> := _}}) -> [simple_ref(Kind)]; ({value, _}) -> throw(#{reason => "algorithm_name_missing"}) end. -sc(Type, Meta) -> hoconsc:mk(Type, Meta). +refs(Kind) -> + [ + bcrypt_ref(Kind), + pbkdf2_ref(Kind), + simple_ref(Kind) + ]. + +pbkdf2_ref(_) -> + hoconsc:ref(?MODULE, pbkdf2). + +bcrypt_ref(rw) -> + hoconsc:ref(?MODULE, bcrypt_rw); +bcrypt_ref(api) -> + hoconsc:ref(?MODULE, bcrypt_rw_api); +bcrypt_ref(_) -> + hoconsc:ref(?MODULE, bcrypt). + +simple_ref(_) -> + hoconsc:ref(?MODULE, simple). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index a06d4b692..371c6f2be 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -34,26 +34,50 @@ tags/0, fields/1, authenticator_type/0, + authenticator_type/1, authenticator_type_without/1, + authenticator_type_without/2, mechanism/1, - backend/1 + backend/1, + namespace/0 ]). -export([ global_auth_fields/0 ]). +-export_type([shema_kind/0]). + -define(AUTHN_MODS_PT_KEY, {?MODULE, authn_schema_mods}). +-define(DEFAULT_SCHEMA_KIND, config). %%-------------------------------------------------------------------- %% Authn Source Schema Behaviour %%-------------------------------------------------------------------- -type schema_ref() :: ?R_REF(module(), hocon_schema:name()). +-type shema_kind() :: + %% api_write: schema for mutating API request validation + api_write + %% config: schema for config validation + | config. +-callback namespace() -> string(). -callback refs() -> [schema_ref()]. --callback select_union_member(emqx_config:raw_config()) -> schema_ref() | undefined | no_return(). +-callback refs(shema_kind()) -> [schema_ref()]. +-callback select_union_member(emqx_config:raw_config()) -> [schema_ref()] | undefined | no_return(). +-callback select_union_member(shema_kind(), emqx_config:raw_config()) -> + [schema_ref()] | undefined | no_return(). -callback fields(hocon_schema:name()) -> [hocon_schema:field()]. +-optional_callbacks([ + select_union_member/1, + select_union_member/2, + refs/0, + refs/1 +]). + +namespace() -> "authn". + roots() -> []. injected_fields(AuthnSchemaMods) -> @@ -67,45 +91,63 @@ tags() -> [<<"Authentication">>]. authenticator_type() -> - hoconsc:union(union_member_selector(provider_schema_mods())). + authenticator_type(?DEFAULT_SCHEMA_KIND). + +authenticator_type(Kind) -> + hoconsc:union(union_member_selector(Kind, provider_schema_mods())). authenticator_type_without(ProviderSchemaMods) -> + authenticator_type_without(?DEFAULT_SCHEMA_KIND, ProviderSchemaMods). + +authenticator_type_without(Kind, ProviderSchemaMods) -> hoconsc:union( - union_member_selector(provider_schema_mods() -- ProviderSchemaMods) + union_member_selector(Kind, provider_schema_mods() -- ProviderSchemaMods) ). -union_member_selector(Mods) -> - AllTypes = config_refs(Mods), +union_member_selector(Kind, Mods) -> + AllTypes = config_refs(Kind, Mods), fun (all_union_members) -> AllTypes; - ({value, Value}) -> select_union_member(Value, Mods) + ({value, Value}) -> select_union_member(Kind, Value, Mods) end. -select_union_member(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) -> +select_union_member(_Kind, #{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) -> throw(#{ reason => "unsupported_mechanism", mechanism => Mechanism, backend => Backend }); -select_union_member(#{<<"mechanism">> := Mechanism}, []) -> +select_union_member(_Kind, #{<<"mechanism">> := Mechanism}, []) -> throw(#{ reason => "unsupported_mechanism", mechanism => Mechanism }); -select_union_member(#{<<"mechanism">> := _} = Value, [Mod | Mods]) -> - case Mod:select_union_member(Value) of +select_union_member(Kind, #{<<"mechanism">> := _} = Value, [Mod | Mods]) -> + case mod_select_union_member(Kind, Value, Mod) of undefined -> - select_union_member(Value, Mods); + select_union_member(Kind, Value, Mods); Member -> Member end; -select_union_member(#{} = _Value, _Mods) -> +select_union_member(_Kind, #{} = _Value, _Mods) -> throw(#{reason => "missing_mechanism_field"}); -select_union_member(Value, _Mods) -> +select_union_member(_Kind, Value, _Mods) -> throw(#{reason => "not_a_struct", value => Value}). -config_refs(Mods) -> - lists:append([Mod:refs() || Mod <- Mods]). +mod_select_union_member(Kind, Value, Mod) -> + emqx_utils:call_first_defined([ + {Mod, select_union_member, [Kind, Value]}, + {Mod, select_union_member, [Value]} + ]). + +config_refs(Kind, Mods) -> + lists:append([mod_refs(Kind, Mod) || Mod <- Mods]). + +mod_refs(Kind, Mod) -> + emqx_utils:call_first_defined([ + {Mod, refs, [Kind]}, + {Mod, refs, []} + ]). root_type() -> hoconsc:array(authenticator_type()). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a9d672922..f782e0e6c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authn.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ create_resource/3, @@ -44,13 +45,13 @@ default_headers_no_content_type/0 ]). --define(AUTHN_PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PASSWORD, - ?PH_PEERHOST, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PASSWORD, + ?VAR_PEERHOST, + ?VAR_CERT_SUBJECT, + ?VAR_CERT_CN_NAME ]). -define(DEFAULT_RESOURCE_OPTS, #{ @@ -107,48 +108,96 @@ check_password_from_selected_map(Algorithm, Selected, Password) -> end. parse_deep(Template) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_template:parse_deep(Template), + handle_disallowed_placeholders(Result, {deep, Template}). parse_str(Template) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_template:parse(Template), + handle_disallowed_placeholders(Result, {string, Template}). parse_sql(Template, ReplaceWith) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => ?AUTHN_PLACEHOLDERS, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + {Statement, handle_disallowed_placeholders(Result, {string, Template})}. + +handle_disallowed_placeholders(Template, Source) -> + case emqx_template:validate(?ALLOWED_VARS, Template) of + ok -> + Template; + {error, Disallowed} -> + ?tp(warning, "authn_template_invalid", #{ + template => Source, + reason => Disallowed, + allowed => #{placeholders => ?ALLOWED_VARS}, + notice => + "Disallowed placeholders will be rendered as is." + " However, consider using `${$}` escaping for literal `$` where" + " needed to avoid unexpected results." + }), + Result = prerender_disallowed_placeholders(Template), + case Source of + {string, _} -> + emqx_template:parse(Result); + {deep, _} -> + emqx_template:parse_deep(Result) + end + end. + +prerender_disallowed_placeholders(Template) -> + {Result, _} = emqx_template:render(Template, #{}, #{ + var_trans => fun(Name, _) -> + % NOTE + % Rendering disallowed placeholders in escaped form, which will then + % parse as a literal string. + case lists:member(Name, ?ALLOWED_VARS) of + true -> "${" ++ Name ++ "}"; + false -> "${$}{" ++ Name ++ "}" + end + end + }), + Result. render_deep(Template, Credential) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun to_string/2} + ), + Term. render_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun to_string/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun to_urlencoded_string/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_template:render( ParamList, mapping_credential(Credential), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun to_sql_valaue/2} + ), + Row. is_superuser(#{<<"is_superuser">> := Value}) -> #{is_superuser => to_bool(Value)}; @@ -269,22 +318,24 @@ without_password(Credential, [Name | Rest]) -> without_password(Credential, Rest) end. -urlencode_var(Var, Value) -> - emqx_http_lib:uri_encode(handle_var(Var, Value)). +to_urlencoded_string(Name, Value) -> + emqx_http_lib:uri_encode(to_string(Name, Value)). -handle_var(_Name, undefined) -> - <<>>; -handle_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); -handle_var(_, Value) -> - emqx_placeholder:bin(Value). +to_string(Name, Value) -> + emqx_template:to_string(render_var(Name, Value)). -handle_sql_var(_Name, undefined) -> +to_sql_valaue(Name, Value) -> + emqx_utils_sql:to_sql_value(render_var(Name, Value)). + +render_var(_, undefined) -> + % NOTE + % Any allowed but undefined binding will be replaced with empty string, even when + % rendering SQL values. <<>>; -handle_sql_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); -handle_sql_var(_, Value) -> - emqx_placeholder:sql_data(Value). +render_var(?VAR_PEERHOST, Value) -> + inet:ntoa(Value); +render_var(_Name, Value) -> + Value. mapping_credential(C = #{cn := CN, dn := DN}) -> C#{cert_common_name => CN, cert_subject => DN}; diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz.erl index 1eed8a663..80fdd4f2d 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz.erl @@ -511,7 +511,10 @@ do_authorize(_Client, _PubSub, _Topic, []) -> do_authorize(Client, PubSub, Topic, [#{enable := false} | Rest]) -> do_authorize(Client, PubSub, Topic, Rest); do_authorize( - Client, + #{ + username := Username, + peerhost := IpAddress + } = Client, PubSub, Topic, [Connector = #{type := Type} | Tail] @@ -521,11 +524,32 @@ do_authorize( try Module:authorize(Client, PubSub, Topic, Connector) of nomatch -> emqx_metrics_worker:inc(authz_metrics, Type, nomatch), + ?TRACE("AUTHZ", "authorization_module_nomatch", #{ + module => Module, + username => Username, + ipaddr => IpAddress, + topic => Topic, + pub_sub => PubSub + }), do_authorize(Client, PubSub, Topic, Tail); %% {matched, allow | deny | ignore} {matched, ignore} -> + ?TRACE("AUTHZ", "authorization_module_match_ignore", #{ + module => Module, + username => Username, + ipaddr => IpAddress, + topic => Topic, + pub_sub => PubSub + }), do_authorize(Client, PubSub, Topic, Tail); ignore -> + ?TRACE("AUTHZ", "authorization_module_ignore", #{ + module => Module, + username => Username, + ipaddr => IpAddress, + topic => Topic, + pub_sub => PubSub + }), do_authorize(Client, PubSub, Topic, Tail); %% {matched, allow | deny} Matched -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_api_sources.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_api_sources.erl index 247f3a9ac..00345a108 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_api_sources.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_api_sources.erl @@ -49,6 +49,8 @@ aggregate_metrics/1 ]). +-export([with_source/2]). + -define(TAGS, [<<"Authorization">>]). api_spec() -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 6e13cac91..ad6dec56b 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -183,19 +183,14 @@ compile_topic(<<"eq ", Topic/binary>>) -> compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> - TopicBin = bin(Topic), - case - emqx_placeholder:preproc_tmpl( - TopicBin, - #{placeholders => [?PH_USERNAME, ?PH_CLIENTID]} - ) - of - [{str, _}] -> emqx_topic:words(TopicBin); - Tokens -> {pattern, Tokens} + Template = emqx_authz_utils:parse_str(Topic, [?VAR_USERNAME, ?VAR_CLIENTID]), + case emqx_template:is_const(Template) of + true -> emqx_topic:words(bin(Topic)); + false -> {pattern, Template} end. bin(L) when is_list(L) -> - list_to_binary(L); + unicode:characters_to_binary(L); bin(B) when is_binary(B) -> B. @@ -307,7 +302,7 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = emqx_placeholder:proc_tmpl(PatternFilter, ClientInfo), + TopicFilter = bin(emqx_template:render_strict(PatternFilter, ClientInfo)), match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl index ac2c2503d..426c7a9f6 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl @@ -136,7 +136,7 @@ authz_fields() -> [ {sources, ?HOCON( - ?ARRAY(?UNION(UnionMemberSelector)), + ?ARRAY(hoconsc:union(UnionMemberSelector)), #{ default => [default_authz()], desc => ?DESC(sources), @@ -153,7 +153,7 @@ api_authz_fields() -> [{sources, ?HOCON(?ARRAY(api_source_type()), #{desc => ?DESC(sources)})}]. api_source_type() -> - ?UNION(api_authz_refs()). + hoconsc:union(api_authz_refs()). api_authz_refs() -> lists:concat([api_source_refs(Mod) || Mod <- source_schema_mods()]). diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 3a0d4f1a1..a17a563ae 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -16,7 +16,9 @@ -module(emqx_authz_utils). +-include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ cleanup_resources/0, @@ -108,48 +110,97 @@ update_config(Path, ConfigRequest) -> }). parse_deep(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => PlaceHolders}). + Result = emqx_template:parse_deep(Template), + handle_disallowed_placeholders(Result, {deep, Template}, PlaceHolders). parse_str(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => PlaceHolders}). + Result = emqx_template:parse(Template), + handle_disallowed_placeholders(Result, {string, Template}, PlaceHolders). parse_sql(Template, ReplaceWith, PlaceHolders) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => PlaceHolders, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + FResult = handle_disallowed_placeholders(Result, {string, Template}, PlaceHolders), + {Statement, FResult}. + +handle_disallowed_placeholders(Template, Source, Allowed) -> + case emqx_template:validate(Allowed, Template) of + ok -> + Template; + {error, Disallowed} -> + ?tp(warning, "authz_template_invalid", #{ + template => Source, + reason => Disallowed, + allowed => #{placeholders => Allowed}, + notice => + "Disallowed placeholders will be rendered as is." + " However, consider using `${$}` escaping for literal `$` where" + " needed to avoid unexpected results." + }), + Result = prerender_disallowed_placeholders(Template, Allowed), + case Source of + {string, _} -> + emqx_template:parse(Result); + {deep, _} -> + emqx_template:parse_deep(Result) + end + end. + +prerender_disallowed_placeholders(Template, Allowed) -> + {Result, _} = emqx_template:render(Template, #{}, #{ + var_trans => fun(Name, _) -> + % NOTE + % Rendering disallowed placeholders in escaped form, which will then + % parse as a literal string. + case lists:member(Name, Allowed) of + true -> "${" ++ Name ++ "}"; + false -> "${$}{" ++ Name ++ "}" + end + end + }), + Result. render_deep(Template, Values) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun to_string/2} + ), + Term. render_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun to_string/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun to_urlencoded_string/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_template:render( ParamList, client_vars(Values), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun to_sql_value/2} + ), + Row. -spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error. parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> @@ -215,22 +266,24 @@ convert_client_var({dn, DN}) -> {cert_subject, DN}; convert_client_var({protocol, Proto}) -> {proto_name, Proto}; convert_client_var(Other) -> Other. -urlencode_var(Var, Value) -> - emqx_http_lib:uri_encode(handle_var(Var, Value)). +to_urlencoded_string(Name, Value) -> + emqx_http_lib:uri_encode(to_string(Name, Value)). -handle_var(_Name, undefined) -> - <<>>; -handle_var([<<"peerhost">>], IpAddr) -> - inet_parse:ntoa(IpAddr); -handle_var(_Name, Value) -> - emqx_placeholder:bin(Value). +to_string(Name, Value) -> + emqx_template:to_string(render_var(Name, Value)). -handle_sql_var(_Name, undefined) -> +to_sql_value(Name, Value) -> + emqx_utils_sql:to_sql_value(render_var(Name, Value)). + +render_var(_, undefined) -> + % NOTE + % Any allowed but undefined binding will be replaced with empty string, even when + % rendering SQL values. <<>>; -handle_sql_var([<<"peerhost">>], IpAddr) -> - inet_parse:ntoa(IpAddr); -handle_sql_var(_Name, Value) -> - emqx_placeholder:sql_data(Value). +render_var(?VAR_PEERHOST, Value) -> + inet:ntoa(Value); +render_var(_Name, Value) -> + Value. bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl index 635b157d9..45a605e6e 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl @@ -63,14 +63,16 @@ end_per_testcase(_, Config) -> init_per_suite(Config) -> Apps = emqx_cth_suite:start( [ - emqx, emqx_conf, + emqx, emqx_auth, + %% to load schema + {emqx_auth_mnesia, #{start => false}}, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ], #{ - work_dir => ?config(priv_dir, Config) + work_dir => filename:join(?config(priv_dir, Config), ?MODULE) } ), _ = emqx_common_test_http:create_default_app(), @@ -535,6 +537,36 @@ ignore_switch_to_global_chain(_) -> ), ok = emqtt:disconnect(Client4). +t_bcrypt_validation(_Config) -> + BaseConf = #{ + mechanism => <<"password_based">>, + backend => <<"built_in_database">>, + user_id_type => <<"username">> + }, + BcryptValid = #{ + name => <<"bcrypt">>, + salt_rounds => 10 + }, + BcryptInvalid = #{ + name => <<"bcrypt">>, + salt_rounds => 15 + }, + + ConfValid = BaseConf#{password_hash_algorithm => BcryptValid}, + ConfInvalid = BaseConf#{password_hash_algorithm => BcryptInvalid}, + + {ok, 400, _} = request( + post, + uri([?CONF_NS]), + ConfInvalid + ), + + {ok, 200, _} = request( + post, + uri([?CONF_NS]), + ConfValid + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl index 747a1d15a..61a15b139 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_chains_SUITE.erl @@ -16,7 +16,6 @@ -module(emqx_authn_chains_SUITE). --behaviour(hocon_schema). -behaviour(emqx_authn_provider). -compile(export_all). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_password_hashing_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_password_hashing_SUITE.erl index 83b923d0e..ac3186bea 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_password_hashing_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_password_hashing_SUITE.erl @@ -185,3 +185,29 @@ hash_examples() -> } } ]. + +t_pbkdf2_schema(_Config) -> + Config = fun(Iterations) -> + #{ + <<"pbkdf2">> => #{ + <<"name">> => <<"pbkdf2">>, + <<"mac_fun">> => <<"sha">>, + <<"iterations">> => Iterations + } + } + end, + + ?assertException( + throw, + {emqx_authn_password_hashing, _}, + hocon_tconf:check_plain(emqx_authn_password_hashing, Config(0), #{}, [pbkdf2]) + ), + ?assertException( + throw, + {emqx_authn_password_hashing, _}, + hocon_tconf:check_plain(emqx_authn_password_hashing, Config(-1), #{}, [pbkdf2]) + ), + ?assertMatch( + #{<<"pbkdf2">> := _}, + hocon_tconf:check_plain(emqx_authn_password_hashing, Config(1), #{}, [pbkdf2]) + ). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl index 6d6ea420f..23532b4af 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl @@ -54,7 +54,7 @@ t_check_schema(_Config) -> ?assertThrow( #{ path := "authentication.1.password_hash_algorithm.name", - matched_type := "builtin_db/authn-hash:simple", + matched_type := "authn:builtin_db/authn-hash:simple", reason := unable_to_convert_to_enum_symbol }, Check(ConfigNotOk) @@ -73,7 +73,7 @@ t_check_schema(_Config) -> #{ path := "authentication.1.password_hash_algorithm", reason := "algorithm_name_missing", - matched_type := "builtin_db" + matched_type := "authn:builtin_db" }, Check(ConfigMissingAlgoName) ). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl index b0451e110..b4835cdaa 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl @@ -22,6 +22,7 @@ -define(ERR(Reason), {error, Reason}). union_member_selector_mongo_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -31,25 +32,26 @@ union_member_selector_mongo_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_single"}), + ?ERR(#{matched_type := "authn:mongo_single"}), check("{mechanism = password_based, backend = mongodb, mongo_type = single}") ) end}, {"replica-set", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_rs"}), + ?ERR(#{matched_type := "authn:mongo_rs"}), check("{mechanism = password_based, backend = mongodb, mongo_type = rs}") ) end}, {"sharded", fun() -> ?assertMatch( - ?ERR(#{matched_type := "mongo_sharded"}), + ?ERR(#{matched_type := "authn:mongo_sharded"}), check("{mechanism = password_based, backend = mongodb, mongo_type = sharded}") ) end} ]. union_member_selector_jwt_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -59,25 +61,26 @@ union_member_selector_jwt_test_() -> end}, {"jwks", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_jwks"}), + ?ERR(#{matched_type := "authn:jwt_jwks"}), check("{mechanism = jwt, use_jwks = true}") ) end}, {"publick-key", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_public_key"}), + ?ERR(#{matched_type := "authn:jwt_public_key"}), check("{mechanism = jwt, use_jwks = false, public_key = 1}") ) end}, {"hmac-based", fun() -> ?assertMatch( - ?ERR(#{matched_type := "jwt_hmac"}), + ?ERR(#{matched_type := "authn:jwt_hmac"}), check("{mechanism = jwt, use_jwks = false}") ) end} ]. union_member_selector_redis_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -87,25 +90,26 @@ union_member_selector_redis_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_single"}), + ?ERR(#{matched_type := "authn:redis_single"}), check("{mechanism = password_based, backend = redis, redis_type = single}") ) end}, {"cluster", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_cluster"}), + ?ERR(#{matched_type := "authn:redis_cluster"}), check("{mechanism = password_based, backend = redis, redis_type = cluster}") ) end}, {"sentinel", fun() -> ?assertMatch( - ?ERR(#{matched_type := "redis_sentinel"}), + ?ERR(#{matched_type := "authn:redis_sentinel"}), check("{mechanism = password_based, backend = redis, redis_type = sentinel}") ) end} ]. union_member_selector_http_test_() -> + ok = ensure_schema_load(), [ {"unknown", fun() -> ?assertMatch( @@ -115,13 +119,13 @@ union_member_selector_http_test_() -> end}, {"get", fun() -> ?assertMatch( - ?ERR(#{matched_type := "http_get"}), + ?ERR(#{matched_type := "authn:http_get"}), check("{mechanism = password_based, backend = http, method = get}") ) end}, {"post", fun() -> ?assertMatch( - ?ERR(#{matched_type := "http_post"}), + ?ERR(#{matched_type := "authn:http_post"}), check("{mechanism = password_based, backend = http, method = post}") ) end} @@ -132,3 +136,7 @@ check(HoconConf) -> #{roots => emqx_authn_schema:global_auth_fields()}, ["authentication= ", HoconConf] ). + +ensure_schema_load() -> + _ = emqx_conf_schema:roots(), + ok. diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl index 1af7d4d1d..37c9ebfc1 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl @@ -70,6 +70,7 @@ init_per_testcase(TestCase, Config) when {ok, _} = emqx:update_config([authorization, deny_action], disconnect), Config; init_per_testcase(_TestCase, Config) -> + _ = file:delete(emqx_authz_file:acl_conf_file()), {ok, _} = emqx_authz:update(?CMD_REPLACE, []), Config. diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl index b34e4fb00..d81a93038 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl @@ -67,6 +67,10 @@ set_special_configs(_App) -> ok. t_compile(_) -> + % NOTE + % Some of the following testcase are relying on the internal representation of + % `emqx_template:t()`. If the internal representation is changed, these testcases + % may fail. ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( @@ -74,13 +78,13 @@ t_compile(_) -> emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ), - ?assertEqual( + ?assertMatch( {allow, {ipaddrs, [ {{127, 0, 0, 1}, {127, 0, 0, 1}, 32}, {{192, 168, 1, 0}, {192, 168, 1, 255}, 24} ]}, - subscribe, [{pattern, [{var, [<<"clientid">>]}]}]}, + subscribe, [{pattern, [{var, "clientid", [_]}]}]}, emqx_authz_rule:compile( {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} ) @@ -102,7 +106,7 @@ t_compile(_) -> {clientid, {re_pattern, _, _, _, _}} ]}, publish, [ - {pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]} + {pattern, [{var, "username", [_]}]}, {pattern, [{var, "clientid", [_]}]} ]}, emqx_authz_rule:compile( {allow, @@ -114,9 +118,9 @@ t_compile(_) -> ) ), - ?assertEqual( + ?assertMatch( {allow, {username, {eq, <<"test">>}}, publish, [ - {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]} + {pattern, [<<"t/foo">>, {var, "username", [_]}, <<"boo">>]} ]}, emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index b5de90ad9..183b9a993 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_http, [ {description, "EMQX External HTTP API Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_http_app, []}}, {applications, [ diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index 1eaac6378..7b7af727d 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -16,10 +16,6 @@ -module(emqx_authn_http_schema). --include("emqx_auth_http.hrl"). --include_lib("emqx_auth/include/emqx_authn.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ @@ -27,9 +23,14 @@ validations/0, desc/1, refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +-include("emqx_auth_http.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)). -define(THROW_VALIDATION_ERROR(ERROR, MESSAGE), throw(#{ @@ -38,6 +39,8 @@ }) ). +namespace() -> "authn". + refs() -> [?R_REF(http_get), ?R_REF(http_post)]. @@ -97,7 +100,7 @@ common_fields() -> {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, {url, fun url/1}, {body, - hoconsc:mk(map([{fuzzy, term(), binary()}]), #{ + hoconsc:mk(typerefl:alias("map", map([{fuzzy, term(), binary()}])), #{ required => false, desc => ?DESC(body) })}, {request_timeout, fun request_timeout/1} diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index ed7051bb6..04f76b4c9 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -38,21 +38,21 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_PROTONAME, - ?PH_MOUNTPOINT, - ?PH_TOPIC, - ?PH_ACTION, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_PROTONAME, + ?VAR_MOUNTPOINT, + ?VAR_TOPIC, + ?VAR_ACTION, + ?VAR_CERT_SUBJECT, + ?VAR_CERT_CN_NAME ]). --define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ - ?PH_QOS, - ?PH_RETAIN +-define(ALLOWED_VARS_RICH_ACTIONS, [ + ?VAR_QOS, + ?VAR_RETAIN ]). description() -> @@ -157,14 +157,14 @@ parse_config( method => Method, base_url => BaseUrl, headers => Headers, - base_path_templete => emqx_authz_utils:parse_str(Path, placeholders()), + base_path_templete => emqx_authz_utils:parse_str(Path, allowed_vars()), base_query_template => emqx_authz_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)), - placeholders() + allowed_vars() ), body_template => emqx_authz_utils:parse_deep( maps:to_list(maps:get(body, Conf, #{})), - placeholders() + allowed_vars() ), request_timeout => ReqTimeout, %% pool_type default value `random` @@ -260,10 +260,10 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L); to_bin(X) -> X. -placeholders() -> - placeholders(emqx_authz:feature_available(rich_actions)). +allowed_vars() -> + allowed_vars(emqx_authz:feature_available(rich_actions)). -placeholders(true) -> - ?PLACEHOLDERS ++ ?PLACEHOLDERS_FOR_RICH_ACTIONS; -placeholders(false) -> - ?PLACEHOLDERS. +allowed_vars(true) -> + ?ALLOWED_VARS ++ ?ALLOWED_VARS_RICH_ACTIONS; +allowed_vars(false) -> + ?ALLOWED_VARS. diff --git a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl index 18ec23757..90a7439a2 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http_schema.erl @@ -26,7 +26,8 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). -export([ @@ -38,6 +39,8 @@ -import(emqx_schema, [mk_duration/2]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. source_refs() -> @@ -96,7 +99,7 @@ http_common_fields() -> mk_duration("Request timeout", #{ required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })}, - {body, ?HOCON(map(), #{required => false, desc => ?DESC(body)})} + {body, ?HOCON(hoconsc:map(name, binary()), #{required => false, desc => ?DESC(body)})} ] ++ lists:keydelete( pool_type, @@ -105,7 +108,7 @@ http_common_fields() -> ). headers(type) -> - list({binary(), binary()}); + typerefl:alias("map", list({binary(), binary()}), #{}, [binary(), binary()]); headers(desc) -> ?DESC(?FUNCTION_NAME); headers(converter) -> @@ -118,7 +121,7 @@ headers(_) -> undefined. headers_no_content_type(type) -> - list({binary(), binary()}); + typerefl:alias("map", list({binary(), binary()}), #{}, [binary(), binary()]); headers_no_content_type(desc) -> ?DESC(?FUNCTION_NAME); headers_no_content_type(converter) -> diff --git a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl index 577b3b638..e307b5bbf 100644 --- a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl @@ -27,7 +27,7 @@ -define(PATH, [?CONF_NS_ATOM]). -define(HTTP_PORT, 32333). --define(HTTP_PATH, "/auth"). +-define(HTTP_PATH, "/auth/[...]"). -define(CREDENTIALS, #{ clientid => <<"clienta">>, username => <<"plain">>, @@ -146,8 +146,12 @@ t_authenticate(_Config) -> test_user_auth(#{ handler := Handler, config_params := SpecificConfgParams, - result := Result + result := Expect }) -> + Result = perform_user_auth(SpecificConfgParams, Handler, ?CREDENTIALS), + ?assertEqual(Expect, Result). + +perform_user_auth(SpecificConfgParams, Handler, Credentials) -> AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams), {ok, _} = emqx:update_config( @@ -157,21 +161,21 @@ test_user_auth(#{ ok = emqx_authn_http_test_server:set_handler(Handler), - ?assertEqual(Result, emqx_access_control:authenticate(?CREDENTIALS)), + Result = emqx_access_control:authenticate(Credentials), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL - ). + ), + + Result. t_authenticate_path_placeholders(_Config) -> - ok = emqx_authn_http_test_server:stop(), - {ok, _} = emqx_authn_http_test_server:start_link(?HTTP_PORT, <<"/[...]">>), ok = emqx_authn_http_test_server:set_handler( fun(Req0, State) -> Req = case cowboy_req:path(Req0) of - <<"/my/p%20ath//us%20er/auth//">> -> + <<"/auth/p%20ath//us%20er/auth//">> -> cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, @@ -193,7 +197,7 @@ t_authenticate_path_placeholders(_Config) -> AuthConfig = maps:merge( raw_http_auth_config(), #{ - <<"url">> => <<"http://127.0.0.1:32333/my/p%20ath//${username}/auth//">>, + <<"url">> => <<"http://127.0.0.1:32333/auth/p%20ath//${username}/auth//">>, <<"body">> => #{} } ), @@ -255,6 +259,39 @@ t_no_value_for_placeholder(_Config) -> ?GLOBAL ). +t_disallowed_placeholders_preserved(_Config) -> + Config = #{ + <<"method">> => <<"post">>, + <<"headers">> => #{<<"content-type">> => <<"application/json">>}, + <<"body">> => #{ + <<"username">> => ?PH_USERNAME, + <<"password">> => ?PH_PASSWORD, + <<"this">> => <<"${whatisthis}">> + } + }, + Handler = fun(Req0, State) -> + {ok, Body, Req1} = cowboy_req:read_body(Req0), + #{ + <<"username">> := <<"plain">>, + <<"password">> := <<"plain">>, + <<"this">> := <<"${whatisthis}">> + } = emqx_utils_json:decode(Body), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(#{result => allow, is_superuser => false}), + Req1 + ), + {ok, Req, State} + end, + ?assertMatch({ok, _}, perform_user_auth(Config, Handler, ?CREDENTIALS)), + + % NOTE: disallowed placeholder left intact, which makes the URL invalid + ConfigUrl = Config#{ + <<"url">> => <<"http://127.0.0.1:32333/auth/${whatisthis}">> + }, + ?assertMatch({error, _}, perform_user_auth(ConfigUrl, Handler, ?CREDENTIALS)). + t_destroy(_Config) -> AuthConfig = raw_http_auth_config(), diff --git a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl index e56e25f5f..845259e78 100644 --- a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl @@ -494,6 +494,67 @@ t_no_value_for_placeholder(_Config) -> emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). +t_disallowed_placeholders_preserved(_Config) -> + ok = setup_handler_and_config( + fun(Req0, State) -> + {ok, Body, Req1} = cowboy_req:read_body(Req0), + ?assertMatch( + #{ + <<"cname">> := <<>>, + <<"usertypo">> := <<"${usertypo}">> + }, + emqx_utils_json:decode(Body) + ), + {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"cname">> => ?PH_CERT_CN_NAME, + <<"usertypo">> => <<"${usertypo}">> + } + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + zone => default, + listener => {tcp, default} + }, + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ). + +t_disallowed_placeholders_path(_Config) -> + ok = setup_handler_and_config( + fun(Req, State) -> + {ok, ?AUTHZ_HTTP_RESP(allow, Req), State} + end, + #{ + <<"url">> => <<"http://127.0.0.1:33333/authz/use%20rs/${typo}">> + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + zone => default, + listener => {tcp, default} + }, + + % % NOTE: disallowed placeholder left intact, which makes the URL invalid + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ). + t_create_replace(_Config) -> ClientInfo = #{ clientid => <<"clientid">>, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 4679e43bb..b4b5ccf02 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_jwt, [ {description, "EMQX JWT Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_jwt_app, []}}, {applications, [ diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl index fc7de7cd8..63da372ff 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_jwt_schema). --include("emqx_auth_jwt.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_jwt.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [ ?R_REF(jwt_hmac), @@ -149,7 +152,8 @@ refresh_interval(validator) -> [fun(I) -> I > 0 end]; refresh_interval(_) -> undefined. verify_claims(type) -> - list(); + %% user input is a map, converted to a list of {binary(), binary()} + typerefl:alias("map", list()); verify_claims(desc) -> ?DESC(?FUNCTION_NAME); verify_claims(default) -> diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 383c4822c..3d4d5f467 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_ldap, [ {description, "EMQX LDAP Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_ldap_app, []}}, {applications, [ diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index 8685faecd..975a7f828 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_authn_ldap). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl index 82f8b9443..000d545b9 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_authn_ldap_bind). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl index 9a21766e3..e5e83daa1 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_ldap_bind_schema). --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ fields/1, desc/1, refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +-include("emqx_auth_ldap.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(ldap_bind)]. diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl index c26ca94e8..fe9917fa1 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_ldap_schema). --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_ldap.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(ldap)]. diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl index eb12fdd37..84f9d7ed6 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -13,6 +13,18 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_authz_ldap). diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl index 491b0debf..e6a060f42 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap_schema.erl @@ -26,9 +26,12 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(ldap) -> diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index e75a9a617..63bceee85 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_authn_ldap_SUITE). diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index a796b8e01..1f390264b 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_authn_ldap_bind_SUITE). diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 569c0e887..210bb1bc9 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_authz_ldap_SUITE). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src index 988d300fb..5cc2c2a31 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mnesia, [ {description, "EMQX Buitl-in Database Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_mnesia_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl index 8e59d94e7..bbbaeddb1 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl @@ -50,7 +50,7 @@ %% Internal exports (RPC) -export([ do_destroy/1, - do_add_user/2, + do_add_user/1, do_delete_user/2, do_update_user/3, import/2, @@ -187,24 +187,22 @@ import_users({Filename0, FileData}, State) -> {error, {unsupported_file_format, Extension}} end. -add_user(UserInfo, State) -> - trans(fun ?MODULE:do_add_user/2, [UserInfo, State]). +add_user( + UserInfo, + State +) -> + UserInfoRecord = user_info_record(UserInfo, State), + trans(fun ?MODULE:do_add_user/1, [UserInfoRecord]). do_add_user( - #{ - user_id := UserID, - password := Password - } = UserInfo, - #{ - user_group := UserGroup, - password_hash_algorithm := Algorithm - } + #user_info{ + user_id = {_UserGroup, UserID} = DBUserID, + is_superuser = IsSuperuser + } = UserInfoRecord ) -> - case mnesia:read(?TAB, {UserGroup, UserID}, write) of + case mnesia:read(?TAB, DBUserID, write) of [] -> - {PasswordHash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password), - IsSuperuser = maps:get(is_superuser, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), + insert_user(UserInfoRecord), {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} @@ -222,38 +220,30 @@ do_delete_user(UserID, #{user_group := UserGroup}) -> end. update_user(UserID, UserInfo, State) -> - trans(fun ?MODULE:do_update_user/3, [UserID, UserInfo, State]). + FieldsToUpdate = fields_to_update( + UserInfo, + [ + hash_and_salt, + is_superuser + ], + State + ), + trans(fun ?MODULE:do_update_user/3, [UserID, FieldsToUpdate, State]). do_update_user( UserID, - UserInfo, + FieldsToUpdate, #{ - user_group := UserGroup, - password_hash_algorithm := Algorithm + user_group := UserGroup } ) -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [ - #user_info{ - password_hash = PasswordHash, - salt = Salt, - is_superuser = IsSuperuser - } - ] -> - NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser), - {NPasswordHash, NSalt} = - case UserInfo of - #{password := Password} -> - emqx_authn_password_hashing:hash( - Algorithm, Password - ); - #{} -> - {PasswordHash, Salt} - end, - insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser), - {ok, #{user_id => UserID, is_superuser => NSuperuser}} + [#user_info{} = UserInfoRecord] -> + NUserInfoRecord = update_user_record(UserInfoRecord, FieldsToUpdate), + insert_user(NUserInfoRecord), + {ok, #{user_id => UserID, is_superuser => NUserInfoRecord#user_info.is_superuser}} end. lookup_user(UserID, #{user_group := UserGroup}) -> @@ -391,13 +381,59 @@ get_user_info_by_seq(_, _, _) -> {error, bad_format}. insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> - UserInfo = #user_info{ + UserInfoRecord = user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), + insert_user(UserInfoRecord). + +insert_user(#user_info{} = UserInfoRecord) -> + mnesia:write(?TAB, UserInfoRecord, write). + +user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> + #user_info{ user_id = {UserGroup, UserID}, password_hash = PasswordHash, salt = Salt, is_superuser = IsSuperuser - }, - mnesia:write(?TAB, UserInfo, write). + }. + +user_info_record( + #{ + user_id := UserID, + password := Password + } = UserInfo, + #{ + password_hash_algorithm := Algorithm, + user_group := UserGroup + } = _State +) -> + IsSuperuser = maps:get(is_superuser, UserInfo, false), + {PasswordHash, Salt} = emqx_authn_password_hashing:hash(Algorithm, Password), + user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser). + +fields_to_update( + #{password := Password} = UserInfo, + [hash_and_salt | Rest], + #{password_hash_algorithm := Algorithm} = State +) -> + [ + {hash_and_salt, + emqx_authn_password_hashing:hash( + Algorithm, Password + )} + | fields_to_update(UserInfo, Rest, State) + ]; +fields_to_update(#{is_superuser := IsSuperuser} = UserInfo, [is_superuser | Rest], State) -> + [{is_superuser, IsSuperuser} | fields_to_update(UserInfo, Rest, State)]; +fields_to_update(UserInfo, [_ | Rest], State) -> + fields_to_update(UserInfo, Rest, State); +fields_to_update(_UserInfo, [], _State) -> + []. + +update_user_record(UserInfoRecord, []) -> + UserInfoRecord; +update_user_record(UserInfoRecord, [{hash_and_salt, {PasswordHash, Salt}} | Rest]) -> + update_user_record(UserInfoRecord#user_info{password_hash = PasswordHash, salt = Salt}, Rest); +update_user_record(UserInfoRecord, [{is_superuser, IsSuperuser} | Rest]) -> + update_user_record(UserInfoRecord#user_info{is_superuser = IsSuperuser}, Rest). %% TODO: Support other type get_user_identity(#{username := Username}, username) -> diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl index 2d57abc90..373d95fc8 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl @@ -24,27 +24,33 @@ -export([ fields/1, desc/1, - refs/0, - select_union_member/1 + refs/1, + select_union_member/2, + namespace/0 ]). -refs() -> +namespace() -> "authn". + +refs(api_write) -> + [?R_REF(builtin_db_api)]; +refs(_) -> [?R_REF(builtin_db)]. -select_union_member(#{ +select_union_member(Kind, #{ <<"mechanism">> := ?AUTHN_MECHANISM_SIMPLE_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN }) -> - refs(); -select_union_member(_) -> + refs(Kind); +select_union_member(_Kind, _Value) -> undefined. fields(builtin_db) -> [ - {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)}, - {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, - {user_id_type, fun user_id_type/1}, {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} - ] ++ emqx_authn_schema:common_fields(). + ] ++ common_fields(); +fields(builtin_db_api) -> + [ + {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1} + ] ++ common_fields(). desc(builtin_db) -> ?DESC(builtin_db); @@ -56,3 +62,10 @@ user_id_type(desc) -> ?DESC(?FUNCTION_NAME); user_id_type(default) -> <<"username">>; user_id_type(required) -> true; user_id_type(_) -> undefined. + +common_fields() -> + [ + {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)}, + {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, + {user_id_type, fun user_id_type/1} + ] ++ emqx_authn_schema:common_fields(). diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl index 641efcf74..a66ae5786 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl @@ -51,7 +51,7 @@ %% Internal exports (RPC) -export([ do_destroy/1, - do_add_user/2, + do_add_user/1, do_delete_user/2, do_update_user/3 ]). @@ -157,19 +157,15 @@ do_destroy(UserGroup) -> ). add_user(UserInfo, State) -> - trans(fun ?MODULE:do_add_user/2, [UserInfo, State]). + UserInfoRecord = user_info_record(UserInfo, State), + trans(fun ?MODULE:do_add_user/1, [UserInfoRecord]). do_add_user( - #{ - user_id := UserID, - password := Password - } = UserInfo, - #{user_group := UserGroup} = State + #user_info{user_id = {UserID, _} = DBUserID, is_superuser = IsSuperuser} = UserInfoRecord ) -> - case mnesia:read(?TAB, {UserGroup, UserID}, write) of + case mnesia:read(?TAB, DBUserID, write) of [] -> - IsSuperuser = maps:get(is_superuser, UserInfo, false), - add_user(UserGroup, UserID, Password, IsSuperuser, State), + mnesia:write(?TAB, UserInfoRecord, write), {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} @@ -187,36 +183,28 @@ do_delete_user(UserID, #{user_group := UserGroup}) -> end. update_user(UserID, User, State) -> - trans(fun ?MODULE:do_update_user/3, [UserID, User, State]). + FieldsToUpdate = fields_to_update( + User, + [ + keys_and_salt, + is_superuser + ], + State + ), + trans(fun ?MODULE:do_update_user/3, [UserID, FieldsToUpdate, State]). do_update_user( UserID, - User, - #{user_group := UserGroup} = State + FieldsToUpdate, + #{user_group := UserGroup} = _State ) -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [#user_info{is_superuser = IsSuperuser} = UserInfo] -> - UserInfo1 = UserInfo#user_info{ - is_superuser = maps:get(is_superuser, User, IsSuperuser) - }, - UserInfo2 = - case maps:get(password, User, undefined) of - undefined -> - UserInfo1; - Password -> - {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( - Password, State - ), - UserInfo1#user_info{ - stored_key = StoredKey, - server_key = ServerKey, - salt = Salt - } - end, - mnesia:write(?TAB, UserInfo2, write), - {ok, format_user_info(UserInfo2)} + [#user_info{} = UserInfo0] -> + UserInfo1 = update_user_record(UserInfo0, FieldsToUpdate), + mnesia:write(?TAB, UserInfo1, write), + {ok, format_user_info(UserInfo1)} end. lookup_user(UserID, #{user_group := UserGroup}) -> @@ -315,19 +303,56 @@ check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algori {error, not_authorized} end. -add_user(UserGroup, UserID, Password, IsSuperuser, State) -> - {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), - write_user(UserGroup, UserID, StoredKey, ServerKey, Salt, IsSuperuser). +user_info_record( + #{ + user_id := UserID, + password := Password + } = UserInfo, + #{user_group := UserGroup} = State +) -> + IsSuperuser = maps:get(is_superuser, UserInfo, false), + user_info_record(UserGroup, UserID, Password, IsSuperuser, State). -write_user(UserGroup, UserID, StoredKey, ServerKey, Salt, IsSuperuser) -> - UserInfo = #user_info{ +user_info_record(UserGroup, UserID, Password, IsSuperuser, State) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), + #user_info{ user_id = {UserGroup, UserID}, stored_key = StoredKey, server_key = ServerKey, salt = Salt, is_superuser = IsSuperuser - }, - mnesia:write(?TAB, UserInfo, write). + }. + +fields_to_update( + #{password := Password} = UserInfo, + [keys_and_salt | Rest], + State +) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), + [ + {keys_and_salt, {StoredKey, ServerKey, Salt}} + | fields_to_update(UserInfo, Rest, State) + ]; +fields_to_update(#{is_superuser := IsSuperuser} = UserInfo, [is_superuser | Rest], State) -> + [{is_superuser, IsSuperuser} | fields_to_update(UserInfo, Rest, State)]; +fields_to_update(UserInfo, [_ | Rest], State) -> + fields_to_update(UserInfo, Rest, State); +fields_to_update(_UserInfo, [], _State) -> + []. + +update_user_record(UserInfoRecord, []) -> + UserInfoRecord; +update_user_record(UserInfoRecord, [{keys_and_salt, {StoredKey, ServerKey, Salt}} | Rest]) -> + update_user_record( + UserInfoRecord#user_info{ + stored_key = StoredKey, + server_key = ServerKey, + salt = Salt + }, + Rest + ); +update_user_record(UserInfoRecord, [{is_superuser, IsSuperuser} | Rest]) -> + update_user_record(UserInfoRecord#user_info{is_superuser = IsSuperuser}, Rest). retrieve(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl index fa22693b3..ef4ec6e05 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + refs() -> [?R_REF(scram)]. diff --git a/apps/emqx_auth_mnesia/src/emqx_authz_api_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authz_api_mnesia.erl index e71b44add..5fc1ec280 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authz_api_mnesia.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). +-include("emqx_auth_mnesia.hrl"). -include_lib("emqx_auth/include/emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -55,6 +56,9 @@ format_result/1 ]). +%% minirest filter callback +-export([is_configured_authz_source/2]). + -define(BAD_REQUEST, 'BAD_REQUEST'). -define(NOT_FOUND, 'NOT_FOUND'). -define(ALREADY_EXISTS, 'ALREADY_EXISTS'). @@ -85,6 +89,7 @@ paths() -> schema("/authorization/sources/built_in_database/rules/users") -> #{ 'operationId' => users, + filter => fun ?MODULE:is_configured_authz_source/2, get => #{ tags => [<<"authorization">>], @@ -131,6 +136,7 @@ schema("/authorization/sources/built_in_database/rules/users") -> schema("/authorization/sources/built_in_database/rules/clients") -> #{ 'operationId' => clients, + filter => fun ?MODULE:is_configured_authz_source/2, get => #{ tags => [<<"authorization">>], @@ -177,6 +183,7 @@ schema("/authorization/sources/built_in_database/rules/clients") -> schema("/authorization/sources/built_in_database/rules/users/:username") -> #{ 'operationId' => user, + filter => fun ?MODULE:is_configured_authz_source/2, get => #{ tags => [<<"authorization">>], @@ -230,6 +237,7 @@ schema("/authorization/sources/built_in_database/rules/users/:username") -> schema("/authorization/sources/built_in_database/rules/clients/:clientid") -> #{ 'operationId' => client, + filter => fun ?MODULE:is_configured_authz_source/2, get => #{ tags => [<<"authorization">>], @@ -283,6 +291,7 @@ schema("/authorization/sources/built_in_database/rules/clients/:clientid") -> schema("/authorization/sources/built_in_database/rules/all") -> #{ 'operationId' => all, + filter => fun ?MODULE:is_configured_authz_source/2, get => #{ tags => [<<"authorization">>], @@ -317,6 +326,7 @@ schema("/authorization/sources/built_in_database/rules/all") -> schema("/authorization/sources/built_in_database/rules") -> #{ 'operationId' => rules, + filter => fun ?MODULE:is_configured_authz_source/2, delete => #{ tags => [<<"authorization">>], @@ -426,6 +436,14 @@ fields(rules) -> %% HTTP API %%-------------------------------------------------------------------- +is_configured_authz_source(Params, _Meta) -> + emqx_authz_api_sources:with_source( + ?AUTHZ_TYPE_BIN, + fun(_Source) -> + {ok, Params} + end + ). + users(get, #{query_string := QueryString}) -> case emqx_mgmt_api:node_query( @@ -440,7 +458,9 @@ users(get, #{query_string := QueryString}) -> {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; {error, Node, Error} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), + Message = list_to_binary( + io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error]) + ), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Result -> {200, Result} @@ -476,7 +496,9 @@ clients(get, #{query_string := QueryString}) -> {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; {error, Node, Error} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), + Message = list_to_binary( + io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error]) + ), {500, #{code => <<"NODE_DOWN">>, message => Message}}; Result -> {200, Result} diff --git a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl index 401d5a494..7e8e463b3 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia.erl @@ -95,7 +95,9 @@ create(Source) -> Source. update(Source) -> Source. -destroy(_Source) -> ok. +destroy(_Source) -> + {atomic, ok} = mria:clear_table(?ACL_TABLE), + ok. authorize( #{ diff --git a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl index cab544bf7..4d467397e 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authz_mnesia_schema.erl @@ -26,9 +26,12 @@ fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(builtin_db) -> diff --git a/apps/emqx_auth_mnesia/test/emqx_authn_scram_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_authn_scram_mnesia_SUITE.erl index abd5518a6..39350e4b9 100644 --- a/apps/emqx_auth_mnesia/test/emqx_authn_scram_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_authn_scram_mnesia_SUITE.erl @@ -314,6 +314,74 @@ t_update_user(_) -> {ok, #{is_superuser := true}} = emqx_authn_scram_mnesia:lookup_user(<<"u">>, State). +t_update_user_keys(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, <<"badpass">>, Algorithm), + + {ok, [#{state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + + emqx_authn_scram_mnesia:update_user( + Username, + #{password => Password}, + State + ), + + ok = emqx_config:put([mqtt, idle_timeout], 500), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => Algorithm + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage} + ) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => Algorithm} + ). + t_list_users(_) -> Config = config(), {ok, State} = emqx_authn_scram_mnesia:create(<<"id">>, Config), diff --git a/apps/emqx_auth_mnesia/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_authz_api_mnesia_SUITE.erl index e4b96b08b..efe4899f0 100644 --- a/apps/emqx_auth_mnesia/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_authz_api_mnesia_SUITE.erl @@ -331,4 +331,163 @@ t_api(_) -> [] ), ?assertEqual(0, emqx_authz_mnesia:record_count()), + + Examples = make_examples(emqx_authz_api_mnesia), + ?assertEqual( + 14, + length(Examples) + ), + + Fixtures1 = fun() -> + {ok, _, _} = + request( + delete, + uri(["authorization", "sources", "built_in_database", "rules", "all"]), + [] + ), + {ok, _, _} = + request( + delete, + uri(["authorization", "sources", "built_in_database", "rules", "users"]), + [] + ), + {ok, _, _} = + request( + delete, + uri(["authorization", "sources", "built_in_database", "rules", "clients"]), + [] + ) + end, + run_examples(Examples, Fixtures1), + + Fixtures2 = fun() -> + %% disable/remove built_in_database + {ok, 204, _} = + request( + delete, + uri(["authorization", "sources", "built_in_database"]), + [] + ) + end, + + run_examples(404, Examples, Fixtures2), + ok. + +%% test helpers +-define(REPLACEMENTS, #{ + ":clientid" => <<"client1">>, + ":username" => <<"user1">> +}). + +run_examples(Examples) -> + %% assume all ok + run_examples( + fun + ({ok, Code, _}) when + Code >= 200, + Code =< 299 + -> + true; + (_Res) -> + ct:pal("check failed: ~p", [_Res]), + false + end, + Examples + ). + +run_examples(Examples, Fixtures) when is_function(Fixtures) -> + Fixtures(), + run_examples(Examples); +run_examples(Check, Examples) when is_function(Check) -> + lists:foreach( + fun({Path, Op, Body} = _Req) -> + ct:pal("req: ~p", [_Req]), + ?assert( + Check( + request(Op, uri(Path), Body) + ) + ) + end, + Examples + ); +run_examples(Code, Examples) when is_number(Code) -> + run_examples( + fun + ({ok, ResCode, _}) when Code =:= ResCode -> true; + (_Res) -> + ct:pal("check failed: ~p", [_Res]), + false + end, + Examples + ). + +run_examples(CodeOrCheck, Examples, Fixtures) when is_function(Fixtures) -> + Fixtures(), + run_examples(CodeOrCheck, Examples). + +make_examples(ApiMod) -> + make_examples(ApiMod, ?REPLACEMENTS). + +-spec make_examples(Mod :: atom()) -> [{Path :: list(), [{Op :: atom(), Body :: term()}]}]. +make_examples(ApiMod, Replacements) -> + Paths = ApiMod:paths(), + lists:flatten( + lists:map( + fun(Path) -> + Schema = ApiMod:schema(Path), + lists:map( + fun({Op, OpSchema}) -> + Body = + case maps:get('requestBody', OpSchema, undefined) of + undefined -> + []; + HoconWithExamples -> + maps:get( + value, + hd( + maps:values( + maps:get( + <<"examples">>, + maps:get(examples, HoconWithExamples) + ) + ) + ) + ) + end, + {replace_parts(to_parts(Path), Replacements), Op, Body} + end, + lists:sort( + fun op_sort/2, maps:to_list(maps:with([get, put, post, delete], Schema)) + ) + ) + end, + Paths + ) + ). + +op_sort({post, _}, {_, _}) -> + true; +op_sort({put, _}, {_, _}) -> + true; +op_sort({get, _}, {delete, _}) -> + true; +op_sort(_, _) -> + false. + +to_parts(Path) -> + string:tokens(Path, "/"). + +replace_parts(Parts, Replacements) -> + lists:map( + fun(Part) -> + %% that's the fun part + case maps:is_key(Part, Replacements) of + true -> + maps:get(Part, Replacements); + false -> + Part + end + end, + Parts + ). diff --git a/apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl index 8f4f92ea2..7d77116e0 100644 --- a/apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl @@ -221,6 +221,35 @@ t_normalize_rules(_Config) -> ) ). +t_destroy(_Config) -> + ClientInfo = emqx_authz_test_lib:base_client_info(), + + ok = emqx_authz_mnesia:store_rules( + {username, <<"username">>}, + [#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}] + ), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ), + + ok = emqx_authz_test_lib:reset_authorizers(), + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ), + + ok = setup_config(), + + %% After destroy, the rules should be empty + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src index 38cf0138f..8970329fe 100644 --- a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src +++ b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mongodb, [ {description, "EMQX MongoDB Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_mongodb_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl index 8f76bedc2..b72a1e83a 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_mongodb_schema). --include("emqx_auth_mongodb.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_mongodb.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [ ?R_REF(mongo_single), diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 3b235ad2c..fdeb9d542 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -35,12 +35,12 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> @@ -49,11 +49,11 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), - FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> - FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), case emqx_authz_utils:update_resource(emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl index aff399e68..bdde704f9 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb_schema.erl @@ -16,17 +16,20 @@ -module(emqx_authz_mongodb_schema). --include("emqx_auth_mongodb.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -export([ type/0, fields/1, desc/1, source_refs/0, - select_union_member/1 + select_union_member/1, + namespace/0 ]). +-include("emqx_auth_mongodb.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. source_refs() -> diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 933e8f819..38750b79a 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mysql, [ {description, "EMQX MySQL Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_mysql_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl index 0189ecc61..6472794fe 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql_schema.erl @@ -16,18 +16,21 @@ -module(emqx_authn_mysql_schema). --include("emqx_auth_mysql.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_auth_mysql.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(mysql)]. diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 4ca71e332..8c9e54ee1 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -37,26 +37,26 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> "AuthZ with Mysql". create(#{query := SQL} = Source0) -> - {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), + {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> - {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), + {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, case emqx_authz_utils:update_resource(emqx_mysql, Source) of {error, Reason} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl index a9ce422e6..43f6ca6fa 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,6 +30,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(mysql) -> diff --git a/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src b/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src index 3157b7bd7..bae3da0cb 100644 --- a/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src +++ b/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_postgresql, [ {description, "EMQX PostgreSQL Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_postgresql_app, []}}, {applications, [ @@ -9,7 +9,7 @@ stdlib, emqx, emqx_auth, - emqx_connector + emqx_postgresql ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl index 1ce2e405c..ba92d2525 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl @@ -46,14 +46,14 @@ create(Config0) -> {Config, State} = parse_config(Config0, ResourceId), {ok, _Data} = emqx_authn_utils:create_resource( ResourceId, - emqx_connector_pgsql, + emqx_postgresql, Config ), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0, ResourceId), - case emqx_authn_utils:update_resource(emqx_connector_pgsql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_postgresql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl index 93819d7bf..ef7d00df3 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + select_union_member( #{ <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN @@ -50,7 +53,7 @@ fields(postgresql) -> {query, fun query/1} ] ++ emqx_authn_schema:common_fields() ++ - proplists:delete(prepare_statement, emqx_connector_pgsql:fields(config)). + proplists:delete(prepare_statement, emqx_postgresql:fields(config)). desc(postgresql) -> ?DESC(postgresql); diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index 27f2d31ee..14b7598a6 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -37,32 +37,32 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> "AuthZ with PostgreSQL". create(#{query := SQL0} = Source) -> - {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), - ResourceID = emqx_authz_utils:make_resource_id(emqx_connector_pgsql), + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), + ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( ResourceID, - emqx_connector_pgsql, + emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ), Source#{annotations => #{id => ResourceID, placeholders => PlaceHolders}}. update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> - {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( - emqx_connector_pgsql, + emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ) of diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl index a52cc4fdd..296b00126 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,11 +30,13 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(postgresql) -> emqx_authz_schema:authz_common_fields(?AUTHZ_TYPE) ++ - emqx_connector_pgsql:fields(config) ++ + emqx_postgresql:fields(config) ++ [{query, query()}]. desc(postgresql) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index 752202610..af1f1db2d 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -19,7 +19,7 @@ -compile(nowarn_export_all). -compile(export_all). --include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("emqx_postgresql/include/emqx_postgresql.hrl"). -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -64,7 +64,7 @@ init_per_suite(Config) -> {ok, _} = emqx_resource:create_local( ?PGSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, - emqx_connector_pgsql, + emqx_postgresql, pgsql_config(), #{} ), @@ -104,7 +104,7 @@ t_update_with_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "postgresql", + matched_type := "authn:postgresql", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl index 25a65f660..ba6cf0604 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl @@ -19,7 +19,7 @@ -compile(nowarn_export_all). -compile(export_all). --include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("emqx_postgresql/include/emqx_postgresql.hrl"). -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl index 4d38e9c96..f486cbd3d 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl @@ -18,7 +18,7 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). +-include_lib("emqx_postgresql/include/emqx_postgresql.hrl"). -include_lib("emqx_auth/include/emqx_authz.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -438,7 +438,7 @@ create_pgsql_resource() -> emqx_resource:create_local( ?PGSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, - emqx_connector_pgsql, + emqx_postgresql, pgsql_config(), #{} ). diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 388fd413c..bd33606d3 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_redis, [ {description, "EMQX Redis Authentication and Authorization"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_auth_redis_app, []}}, {applications, [ diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_validations.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_validations.erl new file mode 100644 index 000000000..e94b67c40 --- /dev/null +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_validations.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_redis_validations). + +-export([ + validate_command/2 +]). + +validate_command([], _Command) -> + ok; +validate_command([Validation | Rest], Command) -> + case validate(Validation, Command) of + ok -> + validate_command(Rest, Command); + {error, _} = Error -> + Error + end. + +validate(not_empty, []) -> + {error, empty_command}; +validate(not_empty, _) -> + ok; +validate({command_name, AllowedNames}, [Name | _]) -> + IsAllowed = lists:any( + fun(AllowedName) -> + string:equal(AllowedName, Name, true, none) + end, + AllowedNames + ), + case IsAllowed of + true -> + ok; + false -> + {error, {invalid_command_name, Name}} + end; +validate({command_name, _}, _) -> + {error, invalid_command_name}; +validate({allowed_fields, AllowedFields}, [_CmdName, _CmdKey | Args]) -> + Unknown = lists:filter(fun(Arg) -> not lists:member(Arg, AllowedFields) end, Args), + case Unknown of + [] -> + ok; + _ -> + {error, {unknown_fields, Unknown}} + end; +validate({allowed_fields, _}, _) -> + ok; +validate({required_field_one_of, Required}, [_CmdName, _CmdKey | Args]) -> + HasRequired = lists:any(fun(Field) -> lists:member(Field, Args) end, Required), + case HasRequired of + true -> + ok; + false -> + {error, {missing_required_field, Required}} + end; +validate({required_field_one_of, Required}, _) -> + {error, {missing_required_field, Required}}. diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 2f0948faf..b7324e251 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -18,100 +18,16 @@ -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("hocon/include/hoconsc.hrl"). --behaviour(hocon_schema). -behaviour(emqx_authn_provider). -export([ - namespace/0, - tags/0, - roots/0, - fields/1, - desc/1 -]). - --export([ - refs/0, - union_member_selector/1, create/2, update/2, authenticate/2, destroy/1 ]). -%%------------------------------------------------------------------------------ -%% Hocon Schema -%%------------------------------------------------------------------------------ - -namespace() -> "authn". - -tags() -> - [<<"Authentication">>]. - -%% used for config check when the schema module is resolved -roots() -> - [ - {?CONF_NS, - hoconsc:mk( - hoconsc:union(fun ?MODULE:union_member_selector/1), - #{} - )} - ]. - -fields(redis_single) -> - common_fields() ++ emqx_redis:fields(single); -fields(redis_cluster) -> - common_fields() ++ emqx_redis:fields(cluster); -fields(redis_sentinel) -> - common_fields() ++ emqx_redis:fields(sentinel). - -desc(redis_single) -> - ?DESC(single); -desc(redis_cluster) -> - ?DESC(cluster); -desc(redis_sentinel) -> - ?DESC(sentinel); -desc(_) -> - "". - -common_fields() -> - [ - {mechanism, emqx_authn_schema:mechanism(password_based)}, - {backend, emqx_authn_schema:backend(redis)}, - {cmd, fun cmd/1}, - {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} - ] ++ emqx_authn_schema:common_fields(). - -cmd(type) -> string(); -cmd(desc) -> ?DESC(?FUNCTION_NAME); -cmd(required) -> true; -cmd(_) -> undefined. - -refs() -> - [ - hoconsc:ref(?MODULE, redis_single), - hoconsc:ref(?MODULE, redis_cluster), - hoconsc:ref(?MODULE, redis_sentinel) - ]. - -union_member_selector(all_union_members) -> - refs(); -union_member_selector({value, Value}) -> - refs(Value). - -refs(#{<<"redis_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, redis_single)]; -refs(#{<<"redis_type">> := <<"cluster">>}) -> - [hoconsc:ref(?MODULE, redis_cluster)]; -refs(#{<<"redis_type">> := <<"sentinel">>}) -> - [hoconsc:ref(?MODULE, redis_sentinel)]; -refs(_) -> - throw(#{ - field_name => redis_type, - expected => "single | cluster | sentinel" - }). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -202,54 +118,51 @@ authenticate( parse_config( #{ - cmd := Cmd, + cmd := CmdStr, password_hash_algorithm := Algorithm } = Config ) -> - try - NCmd = parse_cmd(Cmd), - ok = emqx_authn_password_hashing:init(Algorithm), - ok = emqx_authn_utils:ensure_apps_started(Algorithm), - State = maps:with([password_hash_algorithm, salt_position], Config), - {Config, State#{cmd => NCmd}} - catch - error:{unsupported_cmd, _Cmd} -> - {error, {unsupported_cmd, Cmd}}; - error:missing_password_hash -> - {error, missing_password_hash}; - error:{unsupported_fields, Fields} -> - {error, {unsupported_fields, Fields}} + case parse_cmd(CmdStr) of + {ok, Cmd} -> + ok = emqx_authn_password_hashing:init(Algorithm), + ok = emqx_authn_utils:ensure_apps_started(Algorithm), + State = maps:with([password_hash_algorithm, salt_position], Config), + {Config, State#{cmd => Cmd}}; + {error, _} = Error -> + Error end. -%% Only support HGET and HMGET -parse_cmd(Cmd) -> - case string:tokens(Cmd, " ") of - [Command, Key, Field | Fields] when Command =:= "HGET" orelse Command =:= "HMGET" -> - NFields = [Field | Fields], - check_fields(NFields), - KeyTemplate = emqx_authn_utils:parse_str(list_to_binary(Key)), - {Command, KeyTemplate, NFields}; - _ -> - error({unsupported_cmd, Cmd}) +parse_cmd(CmdStr) -> + case emqx_redis_command:split(CmdStr) of + {ok, Cmd} -> + case validate_cmd(Cmd) of + ok -> + [CommandName, Key | Fields] = Cmd, + {ok, {CommandName, emqx_authn_utils:parse_str(Key), Fields}}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error end. -check_fields(Fields) -> - HasPassHash = lists:member("password_hash", Fields) orelse lists:member("password", Fields), - KnownFields = ["password_hash", "password", "salt", "is_superuser"], - UnknownFields = [F || F <- Fields, not lists:member(F, KnownFields)], - - case {HasPassHash, UnknownFields} of - {true, []} -> ok; - {true, _} -> error({unsupported_fields, UnknownFields}); - {false, _} -> error(missing_password_hash) - end. +validate_cmd(Cmd) -> + emqx_auth_redis_validations:validate_command( + [ + not_empty, + {command_name, [<<"hget">>, <<"hmget">>]}, + {allowed_fields, [<<"password_hash">>, <<"password">>, <<"salt">>, <<"is_superuser">>]}, + {required_field_one_of, [<<"password_hash">>, <<"password">>]} + ], + Cmd + ). merge(Fields, Value) when not is_list(Value) -> merge(Fields, [Value]); merge(Fields, Values) -> maps:from_list( [ - {list_to_binary(K), V} + {K, V} || {K, V} <- lists:zip(Fields, Values), V =/= undefined ] ). diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl b/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl index 4f1b63633..f3e124ca1 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis_schema.erl @@ -22,12 +22,15 @@ -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +namespace() -> "authn". + refs() -> [ ?R_REF(redis_single), @@ -85,7 +88,7 @@ common_fields() -> {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} ] ++ emqx_authn_schema:common_fields(). -cmd(type) -> string(); +cmd(type) -> binary(); cmd(desc) -> ?DESC(?FUNCTION_NAME); cmd(required) -> true; cmd(_) -> undefined. diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index be83223e4..ca4a11742 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -35,27 +35,25 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT, - ?PH_PEERHOST, - ?PH_CLIENTID, - ?PH_USERNAME +-define(ALLOWED_VARS, [ + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT, + ?VAR_PEERHOST, + ?VAR_CLIENTID, + ?VAR_USERNAME ]). description() -> "AuthZ with Redis". create(#{cmd := CmdStr} = Source) -> - Cmd = tokens(CmdStr), + CmdTemplate = parse_cmd(CmdStr), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS), {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source), Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. update(#{cmd := CmdStr} = Source) -> - Cmd = tokens(CmdStr), - CmdTemplate = emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS), + CmdTemplate = parse_cmd(CmdStr), case emqx_authz_utils:update_resource(emqx_redis, Source) of {error, Reason} -> error({load_config_error, Reason}); @@ -131,9 +129,28 @@ compile_rule(RuleBin, TopicFilterRaw) -> error(Reason) end. -tokens(Query) -> - Tokens = binary:split(Query, <<" ">>, [global]), - [Token || Token <- Tokens, size(Token) > 0]. +parse_cmd(Query) -> + case emqx_redis_command:split(Query) of + {ok, Cmd} -> + ok = validate_cmd(Cmd), + emqx_authz_utils:parse_deep(Cmd, ?ALLOWED_VARS); + {error, Reason} -> + error({invalid_redis_cmd, Reason, Query}) + end. + +validate_cmd(Cmd) -> + case + emqx_auth_redis_validations:validate_command( + [ + not_empty, + {command_name, [<<"hmget">>, <<"hgetall">>]} + ], + Cmd + ) + of + ok -> ok; + {error, Reason} -> error({invalid_redis_cmd, Reason, Cmd}) + end. parse_rule(<<"publish">>) -> #{<<"action">> => <<"publish">>}; diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl b/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl index 755192bfc..5cd084795 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis_schema.erl @@ -22,6 +22,7 @@ -behaviour(emqx_authz_schema). -export([ + namespace/0, type/0, fields/1, desc/1, @@ -29,6 +30,8 @@ select_union_member/1 ]). +namespace() -> "authz". + type() -> ?AUTHZ_TYPE. fields(redis_single) -> diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index b3f4a15a3..e7673b790 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -170,7 +170,7 @@ test_create_invalid_config(InvalidAuthConfig, Path) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "redis_single", + matched_type := "authn:redis_single", path := Path }}, emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, InvalidAuthConfig}) @@ -336,7 +336,22 @@ user_seeds() -> config_params => #{}, result => {ok, #{is_superuser => true}} }, - + #{ + data => #{ + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> + }, + credentials => #{ + username => <<"plain">>, + password => <<"plain">> + }, + key => <<"mqtt_user:plain">>, + config_params => #{ + <<"cmd">> => <<"HmGeT mqtt_user:${username} password_hash salt is_superuser">> + }, + result => {ok, #{is_superuser => true}} + }, #{ data => #{ password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>, diff --git a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl index 962333cd2..1c52cee17 100644 --- a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl @@ -112,7 +112,9 @@ t_create_invalid_config(_Config) -> ). t_redis_error(_Config) -> - ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}), + q([<<"SET">>, <<"notahash">>, <<"stringvalue">>]), + + ok = setup_config(#{<<"cmd">> => <<"HGETALL notahash">>}), ClientInfo = emqx_authz_test_lib:base_client_info(), @@ -121,6 +123,24 @@ t_redis_error(_Config) -> emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"a">>) ). +t_invalid_command(_Config) -> + Config = raw_redis_authz_config(), + + ?assertMatch( + {error, _}, + emqx_authz:update(?CMD_REPLACE, [Config#{<<"cmd">> => <<"HGET key">>}]) + ), + + ?assertMatch( + {ok, _}, + emqx_authz:update(?CMD_REPLACE, [Config#{<<"cmd">> => <<"HGETALL key">>}]) + ), + + ?assertMatch( + {error, _}, + emqx_authz:update({?CMD_REPLACE, redis}, Config#{<<"cmd">> => <<"HGET key">>}) + ). + %%------------------------------------------------------------------------------ %% Cases %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 9456575d4..3e0875e1f 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -115,9 +115,9 @@ resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector; resource_type(redis_single) -> emqx_bridge_redis_connector; resource_type(redis_sentinel) -> emqx_bridge_redis_connector; resource_type(redis_cluster) -> emqx_bridge_redis_connector; -resource_type(pgsql) -> emqx_connector_pgsql; -resource_type(timescale) -> emqx_connector_pgsql; -resource_type(matrix) -> emqx_connector_pgsql; +resource_type(pgsql) -> emqx_postgresql; +resource_type(timescale) -> emqx_postgresql; +resource_type(matrix) -> emqx_postgresql; resource_type(tdengine) -> emqx_bridge_tdengine_connector; resource_type(clickhouse) -> emqx_bridge_clickhouse_connector; resource_type(dynamo) -> emqx_bridge_dynamo_connector; diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index d0821ea83..d4c16e13c 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 685fd3397..bb4a13875 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -101,7 +101,7 @@ fields(connector_config) -> )}, {service_account_json, sc( - service_account_json(), + ?MODULE:service_account_json(), #{ required => true, validator => fun ?MODULE:service_account_json_validator/1, diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index 4c5a15b79..a8a938a0b 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_greptimedb, [ {description, "EMQX GreptimeDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index d63103e2e..f5ae714d7 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -126,7 +126,7 @@ desc(_) -> undefined. write_syntax(type) -> - emqx_bridge_influxdb:write_syntax(); + emqx_bridge_influxdb:write_syntax_type(); write_syntax(required) -> true; write_syntax(validator) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index e5c559bd5..87d7e57a6 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_http, [ {description, "EMQX HTTP Bridge and Connector Application"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]}, {env, []}, diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 5d1b1947c..743ab97fe 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -46,14 +46,6 @@ -export([validate_method/1, join_paths/2]). --type connect_timeout() :: emqx_schema:duration() | infinity. --type pool_type() :: random | hash. - --reflect_type([ - connect_timeout/0, - pool_type/0 -]). - -define(DEFAULT_PIPELINE_SIZE, 100). -define(DEFAULT_REQUEST_TIMEOUT_MS, 30_000). @@ -89,7 +81,7 @@ fields(config) -> )}, {pool_type, sc( - pool_type(), + hoconsc:enum([random, hash]), #{ default => random, desc => ?DESC("pool_type") @@ -479,61 +471,47 @@ preprocess_request( } = Req ) -> #{ - method => emqx_placeholder:preproc_tmpl(to_bin(Method)), - path => emqx_placeholder:preproc_tmpl(Path), - body => maybe_preproc_tmpl(body, Req), - headers => wrap_auth_header(preproc_headers(Headers)), + method => parse_template(to_bin(Method)), + path => parse_template(Path), + body => maybe_parse_template(body, Req), + headers => parse_headers(Headers), request_timeout => maps:get(request_timeout, Req, ?DEFAULT_REQUEST_TIMEOUT_MS), max_retries => maps:get(max_retries, Req, 2) }. -preproc_headers(Headers) when is_map(Headers) -> +parse_headers(Headers) when is_map(Headers) -> maps:fold( - fun(K, V, Acc) -> - [ - { - emqx_placeholder:preproc_tmpl(to_bin(K)), - emqx_placeholder:preproc_tmpl(to_bin(V)) - } - | Acc - ] - end, + fun(K, V, Acc) -> [parse_header(K, V) | Acc] end, [], Headers ); -preproc_headers(Headers) when is_list(Headers) -> +parse_headers(Headers) when is_list(Headers) -> lists:map( - fun({K, V}) -> - { - emqx_placeholder:preproc_tmpl(to_bin(K)), - emqx_placeholder:preproc_tmpl(to_bin(V)) - } - end, + fun({K, V}) -> parse_header(K, V) end, Headers ). -wrap_auth_header(Headers) -> - lists:map(fun maybe_wrap_auth_header/1, Headers). +parse_header(K, V) -> + KStr = to_bin(K), + VTpl = parse_template(to_bin(V)), + {parse_template(KStr), maybe_wrap_auth_header(KStr, VTpl)}. -maybe_wrap_auth_header({[{str, Key}] = StrKey, Val}) -> - {_, MaybeWrapped} = maybe_wrap_auth_header({Key, Val}), - {StrKey, MaybeWrapped}; -maybe_wrap_auth_header({Key, Val} = Header) when - is_binary(Key), (size(Key) =:= 19 orelse size(Key) =:= 13) +maybe_wrap_auth_header(Key, VTpl) when + (byte_size(Key) =:= 19 orelse byte_size(Key) =:= 13) -> %% We check the size of potential keys in the guard above and consider only %% those that match the number of characters of either "Authorization" or %% "Proxy-Authorization". case try_bin_to_lower(Key) of <<"authorization">> -> - {Key, emqx_secret:wrap(Val)}; + emqx_secret:wrap(VTpl); <<"proxy-authorization">> -> - {Key, emqx_secret:wrap(Val)}; + emqx_secret:wrap(VTpl); _Other -> - Header + VTpl end; -maybe_wrap_auth_header(Header) -> - Header. +maybe_wrap_auth_header(_Key, VTpl) -> + VTpl. try_bin_to_lower(Bin) -> try iolist_to_binary(string:lowercase(Bin)) of @@ -542,46 +520,57 @@ try_bin_to_lower(Bin) -> _:_ -> Bin end. -maybe_preproc_tmpl(Key, Conf) -> +maybe_parse_template(Key, Conf) -> case maps:get(Key, Conf, undefined) of undefined -> undefined; - Val -> emqx_placeholder:preproc_tmpl(Val) + Val -> parse_template(Val) end. +parse_template(String) -> + emqx_template:parse(String). + process_request( #{ - method := MethodTks, - path := PathTks, - body := BodyTks, - headers := HeadersTks, + method := MethodTemplate, + path := PathTemplate, + body := BodyTemplate, + headers := HeadersTemplate, request_timeout := ReqTimeout } = Conf, Msg ) -> Conf#{ - method => make_method(emqx_placeholder:proc_tmpl(MethodTks, Msg)), - path => emqx_placeholder:proc_tmpl(PathTks, Msg), - body => process_request_body(BodyTks, Msg), - headers => proc_headers(HeadersTks, Msg), + method => make_method(render_template_string(MethodTemplate, Msg)), + path => unicode:characters_to_list(render_template(PathTemplate, Msg)), + body => render_request_body(BodyTemplate, Msg), + headers => render_headers(HeadersTemplate, Msg), request_timeout => ReqTimeout }. -process_request_body(undefined, Msg) -> +render_request_body(undefined, Msg) -> emqx_utils_json:encode(Msg); -process_request_body(BodyTks, Msg) -> - emqx_placeholder:proc_tmpl(BodyTks, Msg). +render_request_body(BodyTks, Msg) -> + render_template(BodyTks, Msg). -proc_headers(HeaderTks, Msg) -> +render_headers(HeaderTks, Msg) -> lists:map( fun({K, V}) -> { - emqx_placeholder:proc_tmpl(K, Msg), - emqx_placeholder:proc_tmpl(emqx_secret:unwrap(V), Msg) + render_template_string(K, Msg), + render_template_string(emqx_secret:unwrap(V), Msg) } end, HeaderTks ). +render_template(Template, Msg) -> + % NOTE: ignoring errors here, missing variables will be rendered as `"undefined"`. + {String, _Errors} = emqx_template:render(Template, {emqx_jsonish, Msg}), + String. + +render_template_string(Template, Msg) -> + unicode:characters_to_binary(render_template(Template, Msg)). + make_method(M) when M == <<"POST">>; M == <<"post">> -> post; make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"GET">>; M == <<"get">> -> get; @@ -716,8 +705,6 @@ maybe_retry(Result, _Context, ReplyFunAndArgs) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). %% The HOCON schema system may generate sensitive keys with this format -is_sensitive_key([{str, StringKey}]) -> - is_sensitive_key(StringKey); is_sensitive_key(Atom) when is_atom(Atom) -> is_sensitive_key(erlang:atom_to_binary(Atom)); is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) -> @@ -742,25 +729,19 @@ redact(Data) -> %% and we also can't know the body format and where the sensitive data will be %% so the easy way to keep data security is redacted the whole body redact_request({Path, Headers}) -> - {Path, redact(Headers)}; + {Path, Headers}; redact_request({Path, Headers, _Body}) -> - {Path, redact(Headers), <<"******">>}. + {Path, Headers, <<"******">>}. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). redact_test_() -> - TestData1 = [ - {<<"content-type">>, <<"application/json">>}, - {<<"Authorization">>, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>} - ], - - TestData2 = #{ - headers => - [ - {[{str, <<"content-type">>}], [{str, <<"application/json">>}]}, - {[{str, <<"Authorization">>}], [{str, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>}]} - ] + TestData = #{ + headers => [ + {<<"content-type">>, <<"application/json">>}, + {<<"Authorization">>, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>} + ] }, [ ?_assert(is_sensitive_key(<<"Authorization">>)), @@ -770,8 +751,7 @@ redact_test_() -> ?_assert(is_sensitive_key('PrOxy-authoRizaTion')), ?_assertNot(is_sensitive_key(<<"Something">>)), ?_assertNot(is_sensitive_key(89)), - ?_assertNotEqual(TestData1, redact(TestData1)), - ?_assertNotEqual(TestData2, redact(TestData2)) + ?_assertNotEqual(TestData, redact(TestData)) ]. join_paths_test_() -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl index 6b5c2b0cd..4f5e2929c 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl @@ -83,7 +83,8 @@ is_wrapped(Secret) when is_function(Secret) -> is_wrapped(_Other) -> false. -untmpl([{_, V} | _]) -> V. +untmpl(Tpl) -> + iolist_to_binary(emqx_template:render_strict(Tpl, #{})). is_unwrapped_headers(Headers) -> lists:all(fun is_unwrapped_header/1, Headers). diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 27fe1659c..c6236d97c 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index 47eeecb4e..acb295752 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -11,7 +11,8 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -export([ - conn_bridge_examples/1 + conn_bridge_examples/1, + write_syntax_type/0 ]). -export([ @@ -29,6 +30,9 @@ %% ------------------------------------------------------------------------------------------------- %% api +write_syntax_type() -> + typerefl:alias("string", write_syntax()). + conn_bridge_examples(Method) -> [ #{ @@ -154,7 +158,7 @@ desc(_) -> undefined. write_syntax(type) -> - ?MODULE:write_syntax(); + write_syntax_type(); write_syntax(required) -> true; write_syntax(validator) -> diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index af64ddf37..943f30629 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1724,6 +1724,7 @@ t_dynamic_mqtt_topic(Config) -> begin ?assertMatch({ok, _}, create_bridge_wait_for_balance(Config)), wait_until_subscribers_are_ready(NPartitions, 40_000), + ping_until_healthy(Config, _Period = 1_500, _Timeout = 24_000), {ok, C} = emqtt:start_link(), on_exit(fun() -> emqtt:stop(C) end), {ok, _} = emqtt:connect(C), diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 3ed40e903..a34b65ede 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -566,7 +566,6 @@ t_simple_sql_query(Config) -> t_missing_data(Config) -> BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) @@ -577,8 +576,8 @@ t_missing_data(Config) -> ), send_message(Config, #{}), {ok, [Event]} = snabbkaffe:receive_events(SRef), - case IsBatch of - true -> + case BatchSize of + N when N > 1 -> ?assertMatch( #{ result := @@ -588,7 +587,7 @@ t_missing_data(Config) -> }, Event ); - false -> + 1 -> ?assertMatch( #{ result := diff --git a/apps/emqx_bridge_pgsql/rebar.config b/apps/emqx_bridge_pgsql/rebar.config index 87c145f26..da2729b70 100644 --- a/apps/emqx_bridge_pgsql/rebar.config +++ b/apps/emqx_bridge_pgsql/rebar.config @@ -3,5 +3,6 @@ {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, - {emqx_bridge, {path, "../../apps/emqx_bridge"}} + {emqx_bridge, {path, "../../apps/emqx_bridge"}}, + {emqx_postgresql, {path, "../../apps/emqx_postgresql"}} ]}. diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index 85131baf0..7a17652e0 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,11 +1,12 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, stdlib, - emqx_resource + emqx_resource, + emqx_postgresql ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl index 12161b9b9..bb15dfad9 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl @@ -82,7 +82,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_connector_pgsql:fields(config) -- + (emqx_postgresql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", pgsql); diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index cd79db43d..156d4bd16 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -324,6 +324,7 @@ connect_and_drop_table(Config) -> connect_and_clear_table(Config) -> Con = connect_direct_pgsql(Config), + _ = epgsql:squery(Con, ?SQL_CREATE_TABLE), {ok, _} = epgsql:squery(Con, ?SQL_DELETE), ok = epgsql:close(Con). @@ -668,7 +669,7 @@ t_missing_table(Config) -> ok end, fun(Trace) -> - ?assertMatch([_, _, _], ?of_kind(pgsql_undefined_table, Trace)), + ?assertMatch([_], ?of_kind(pgsql_undefined_table, Trace)), ok end ), diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl index 7169ea3d2..29299dcc9 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl @@ -11,7 +11,7 @@ %%=========================================================================== pulsar_producer_validations_test() -> - Name = my_producer, + Name = list_to_atom("my_producer"), Conf0 = pulsar_producer_hocon(), Conf1 = Conf0 ++ diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index c7d931c93..7e32b5a89 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index ff439b676..2af1c16c8 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -20,7 +20,7 @@ -behaviour(ecpool_worker). %% hocon_schema callbacks --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). %% HTTP API callbacks -export([values/1]). @@ -43,6 +43,8 @@ %% Internal callbacks -export([publish_messages/3]). +namespace() -> "rabbitmq". + roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index 2b4d48173..83737e746 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -44,13 +44,12 @@ emqx_authz_redis_schema, emqx_authz_mysql_schema, emqx_authz_postgresql_schema, - emqx_authz_mongodb_schema -]). - --define(EE_AUTHZ_SOURCE_SCHEMA_MODS, [ + emqx_authz_mongodb_schema, emqx_authz_ldap_schema ]). +-define(EE_AUTHZ_SOURCE_SCHEMA_MODS, []). + -define(CE_AUTHN_PROVIDER_SCHEMA_MODS, [ emqx_authn_mnesia_schema, emqx_authn_mysql_schema, @@ -59,12 +58,12 @@ emqx_authn_redis_schema, emqx_authn_http_schema, emqx_authn_jwt_schema, - emqx_authn_scram_mnesia_schema + emqx_authn_scram_mnesia_schema, + emqx_authn_ldap_schema, + emqx_authn_ldap_bind_schema ]). -define(EE_AUTHN_PROVIDER_SCHEMA_MODS, [ - emqx_authn_ldap_schema, - emqx_authn_ldap_bind_schema, emqx_gcp_device_authn_schema ]). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 4e00c1c57..c986a65ee 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -212,14 +212,32 @@ gen_config_md(Dir, SchemaModule, Lang) -> -spec schema_module() -> module(). schema_module() -> case os:getenv("SCHEMA_MOD") of - false -> emqx_conf_schema; - Value -> list_to_existing_atom(Value) + false -> + resolve_schema_module(); + Value -> + list_to_existing_atom(Value) end. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +-ifdef(TEST). +resolve_schema_module() -> + case os:getenv("PROFILE") of + "emqx" -> + emqx_conf_schema; + "emqx-enterprise" -> + emqx_enterprise_schema; + false -> + error("PROFILE environment variable is not set") + end. +-else. +-spec resolve_schema_module() -> no_return(). +resolve_schema_module() -> + error("SCHEMA_MOD environment variable is not set"). +-endif. + %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. make_desc_resolver(Lang) -> fun @@ -277,7 +295,7 @@ hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> }, SubRefs }; -hocon_schema_to_spec(?UNION(Types), LocalModule) -> +hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) -> {OneOf, Refs} = lists:foldl( fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), @@ -290,149 +308,8 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> {#{type => enum, symbols => [Atom]}, []}. -typename_to_spec("user_id_type()", _Mod) -> - #{type => enum, symbols => [clientid, username]}; -typename_to_spec("term()", _Mod) -> - #{type => string}; -typename_to_spec("boolean()", _Mod) -> - #{type => boolean}; -typename_to_spec("binary()", _Mod) -> - #{type => string}; -typename_to_spec("float()", _Mod) -> - #{type => number}; -typename_to_spec("integer()", _Mod) -> - #{type => number}; -typename_to_spec("non_neg_integer()", _Mod) -> - #{type => number, minimum => 0}; -typename_to_spec("number()", _Mod) -> - #{type => number}; -typename_to_spec("string()", _Mod) -> - #{type => string}; -typename_to_spec("atom()", _Mod) -> - #{type => string}; -typename_to_spec("duration()", _Mod) -> - #{type => duration}; -typename_to_spec("timeout_duration()", _Mod) -> - #{type => duration}; -typename_to_spec("duration_s()", _Mod) -> - #{type => duration}; -typename_to_spec("timeout_duration_s()", _Mod) -> - #{type => duration}; -typename_to_spec("duration_ms()", _Mod) -> - #{type => duration}; -typename_to_spec("timeout_duration_ms()", _Mod) -> - #{type => duration}; -typename_to_spec("percent()", _Mod) -> - #{type => percent}; -typename_to_spec("file()", _Mod) -> - #{type => string}; -typename_to_spec("ip_port()", _Mod) -> - #{type => ip_port}; -typename_to_spec("url()", _Mod) -> - #{type => url}; -typename_to_spec("bytesize()", _Mod) -> - #{type => 'byteSize'}; -typename_to_spec("wordsize()", _Mod) -> - #{type => 'byteSize'}; -typename_to_spec("qos()", _Mod) -> - #{type => enum, symbols => [0, 1, 2]}; -typename_to_spec("comma_separated_list()", _Mod) -> - #{type => comma_separated_string}; -typename_to_spec("comma_separated_atoms()", _Mod) -> - #{type => comma_separated_string}; -typename_to_spec("pool_type()", _Mod) -> - #{type => enum, symbols => [random, hash]}; -typename_to_spec("log_level()", _Mod) -> - #{ - type => enum, - symbols => [ - debug, - info, - notice, - warning, - error, - critical, - alert, - emergency, - all - ] - }; -typename_to_spec("rate()", _Mod) -> - #{type => string}; -typename_to_spec("capacity()", _Mod) -> - #{type => string}; -typename_to_spec("burst_rate()", _Mod) -> - #{type => string}; -typename_to_spec("failure_strategy()", _Mod) -> - #{type => enum, symbols => [force, drop, throw]}; -typename_to_spec("initial()", _Mod) -> - #{type => string}; -typename_to_spec("map()", _Mod) -> - #{type => object}; -typename_to_spec("#{" ++ _, Mod) -> - typename_to_spec("map()", Mod); -typename_to_spec(Name, Mod) -> - Spec = range(Name), - Spec1 = remote_module_type(Spec, Name, Mod), - Spec2 = typerefl_array(Spec1, Name, Mod), - Spec3 = integer(Spec2, Name), - default_type(Spec3). - -default_type(nomatch) -> #{type => string}; -default_type(Type) -> Type. - -range(Name) -> - case string:split(Name, "..") of - %% 1..10 1..inf -inf..10 - [MinStr, MaxStr] -> - Schema = #{type => number}, - Schema1 = add_integer_prop(Schema, minimum, MinStr), - add_integer_prop(Schema1, maximum, MaxStr); - _ -> - nomatch - end. - -%% Module:Type -remote_module_type(nomatch, Name, Mod) -> - case string:split(Name, ":") of - [_Module, Type] -> typename_to_spec(Type, Mod); - _ -> nomatch - end; -remote_module_type(Spec, _Name, _Mod) -> - Spec. - -%% [string()] or [integer()] or [xxx]. -typerefl_array(nomatch, Name, Mod) -> - case string:trim(Name, leading, "[") of - Name -> - nomatch; - Name1 -> - case string:trim(Name1, trailing, "]") of - Name1 -> - notmatch; - Name2 -> - Schema = typename_to_spec(Name2, Mod), - #{type => array, items => Schema} - end - end; -typerefl_array(Spec, _Name, _Mod) -> - Spec. - -%% integer(1) -integer(nomatch, Name) -> - case string:to_integer(Name) of - {Int, []} -> #{type => enum, symbols => [Int], default => Int}; - _ -> nomatch - end; -integer(Spec, _Name) -> - Spec. - -add_integer_prop(Schema, Key, Value) -> - case string:to_integer(Value) of - {error, no_integer} -> Schema; - {Int, []} when Key =:= minimum -> Schema#{Key => Int}; - {Int, []} -> Schema#{Key => Int} - end. +typename_to_spec(TypeStr, Module) -> + emqx_conf_schema_types:readable_dashboard(Module, TypeStr). to_bin(List) when is_list(List) -> case io_lib:printable_list(List) of diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ac13ece61..3a2b5d972 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -28,21 +28,14 @@ -include("emqx_conf.hrl"). --type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. --type file() :: string(). --type cipher() :: map(). - -behaviour(hocon_schema). --reflect_type([ - log_level/0, - file/0, - cipher/0 -]). - -export([ namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1, tags/0 ]). + +-export([log_level/0]). + -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([upgrade_raw_conf/1]). @@ -548,7 +541,7 @@ fields("node") -> )}, {"crash_dump_file", sc( - file(), + string(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), @@ -659,7 +652,7 @@ fields("node") -> hoconsc:enum([gen_rpc, rpc]), #{ mapping => "mria.rlog_rpc_module", - default => gen_rpc, + default => rpc, 'readOnly' => true, importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_rpc_module) @@ -839,7 +832,7 @@ fields("rpc") -> )}, {"certfile", sc( - file(), + string(), #{ mapping => "gen_rpc.certfile", converter => fun ensure_unicode_path/2, @@ -848,7 +841,7 @@ fields("rpc") -> )}, {"keyfile", sc( - file(), + string(), #{ mapping => "gen_rpc.keyfile", converter => fun ensure_unicode_path/2, @@ -857,7 +850,7 @@ fields("rpc") -> )}, {"cacertfile", sc( - file(), + string(), #{ mapping => "gen_rpc.cacertfile", converter => fun ensure_unicode_path/2, @@ -985,7 +978,7 @@ fields("log") -> })}, {"file", sc( - ?UNION([ + hoconsc:union([ ?R_REF("log_file_handler"), ?MAP(handler_name, ?R_REF("log_file_handler")) ]), @@ -1004,7 +997,7 @@ fields("log_file_handler") -> [ {"path", sc( - file(), + string(), #{ desc => ?DESC("log_file_handler_file"), default => <<"${EMQX_LOG_DIR}/emqx.log">>, @@ -1538,3 +1531,6 @@ ensure_unicode_path(Path, _) when is_list(Path) -> Path; ensure_unicode_path(Path, _) -> throw({"not_string", Path}). + +log_level() -> + hoconsc:enum([debug, info, notice, warning, error, critical, alert, emergency, all]). diff --git a/apps/emqx_conf/src/emqx_conf_schema_types.erl b/apps/emqx_conf/src/emqx_conf_schema_types.erl new file mode 100644 index 000000000..e948142fc --- /dev/null +++ b/apps/emqx_conf/src/emqx_conf_schema_types.erl @@ -0,0 +1,334 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_schema_types). + +-export([readable/2]). +-export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]). + +%% Takes a typerefl name or hocon schema's display name and returns +%% a map of different flavors of more readable type specs. +%% - swagger: for swagger spec +%% - dashboard: to facilitate the dashboard UI rendering +%% - docgen: for documenation generation +readable(Module, TypeStr) when is_binary(TypeStr) -> + readable(Module, binary_to_list(TypeStr)); +readable(Module, TypeStr) when is_list(TypeStr) -> + try + %% Module is ignored so far as all types are distinguished by their names + readable(TypeStr) + catch + throw:unknown_type -> + fail(#{reason => unknown_type, type => TypeStr, module => Module}) + end. + +readable_swagger(Module, TypeStr) -> + get_readable(Module, TypeStr, swagger). + +readable_dashboard(Module, TypeStr) -> + get_readable(Module, TypeStr, dashboard). + +readable_docgen(Module, TypeStr) -> + get_readable(Module, TypeStr, docgen). + +get_readable(Module, TypeStr, Flavor) -> + Map = readable(Module, TypeStr), + case maps:get(Flavor, Map, undefined) of + undefined -> fail(#{reason => unknown_type, module => Module, type => TypeStr}); + Value -> Value + end. + +%% Fail the build or test. Production code should never get here. +-spec fail(_) -> no_return(). +fail(Reason) -> + io:format(standard_error, "ERROR: ~p~n", [Reason]), + error(Reason). + +readable("boolean()") -> + #{ + swagger => #{type => boolean}, + dashboard => #{type => boolean}, + docgen => #{type => "Boolean"} + }; +readable("binary()") -> + #{ + swagger => #{type => string}, + dashboard => #{type => string}, + docgen => #{type => "String"} + }; +readable("float()") -> + #{ + swagger => #{type => number}, + dashboard => #{type => number}, + docgen => #{type => "Float"} + }; +readable("integer()") -> + #{ + swagger => #{type => integer}, + dashboard => #{type => integer}, + docgen => #{type => "Integer"} + }; +readable("non_neg_integer()") -> + #{ + swagger => #{type => integer, minimum => 0}, + dashboard => #{type => integer, minimum => 0}, + docgen => #{type => "Integer(0..+inf)"} + }; +readable("pos_integer()") -> + #{ + swagger => #{type => integer, minimum => 1}, + dashboard => #{type => integer, minimum => 1}, + docgen => #{type => "Integer(1..+inf)"} + }; +readable("number()") -> + #{ + swagger => #{type => number}, + dashboard => #{type => number}, + docgen => #{type => "Number"} + }; +readable("string()") -> + #{ + swagger => #{type => string}, + dashboard => #{type => string}, + docgen => #{type => "String"} + }; +readable("atom()") -> + #{ + swagger => #{type => string}, + dashboard => #{type => string}, + docgen => #{type => "String"} + }; +readable("epoch_second()") -> + %% only for swagger + #{ + swagger => #{ + <<"oneOf">> => [ + #{type => integer, example => 1640995200, description => <<"epoch-second">>}, + #{ + type => string, + example => <<"2022-01-01T00:00:00.000Z">>, + format => <<"date-time">> + } + ] + } + }; +readable("epoch_millisecond()") -> + %% only for swagger + #{ + swagger => #{ + <<"oneOf">> => [ + #{ + type => integer, + example => 1640995200000, + description => <<"epoch-millisecond">> + }, + #{ + type => string, + example => <<"2022-01-01T00:00:00.000Z">>, + format => <<"date-time">> + } + ] + } + }; +readable("duration()") -> + #{ + swagger => #{type => string, example => <<"12m">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"12m">>} + }; +readable("duration_s()") -> + #{ + swagger => #{type => string, example => <<"1h">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"1h">>} + }; +readable("duration_ms()") -> + #{ + swagger => #{type => string, example => <<"32s">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"32s">>} + }; +readable("timeout_duration()") -> + #{ + swagger => #{type => string, example => <<"12m">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"12m">>} + }; +readable("timeout_duration_s()") -> + #{ + swagger => #{type => string, example => <<"1h">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"1h">>} + }; +readable("timeout_duration_ms()") -> + #{ + swagger => #{type => string, example => <<"32s">>}, + dashboard => #{type => duration}, + docgen => #{type => "String", example => <<"32s">>} + }; +readable("percent()") -> + #{ + swagger => #{type => string, example => <<"12%">>}, + dashboard => #{type => percent}, + docgen => #{type => "String", example => <<"12%">>} + }; +readable("ip_port()") -> + #{ + swagger => #{type => string, example => <<"127.0.0.1:80">>}, + dashboard => #{type => ip_port}, + docgen => #{type => "String", example => <<"127.0.0.1:80">>} + }; +readable("url()") -> + #{ + swagger => #{type => string, example => <<"http://127.0.0.1">>}, + dashboard => #{type => url}, + docgen => #{type => "String", example => <<"http://127.0.0.1">>} + }; +readable("bytesize()") -> + #{ + swagger => #{type => string, example => <<"32MB">>}, + dashboard => #{type => 'byteSize'}, + docgen => #{type => "String", example => <<"32MB">>} + }; +readable("wordsize()") -> + #{ + swagger => #{type => string, example => <<"1024KB">>}, + dashboard => #{type => 'wordSize'}, + docgen => #{type => "String", example => <<"1024KB">>} + }; +readable("map(" ++ Map) -> + [$) | _MapArgs] = lists:reverse(Map), + %% TODO: for docgen, parse map args. e.g. Map(String,String) + #{ + swagger => #{type => object, example => #{}}, + dashboard => #{type => object}, + docgen => #{type => "Map", example => #{}} + }; +readable("qos()") -> + #{ + swagger => #{type => integer, minimum => 0, maximum => 2, example => 0}, + dashboard => #{type => enum, symbols => [0, 1, 2]}, + docgen => #{type => "Integer(0..2)", example => 0} + }; +readable("comma_separated_list()") -> + #{ + swagger => #{type => string, example => <<"item1,item2">>}, + dashboard => #{type => comma_separated_string}, + docgen => #{type => "String", example => <<"item1,item2">>} + }; +readable("comma_separated_binary()") -> + #{ + swagger => #{type => string, example => <<"item1,item2">>}, + dashboard => #{type => comma_separated_string}, + docgen => #{type => "String", example => <<"item1,item2">>} + }; +readable("comma_separated_atoms()") -> + #{ + swagger => #{type => string, example => <<"item1,item2">>}, + dashboard => #{type => comma_separated_string}, + docgen => #{type => "String", example => <<"item1,item2">>} + }; +readable("service_account_json()") -> + %% This is a bit special, + %% service_account_josn in swagger spec is an object + %% the same in documenation. + %% However, dashboard wish it to be a string + %% TODO: + %% - Change type definition to stirng(). + %% - Convert the embedded object to a escaped JSON string. + %% - Delete this function clause once the above is done. + #{ + swagger => #{type => object}, + dashboard => #{type => string}, + docgen => #{type => "Map"} + }; +readable("json_binary()") -> + #{ + swagger => #{type => string, example => <<"{\"a\": [1,true]}">>}, + dashboard => #{type => string}, + docgen => #{type => "String", example => <<"{\"a\": [1,true]}">>} + }; +readable("port_number()") -> + Result = try_range("1..65535"), + true = is_map(Result), + Result; +readable(TypeStr0) -> + case string:split(TypeStr0, ":") of + [ModuleStr, TypeStr] -> + Module = list_to_existing_atom(ModuleStr), + readable(Module, TypeStr); + _ -> + parse(TypeStr0) + end. + +parse(TypeStr) -> + try_parse(TypeStr, [ + fun try_typerefl_array/1, + fun try_range/1 + ]). + +try_parse(_TypeStr, []) -> + throw(unknown_type); +try_parse(TypeStr, [ParseFun | More]) -> + case ParseFun(TypeStr) of + nomatch -> + try_parse(TypeStr, More); + Result -> + Result + end. + +%% [string()] or [integer()] or [xxx] or [xxx,...] +try_typerefl_array(Name) -> + case string:trim(Name, leading, "[") of + Name -> + nomatch; + Name1 -> + case string:trim(Name1, trailing, ",.]") of + Name1 -> + notmatch; + Name2 -> + Flavors = readable(Name2), + DocgenSpec = maps:get(docgen, Flavors), + DocgenType = maps:get(type, DocgenSpec), + #{ + swagger => #{type => array, items => maps:get(swagger, Flavors)}, + dashboard => #{type => array, items => maps:get(dashboard, Flavors)}, + docgen => #{type => "Array(" ++ DocgenType ++ ")"} + } + end + end. + +try_range(Name) -> + case string:split(Name, "..") of + %% 1..10 1..inf -inf..10 + [MinStr, MaxStr] -> + Schema0 = #{type => integer}, + Schema1 = add_integer_prop(Schema0, minimum, MinStr), + Schema = add_integer_prop(Schema1, maximum, MaxStr), + #{ + swagger => Schema, + dashboard => Schema, + docgen => #{type => "Integer(" ++ MinStr ++ ".." ++ MaxStr ++ ")"} + }; + _ -> + nomatch + end. + +add_integer_prop(Schema, Key, Value) -> + case string:to_integer(Value) of + {error, no_integer} -> Schema; + {Int, []} -> Schema#{Key => Int} + end. diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index cdb6ddd92..216dca031 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -22,7 +22,6 @@ -define(MYSQL_DEFAULT_PORT, 3306). -define(MONGO_DEFAULT_PORT, 27017). -define(REDIS_DEFAULT_PORT, 6379). --define(PGSQL_DEFAULT_PORT, 5432). -define(CLICKHOUSE_DEFAULT_PORT, 8123). -define(AUTO_RECONNECT_INTERVAL, 2). diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 78515abe6..94da3c580 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -8,8 +8,7 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, - {emqx_resource, {path, "../emqx_resource"}}, - {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}} + {emqx_resource, {path, "../emqx_resource"}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index d23a36e49..cc78829e7 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -10,8 +10,6 @@ ecpool, emqx_resource, eredis_cluster, - eredis, - epgsql, ehttpc, jose, emqx, diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index a277fe8c8..07e7fe375 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -20,13 +20,13 @@ -include_lib("hocon/include/hoconsc.hrl"). -export([ + pool_size/1, relational_db_fields/0, ssl_fields/0, prepare_statement_fields/0 ]). -export([ - pool_size/1, database/1, username/1, password/1, @@ -35,13 +35,11 @@ ]). -type database() :: binary(). --type pool_size() :: pos_integer(). -type username() :: binary(). -type password() :: binary(). -reflect_type([ database/0, - pool_size/0, username/0, password/0 ]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 2e3eb1d32..97691c6cd 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.29"}, + {vsn, "5.0.30"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 75e93fdd1..c1379d4d6 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -345,15 +345,7 @@ parse_spec_ref(Module, Path, Options) -> erlang:apply(Module, schema, [Path]) catch Error:Reason:Stacktrace -> - %% This error is intended to fail the build - %% hence print to standard_error - io:format( - standard_error, - "Failed to generate swagger for path ~p in module ~p~n" - "error:~p~nreason:~p~n~p~n", - [Module, Path, Error, Reason, Stacktrace] - ), - error({failed_to_generate_swagger_spec, Module, Path}) + failed_to_generate_swagger_spec(Module, Path, Error, Reason, Stacktrace) end, OperationId = maps:get('operationId', Schema), {Specs, Refs} = maps:fold( @@ -369,6 +361,24 @@ parse_spec_ref(Module, Path, Options) -> RouteOpts = generate_route_opts(Schema, Options), {OperationId, Specs, Refs, RouteOpts}. +-ifdef(TEST). +-spec failed_to_generate_swagger_spec(_, _, _, _, _) -> no_return(). +failed_to_generate_swagger_spec(Module, Path, _Error, _Reason, _Stacktrace) -> + error({failed_to_generate_swagger_spec, Module, Path}). +-else. +-spec failed_to_generate_swagger_spec(_, _, _, _, _) -> no_return(). +failed_to_generate_swagger_spec(Module, Path, Error, Reason, Stacktrace) -> + %% This error is intended to fail the build + %% hence print to standard_error + io:format( + standard_error, + "Failed to generate swagger for path ~p in module ~p~n" + "error:~p~nreason:~p~n~p~n", + [Module, Path, Error, Reason, Stacktrace] + ), + error({failed_to_generate_swagger_spec, Module, Path}). + +-endif. generate_route_opts(Schema, Options) -> #{filter => compose_filters(filter(Options), custom_filter(Schema))}. @@ -776,7 +786,7 @@ hocon_schema_to_spec(?MAP(Name, Type), LocalModule) -> }, SubRefs }; -hocon_schema_to_spec(?UNION(Types), LocalModule) -> +hocon_schema_to_spec(?UNION(Types, _DisplayName), LocalModule) -> {OneOf, Refs} = lists:foldl( fun(Type, {Acc, RefsAcc}) -> {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), @@ -789,190 +799,8 @@ hocon_schema_to_spec(?UNION(Types), LocalModule) -> hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> {#{type => string, enum => [Atom]}, []}. -typename_to_spec("term()", _Mod) -> - #{type => string, example => <<"any">>}; -typename_to_spec("boolean()", _Mod) -> - #{type => boolean}; -typename_to_spec("binary()", _Mod) -> - #{type => string}; -typename_to_spec("float()", _Mod) -> - #{type => number}; -typename_to_spec("integer()", _Mod) -> - #{type => integer}; -typename_to_spec("non_neg_integer()", _Mod) -> - #{type => integer, minimum => 0}; -typename_to_spec("pos_integer()", _Mod) -> - #{type => integer, minimum => 1}; -typename_to_spec("number()", _Mod) -> - #{type => number}; -typename_to_spec("string()", _Mod) -> - #{type => string}; -typename_to_spec("atom()", _Mod) -> - #{type => string}; -typename_to_spec("epoch_second()", _Mod) -> - #{ - <<"oneOf">> => [ - #{type => integer, example => 1640995200, description => <<"epoch-second">>}, - #{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>} - ] - }; -typename_to_spec("epoch_millisecond()", _Mod) -> - #{ - <<"oneOf">> => [ - #{type => integer, example => 1640995200000, description => <<"epoch-millisecond">>}, - #{type => string, example => <<"2022-01-01T00:00:00.000Z">>, format => <<"date-time">>} - ] - }; -typename_to_spec("duration()", _Mod) -> - #{type => string, example => <<"12m">>}; -typename_to_spec("duration_s()", _Mod) -> - #{type => string, example => <<"1h">>}; -typename_to_spec("duration_ms()", _Mod) -> - #{type => string, example => <<"32s">>}; -typename_to_spec("timeout_duration()", _Mod) -> - #{type => string, example => <<"12m">>}; -typename_to_spec("timeout_duration_s()", _Mod) -> - #{type => string, example => <<"1h">>}; -typename_to_spec("timeout_duration_ms()", _Mod) -> - #{type => string, example => <<"32s">>}; -typename_to_spec("percent()", _Mod) -> - #{type => number, example => <<"12%">>}; -typename_to_spec("file()", _Mod) -> - #{type => string, example => <<"/path/to/file">>}; -typename_to_spec("ip_port()", _Mod) -> - #{type => string, example => <<"127.0.0.1:80">>}; -typename_to_spec("write_syntax()", _Mod) -> - #{ - type => string, - example => - <<"${topic},clientid=${clientid}", " ", "payload=${payload},", - "${clientid}_int_value=${payload.int_key}i,", "bool=${payload.bool}">> - }; -typename_to_spec("url()", _Mod) -> - #{type => string, example => <<"http://127.0.0.1">>}; -typename_to_spec("connect_timeout()", Mod) -> - typename_to_spec("timeout()", Mod); -typename_to_spec("timeout()", _Mod) -> - #{ - <<"oneOf">> => [ - #{type => string, example => infinity}, - #{type => integer} - ], - example => infinity - }; -typename_to_spec("bytesize()", _Mod) -> - #{type => string, example => <<"32MB">>}; -typename_to_spec("wordsize()", _Mod) -> - #{type => string, example => <<"1024KB">>}; -typename_to_spec("map()", _Mod) -> - #{type => object, example => #{}}; -typename_to_spec("service_account_json()", _Mod) -> - #{type => object, example => #{}}; -typename_to_spec("#{" ++ _, Mod) -> - typename_to_spec("map()", Mod); -typename_to_spec("qos()", _Mod) -> - #{type => integer, minimum => 0, maximum => 2, example => 0}; -typename_to_spec("{binary(), binary()}", _Mod) -> - #{type => object, example => #{}}; -typename_to_spec("{string(), string()}", _Mod) -> - #{type => object, example => #{}}; -typename_to_spec("comma_separated_list()", _Mod) -> - #{type => string, example => <<"item1,item2">>}; -typename_to_spec("comma_separated_binary()", _Mod) -> - #{type => string, example => <<"item1,item2">>}; -typename_to_spec("comma_separated_atoms()", _Mod) -> - #{type => string, example => <<"item1,item2">>}; -typename_to_spec("pool_type()", _Mod) -> - #{type => string, enum => [random, hash]}; -typename_to_spec("log_level()", _Mod) -> - #{ - type => string, - enum => [debug, info, notice, warning, error, critical, alert, emergency, all] - }; -typename_to_spec("rate()", _Mod) -> - #{type => string, example => <<"10MB">>}; -typename_to_spec("burst()", _Mod) -> - #{type => string, example => <<"100MB">>}; -typename_to_spec("burst_rate()", _Mod) -> - %% 0/0s = no burst - #{type => string, example => <<"10MB">>}; -typename_to_spec("failure_strategy()", _Mod) -> - #{type => string, example => <<"force">>}; -typename_to_spec("initial()", _Mod) -> - #{type => string, example => <<"0MB">>}; -typename_to_spec("bucket_name()", _Mod) -> - #{type => string, example => <<"retainer">>}; -typename_to_spec("json_binary()", _Mod) -> - #{type => string, example => <<"{\"a\": [1,true]}">>}; -typename_to_spec("port_number()", _Mod) -> - range("1..65535"); -typename_to_spec("secret_access_key()", _Mod) -> - #{type => string, example => <<"TW8dPwmjpjJJuLW....">>}; -typename_to_spec(Name, Mod) -> - try_convert_to_spec(Name, Mod, [ - fun try_remote_module_type/2, - fun try_typerefl_array/2, - fun try_range/2, - fun try_integer/2 - ]). - -range(Name) -> - #{} = try_range(Name, undefined). - -try_convert_to_spec(Name, Mod, []) -> - throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}); -try_convert_to_spec(Name, Mod, [Converter | Rest]) -> - case Converter(Name, Mod) of - nomatch -> try_convert_to_spec(Name, Mod, Rest); - Spec -> Spec - end. - -try_range(Name, _Mod) -> - case string:split(Name, "..") of - %% 1..10 1..inf -inf..10 - [MinStr, MaxStr] -> - Schema = #{type => integer}, - Schema1 = add_integer_prop(Schema, minimum, MinStr), - add_integer_prop(Schema1, maximum, MaxStr); - _ -> - nomatch - end. - -%% Module:Type -try_remote_module_type(Name, Mod) -> - case string:split(Name, ":") of - [_Module, Type] -> typename_to_spec(Type, Mod); - _ -> nomatch - end. - -%% [string()] or [integer()] or [xxx] or [xxx,...] -try_typerefl_array(Name, Mod) -> - case string:trim(Name, leading, "[") of - Name -> - nomatch; - Name1 -> - case string:trim(Name1, trailing, ",.]") of - Name1 -> - notmatch; - Name2 -> - Schema = typename_to_spec(Name2, Mod), - #{type => array, items => Schema} - end - end. - -%% integer(1) -try_integer(Name, _Mod) -> - case string:to_integer(Name) of - {Int, []} -> #{type => integer, enum => [Int], default => Int}; - _ -> nomatch - end. - -add_integer_prop(Schema, Key, Value) -> - case string:to_integer(Value) of - {error, no_integer} -> Schema; - {Int, []} when Key =:= minimum -> Schema#{Key => Int}; - {Int, []} -> Schema#{Key => Int} - end. +typename_to_spec(TypeStr, Module) -> + emqx_conf_schema_types:readable_swagger(Module, TypeStr). to_bin(List) when is_list(List) -> case io_lib:printable_list(List) of diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 1c840e90c..9a9875935 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -248,8 +248,8 @@ clean_expired_jwt(Now) -> -if(?EMQX_RELEASE_EDITION == ee). check_rbac(Req, JWT) -> - #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = _Username} = JWT, - case emqx_dashboard_rbac:check_rbac(Req, Extra) of + #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = Username} = JWT, + case emqx_dashboard_rbac:check_rbac(Req, Username, Extra) of true -> save_new_jwt(JWT); _ -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 2457cd56a..b5c55622b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -816,7 +816,7 @@ to_schema(Body) -> fields(good_ref) -> [ {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, - {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, + {log_dir, mk(string(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; fields(nest_ref) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index d84f17c44..5987ad8fa 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -317,68 +317,68 @@ t_sub_fields(_Config) -> validate(Path, Object, ExpectRefs), ok. -t_complicated_type(_Config) -> +t_complex_type(_Config) -> Path = "/ref/complex_type", - Object = #{ - <<"content">> => #{ - <<"application/json">> => - #{ - <<"schema">> => #{ - <<"properties">> => - [ - {<<"no_neg_integer">>, #{minimum => 0, type => integer}}, - {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}}, - {<<"server">>, #{example => <<"127.0.0.1:80">>, type => string}}, - {<<"connect_timeout">>, #{ - example => infinity, - <<"oneOf">> => [ - #{example => infinity, type => string}, - #{type => integer} - ] - }}, - {<<"pool_type">>, #{enum => [random, hash], type => string}}, - {<<"timeout">>, #{ - example => infinity, - <<"oneOf">> => [ - #{example => infinity, type => string}, #{type => integer} - ] - }}, - {<<"bytesize">>, #{example => <<"32MB">>, type => string}}, - {<<"wordsize">>, #{example => <<"1024KB">>, type => string}}, - {<<"maps">>, #{example => #{}, type => object}}, - {<<"comma_separated_list">>, #{ - example => <<"item1,item2">>, type => string - }}, - {<<"comma_separated_atoms">>, #{ - example => <<"item1,item2">>, type => string - }}, - {<<"log_level">>, #{ - enum => [ - debug, - info, - notice, - warning, - error, - critical, - alert, - emergency, - all - ], - type => string - }}, - {<<"fix_integer">>, #{ - default => 100, enum => [100], type => integer - }} - ], - <<"type">> => object - } - } - } - }, {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), - ?assertEqual(Object, maps:get(<<"200">>, Response)), + ResponseBody = maps:get(<<"200">>, Response), + Content = maps:get(<<"content">>, ResponseBody), + JsonContent = maps:get(<<"application/json">>, Content), + Schema = maps:get(<<"schema">>, JsonContent), + ?assertMatch(#{<<"type">> := object}, Schema), + Properties = maps:get(<<"properties">>, Schema), + ?assertMatch( + [ + {<<"no_neg_integer">>, #{minimum := 0, type := integer}}, + {<<"url">>, #{ + example := <<"http://127.0.0.1">>, type := string + }}, + {<<"server">>, #{ + example := <<"127.0.0.1:80">>, type := string + }}, + {<<"connect_timeout">>, #{ + example := _, type := string + }}, + {<<"pool_type">>, #{ + enum := [random, hash], type := string + }}, + {<<"timeout">>, #{ + <<"oneOf">> := [ + #{example := _, type := string}, + #{enum := [infinity], type := string} + ] + }}, + {<<"bytesize">>, #{ + example := <<"32MB">>, type := string + }}, + {<<"wordsize">>, #{ + example := <<"1024KB">>, type := string + }}, + {<<"maps">>, #{example := #{}, type := object}}, + {<<"comma_separated_list">>, #{ + example := <<"item1,item2">>, type := string + }}, + {<<"comma_separated_atoms">>, #{ + example := <<"item1,item2">>, type := string + }}, + {<<"log_level">>, #{ + enum := [ + debug, + info, + notice, + warning, + error, + critical, + alert, + emergency, + all + ], + type := string + }} + ], + Properties + ), ?assertEqual([], Refs), ok. @@ -410,7 +410,7 @@ t_ref_array_with_key(_Config) -> {<<"percent_ex">>, #{ description => <<"percent example">>, example => <<"12%">>, - type => number + type => string }}, {<<"duration_ms_ex">>, #{ description => <<"duration ms example">>, @@ -647,17 +647,16 @@ schema("/ref/complex_type") -> {no_neg_integer, hoconsc:mk(non_neg_integer(), #{})}, {url, hoconsc:mk(url(), #{})}, {server, hoconsc:mk(emqx_schema:ip_port(), #{})}, - {connect_timeout, - hoconsc:mk(emqx_bridge_http_connector:connect_timeout(), #{})}, - {pool_type, hoconsc:mk(emqx_bridge_http_connector:pool_type(), #{})}, - {timeout, hoconsc:mk(timeout(), #{})}, + {connect_timeout, hoconsc:mk(emqx_schema:timeout_duration(), #{})}, + {pool_type, hoconsc:mk(hoconsc:enum([random, hash]), #{})}, + {timeout, + hoconsc:mk(hoconsc:union([infinity, emqx_schema:timeout_duration()]), #{})}, {bytesize, hoconsc:mk(emqx_schema:bytesize(), #{})}, {wordsize, hoconsc:mk(emqx_schema:wordsize(), #{})}, {maps, hoconsc:mk(map(), #{})}, {comma_separated_list, hoconsc:mk(emqx_schema:comma_separated_list(), #{})}, {comma_separated_atoms, hoconsc:mk(emqx_schema:comma_separated_atoms(), #{})}, - {log_level, hoconsc:mk(emqx_conf_schema:log_level(), #{})}, - {fix_integer, hoconsc:mk(typerefl:integer(100), #{})} + {log_level, hoconsc:mk(emqx_conf_schema:log_level(), #{})} ] } } @@ -684,7 +683,7 @@ to_schema(Object) -> fields(good_ref) -> [ {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, - {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, + {log_dir, mk(string(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; fields(nest_ref) -> diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src index 190764e2f..ec8e6cd3f 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_rbac, [ {description, "EMQX Dashboard RBAC"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index 28bd8960e..57132b65b 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -6,18 +6,18 @@ -include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). --export([check_rbac/2, role/1, valid_role/1]). +-export([check_rbac/3, role/1, valid_role/1]). -dialyzer({nowarn_function, role/1}). %%===================================================================== %% API -check_rbac(Req, Extra) -> +check_rbac(Req, Username, Extra) -> Role = role(Extra), Method = cowboy_req:method(Req), AbsPath = cowboy_req:path(Req), case emqx_dashboard_swagger:get_relative_uri(AbsPath) of {ok, Path} -> - check_rbac(Role, Method, Path); + check_rbac(Role, Method, Path, Username); _ -> false end. @@ -41,14 +41,21 @@ valid_role(Role) -> {error, <<"Role does not exist">>} end. %% =================================================================== -check_rbac(?ROLE_SUPERUSER, _, _) -> +check_rbac(?ROLE_SUPERUSER, _, _, _) -> true; -check_rbac(?ROLE_VIEWER, <<"GET">>, _) -> +check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) -> true; -%% this API is a special case -check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) -> +%% everyone should allow to logout +check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>, _) -> true; -check_rbac(_, _, _) -> +%% viewer should allow to change self password, +%% superuser should allow to change any user +check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/users/", SubPath/binary>>, Username) -> + case binary:split(SubPath, <<"/">>, [global]) of + [Username, <<"change_pwd">>] -> true; + _ -> false + end; +check_rbac(_, _, _, _) -> false. role_list() -> diff --git a/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl b/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl index b1a51a3c9..eeac8dadf 100644 --- a/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl +++ b/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl @@ -160,6 +160,34 @@ t_login_out(_) -> {ok, Username} = emqx_dashboard_admin:verify_token(FakeReq, Token), ok. +t_change_pwd(_) -> + Viewer1 = <<"viewer1">>, + Viewer2 = <<"viewer2">>, + SuperUser = <<"super_user">>, + Password = <<"public_www1">>, + Desc = <<"desc">>, + {ok, _} = emqx_dashboard_admin:add_user(Viewer1, Password, ?ROLE_VIEWER, Desc), + {ok, _} = emqx_dashboard_admin:add_user(Viewer2, Password, ?ROLE_VIEWER, Desc), + {ok, _} = emqx_dashboard_admin:add_user(SuperUser, Password, ?ROLE_SUPERUSER, Desc), + {ok, ?ROLE_VIEWER, Viewer1Token} = emqx_dashboard_admin:sign_token(Viewer1, Password), + {ok, ?ROLE_SUPERUSER, SuperToken} = emqx_dashboard_admin:sign_token(SuperUser, Password), + %% viewer can change own password + ?assertEqual({ok, Viewer1}, change_pwd(Viewer1Token, Viewer1)), + %% viewer can't change other's password + ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, Viewer2)), + ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, SuperUser)), + %% superuser can change other's password + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer1)), + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer2)), + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, SuperUser)), + ok. + +change_pwd(Token, Username) -> + Path = "/users/" ++ binary_to_list(Username) ++ "/change_pwd", + Path1 = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri(Path)), + Req = #{method => <<"POST">>, path => Path1}, + emqx_dashboard_admin:verify_token(Req, Token). + add_default_superuser() -> {ok, _NewUser} = emqx_dashboard_admin:add_user( ?DEFAULT_SUPERUSER, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index e00a3cbfa..71788947b 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl index 499e24c5b..583f1d683 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl @@ -12,6 +12,7 @@ -behaviour(emqx_dashboard_sso). -export([ + namespace/0, fields/1, desc/1 ]). @@ -30,6 +31,9 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> + "sso". + hocon_ref() -> hoconsc:ref(?MODULE, ldap). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index 92f9ba519..aa032a3cc 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -8,7 +8,7 @@ -include_lib("typerefl/include/types.hrl"). %% Hocon --export([fields/1, desc/1]). +-export([namespace/0, fields/1, desc/1]). -export([ common_backend_schema/1, @@ -21,6 +21,8 @@ %%------------------------------------------------------------------------------ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "sso". + fields(sso) -> lists:map( fun({Type, Module}) -> diff --git a/apps/emqx_durable_storage/IMPLEMENTATION.md b/apps/emqx_durable_storage/IMPLEMENTATION.md index 9c0c5928c..33f02dfc4 100644 --- a/apps/emqx_durable_storage/IMPLEMENTATION.md +++ b/apps/emqx_durable_storage/IMPLEMENTATION.md @@ -31,48 +31,6 @@ Read pattern: pseudoserial Number of records: O(total write throughput * retention time) -## Session storage - -Data there is updated when: - -- A new client connects with clean session = false -- Client subscribes to a topic -- Client unsubscribes to a topic -- Garbage collection is performed - -Write throughput: low - -Data is read when a client connects and replay agents are started - -Read throughput: low - -Data format: - -`#session{clientId = "foobar", iterators = [ItKey1, ItKey2, ItKey3, ...]}` - -Number of records: O(N clients) - -Size of record: O(N subscriptions per clients) - -## Iterator storage - -Data is written every time a client acks a message. - -Data is read when a client reconnects and we restart replay agents. - -`#iterator{key = IterKey, data = Blob}` - -Number of records: O(N clients * N subscriptions per client) - -Size of record: O(1) - -Write throughput: high, lots of small updates - -Write pattern: mostly key overwrite - -Read throughput: low - -Read pattern: random # Push vs. Pull model diff --git a/apps/emqx_durable_storage/README.md b/apps/emqx_durable_storage/README.md index 7de43bee0..f01af0c37 100644 --- a/apps/emqx_durable_storage/README.md +++ b/apps/emqx_durable_storage/README.md @@ -1,9 +1,10 @@ # EMQX Replay -`emqx_ds` is a durable storage for MQTT messages within EMQX. -It implements the following scenarios: -- Persisting messages published by clients -- +`emqx_ds` is a generic durable storage for MQTT messages within EMQX. + +Concepts: + + > 0. App overview introduction > 1. let people know what your project can do specifically. Is it a base diff --git a/apps/emqx_durable_storage/include/emqx_ds.hrl b/apps/emqx_durable_storage/include/emqx_ds.hrl new file mode 100644 index 000000000..c9ee4b7f7 --- /dev/null +++ b/apps/emqx_durable_storage/include/emqx_ds.hrl @@ -0,0 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_DS_HRL_HRL). +-define(EMQX_DS_HRL_HRL, true). + +-endif. diff --git a/apps/emqx_durable_storage/rebar.config b/apps/emqx_durable_storage/rebar.config new file mode 100644 index 000000000..f04819025 --- /dev/null +++ b/apps/emqx_durable_storage/rebar.config @@ -0,0 +1,3 @@ +%% -*- mode:erlang -*- +{deps, + [{emqx_utils, {path, "../emqx_utils"}}]}. diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index feaa37bc0..725d62673 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -13,50 +13,48 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- + +%% @doc Main interface module for `emqx_durable_storage' application. +%% +%% It takes care of forwarding calls to the underlying DBMS. Currently +%% only the embedded `emqx_ds_replication_layer' storage is supported, +%% so all the calls are simply passed through. -module(emqx_ds). --include_lib("stdlib/include/ms_transform.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). +%% Management API: +-export([open_db/2, drop_db/1]). -%% API: --export([ensure_shard/2]). -%% Messages: --export([message_store/2, message_store/1, message_stats/0]). -%% Iterator: --export([iterator_update/2, iterator_next/1, iterator_stats/0]). +%% Message storage API: +-export([store_batch/2, store_batch/3]). -%% internal exports: +%% Message replay API: +-export([get_streams/3, make_iterator/4, next/3]). + +%% Misc. API: -export([]). -export_type([ - keyspace/0, - message_id/0, - message_stats/0, - message_store_opts/0, - replay/0, - replay_id/0, - iterator_id/0, - iterator/0, - shard/0, - shard_id/0, - topic/0, + create_db_opts/0, + builtin_db_opts/0, + db/0, + time/0, topic_filter/0, - time/0 + topic/0, + stream/0, + stream_rank/0, + iterator/0, + message_id/0, + next_result/1, next_result/0, + store_batch_result/0, + make_iterator_result/1, make_iterator_result/0, + get_iterator_result/1 ]). %%================================================================================ %% Type declarations %%================================================================================ --type iterator() :: term(). - --type iterator_id() :: binary(). - --type message_store_opts() :: #{}. - --type message_stats() :: #{}. - --type message_id() :: binary(). +-type db() :: atom(). %% Parsed topic. -type topic() :: list(binary()). @@ -64,9 +62,22 @@ %% Parsed topic filter. -type topic_filter() :: list(binary() | '+' | '#' | ''). --type keyspace() :: atom(). --type shard_id() :: binary(). --type shard() :: {keyspace(), shard_id()}. +-type stream_rank() :: {term(), integer()}. + +-opaque stream() :: emqx_ds_replication_layer:stream(). + +-opaque iterator() :: emqx_ds_replication_layer:iterator(). + +-type store_batch_result() :: ok | {error, _}. + +-type make_iterator_result(Iterator) :: {ok, Iterator} | {error, _}. + +-type make_iterator_result() :: make_iterator_result(iterator()). + +-type next_result(Iterator) :: + {ok, Iterator, [emqx_types:message()]} | {ok, end_of_stream} | {error, _}. + +-type next_result() :: next_result(iterator()). %% Timestamp %% Earliest possible timestamp is 0. @@ -74,70 +85,132 @@ %% use in emqx_guid. Otherwise, the iterators won't match the message timestamps. -type time() :: non_neg_integer(). --type replay_id() :: binary(). +-type message_store_opts() :: #{}. --type replay() :: { - _TopicFilter :: topic_filter(), - _StartTime :: time() -}. +-type builtin_db_opts() :: + #{ + backend := builtin, + storage := emqx_ds_storage_layer:prototype() + }. + +-type create_db_opts() :: + builtin_db_opts(). + +-type message_id() :: emqx_ds_replication_layer:message_id(). + +-type get_iterator_result(Iterator) :: {ok, Iterator} | undefined. + +-define(persistent_term(DB), {emqx_ds_db_backend, DB}). + +-define(module(DB), (persistent_term:get(?persistent_term(DB)))). + +%%================================================================================ +%% Behavior callbacks +%%================================================================================ + +-callback open_db(db(), create_db_opts()) -> ok | {error, _}. + +-callback drop_db(db()) -> ok | {error, _}. + +-callback store_batch(db(), [emqx_types:message()], message_store_opts()) -> store_batch_result(). + +-callback get_streams(db(), topic_filter(), time()) -> [{stream_rank(), stream()}]. + +-callback make_iterator(db(), _Stream, topic_filter(), time()) -> make_iterator_result(_Iterator). + +-callback next(db(), Iterator, pos_integer()) -> next_result(Iterator). %%================================================================================ %% API funcions %%================================================================================ --spec ensure_shard(shard(), emqx_ds_storage_layer:options()) -> - ok | {error, _Reason}. -ensure_shard(Shard, Options) -> - case emqx_ds_storage_layer_sup:start_shard(Shard, Options) of - {ok, _Pid} -> +%% @doc Different DBs are completely independent from each other. They +%% could represent something like different tenants. +-spec open_db(db(), create_db_opts()) -> ok. +open_db(DB, Opts = #{backend := Backend}) when Backend =:= builtin -> + Module = + case Backend of + builtin -> emqx_ds_replication_layer + end, + persistent_term:put(?persistent_term(DB), Module), + ?module(DB):open_db(DB, Opts). + +%% @doc TODO: currently if one or a few shards are down, they won't be + +%% deleted. +-spec drop_db(db()) -> ok. +drop_db(DB) -> + case persistent_term:get(?persistent_term(DB), undefined) of + undefined -> ok; - {error, {already_started, _Pid}} -> - ok; - {error, Reason} -> - {error, Reason} + Module -> + Module:drop_db(DB) end. -%%-------------------------------------------------------------------------------- -%% Message -%%-------------------------------------------------------------------------------- --spec message_store([emqx_types:message()], message_store_opts()) -> - {ok, [message_id()]} | {error, _}. -message_store(_Msg, _Opts) -> - %% TODO - {error, not_implemented}. +-spec store_batch(db(), [emqx_types:message()], message_store_opts()) -> store_batch_result(). +store_batch(DB, Msgs, Opts) -> + ?module(DB):store_batch(DB, Msgs, Opts). --spec message_store([emqx_types:message()]) -> {ok, [message_id()]} | {error, _}. -message_store(Msg) -> - %% TODO - message_store(Msg, #{}). +-spec store_batch(db(), [emqx_types:message()]) -> store_batch_result(). +store_batch(DB, Msgs) -> + store_batch(DB, Msgs, #{}). --spec message_stats() -> message_stats(). -message_stats() -> - #{}. +%% @doc Get a list of streams needed for replaying a topic filter. +%% +%% Motivation: under the hood, EMQX may store different topics at +%% different locations or even in different databases. A wildcard +%% topic filter may require pulling data from any number of locations. +%% +%% Stream is an abstraction exposed by `emqx_ds' that, on one hand, +%% reflects the notion that different topics can be stored +%% differently, but hides the implementation details. +%% +%% While having to work with multiple iterators to replay a topic +%% filter may be cumbersome, it opens up some possibilities: +%% +%% 1. It's possible to parallelize replays +%% +%% 2. Streams can be shared between different clients to implement +%% shared subscriptions +%% +%% IMPORTANT RULES: +%% +%% 0. There is no 1-to-1 mapping between MQTT topics and streams. One +%% stream can contain any number of MQTT topics. +%% +%% 1. New streams matching the topic filter and start time can appear +%% without notice, so the replayer must periodically call this +%% function to get the updated list of streams. +%% +%% 2. Streams may depend on one another. Therefore, care should be +%% taken while replaying them in parallel to avoid out-of-order +%% replay. This function returns stream together with its +%% "coordinate": `stream_rank()'. +%% +%% Stream rank is a tuple of two integers, let's call them X and Y. If +%% X coordinate of two streams is different, they are independent and +%% can be replayed in parallel. If it's the same, then the stream with +%% smaller Y coordinate should be replayed first. If Y coordinates are +%% equal, then the streams are independent. +%% +%% Stream is fully consumed when `next/3' function returns +%% `end_of_stream'. Then and only then the client can proceed to +%% replaying streams that depend on the given one. +-spec get_streams(db(), topic_filter(), time()) -> [{stream_rank(), stream()}]. +get_streams(DB, TopicFilter, StartTime) -> + ?module(DB):get_streams(DB, TopicFilter, StartTime). -%%-------------------------------------------------------------------------------- -%% Session -%%-------------------------------------------------------------------------------- +-spec make_iterator(db(), stream(), topic_filter(), time()) -> make_iterator_result(). +make_iterator(DB, Stream, TopicFilter, StartTime) -> + ?module(DB):make_iterator(DB, Stream, TopicFilter, StartTime). -%%-------------------------------------------------------------------------------- -%% Iterator (pull API) -%%-------------------------------------------------------------------------------- +-spec next(db(), iterator(), pos_integer()) -> next_result(). +next(DB, Iter, BatchSize) -> + ?module(DB):next(DB, Iter, BatchSize). -%% @doc Called when a client acks a message --spec iterator_update(iterator_id(), iterator()) -> ok. -iterator_update(_IterId, _Iter) -> - %% TODO - ok. - -%% @doc Called when a client acks a message --spec iterator_next(iterator()) -> {value, emqx_types:message(), iterator()} | none | {error, _}. -iterator_next(_Iter) -> - %% TODO - none. - --spec iterator_stats() -> #{}. -iterator_stats() -> - #{}. +%%================================================================================ +%% Internal exports +%%================================================================================ %%================================================================================ %% Internal functions diff --git a/apps/emqx_connector/src/emqx_connector_utils.erl b/apps/emqx_durable_storage/src/emqx_ds_bitmask.hrl similarity index 54% rename from apps/emqx_connector/src/emqx_connector_utils.erl rename to apps/emqx_durable_storage/src/emqx_ds_bitmask.hrl index 6000f6be5..31af0e034 100644 --- a/apps/emqx_connector/src/emqx_connector_utils.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_bitmask.hrl @@ -13,23 +13,24 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- +-ifndef(EMQX_DS_BITMASK_HRL). +-define(EMQX_DS_BITMASK_HRL, true). --module(emqx_connector_utils). +-record(filter_scan_action, { + offset :: emqx_ds_bitmask_keymapper:offset(), + size :: emqx_ds_bitmask_keymapper:bitsize(), + min :: non_neg_integer(), + max :: non_neg_integer() +}). --export([split_insert_sql/1]). +-record(filter, { + size :: non_neg_integer(), + bitfilter :: non_neg_integer(), + bitmask :: non_neg_integer(), + %% Ranges (in _bitsource_ basis): + bitsource_ranges :: array:array(#filter_scan_action{}), + range_min :: non_neg_integer(), + range_max :: non_neg_integer() +}). -%% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">> -split_insert_sql(SQL) -> - case re:split(SQL, "((?i)values)", [{return, binary}]) of - [Part1, _, Part3] -> - case string:trim(Part1, leading) of - <<"insert", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - <<"INSERT", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - _ -> - {error, not_insert_sql} - end; - _ -> - {error, not_insert_sql} - end. +-endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_bitmask_keymapper.erl b/apps/emqx_durable_storage/src/emqx_ds_bitmask_keymapper.erl new file mode 100644 index 000000000..a3b65c7e6 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_bitmask_keymapper.erl @@ -0,0 +1,824 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ds_bitmask_keymapper). + +%%================================================================================ +%% @doc This module is used to map N-dimensional coordinates to a +%% 1-dimensional space. +%% +%% Example: +%% +%% Let us assume that `T' is a topic and `t' is time. These are the two +%% dimensions used to index messages. They can be viewed as +%% "coordinates" of an MQTT message in a 2D space. +%% +%% Oftentimes, when wildcard subscription is used, keys must be +%% scanned in both dimensions simultaneously. +%% +%% Rocksdb allows to iterate over sorted keys very fast. This means we +%% need to map our two-dimentional keys to a single index that is +%% sorted in a way that helps to iterate over both time and topic +%% without having to do a lot of random seeks. +%% +%% == Mapping of 2D keys to rocksdb keys == +%% +%% We use "zigzag" pattern to store messages, where rocksdb key is +%% composed like like this: +%% +%% |ttttt|TTTTTTTTT|tttt| +%% ^ ^ ^ +%% | | | +%% +-------+ | +---------+ +%% | | | +%% most significant topic hash least significant +%% bits of timestamp bits of timestamp +%% (a.k.a epoch) (a.k.a time offset) +%% +%% Topic hash is level-aware: each topic level is hashed separately +%% and the resulting hashes are bitwise-concatentated. This allows us +%% to map topics to fixed-length bitstrings while keeping some degree +%% of information about the hierarchy. +%% +%% Next important concept is what we call "epoch". Duration of the +%% epoch is determined by maximum time offset. Epoch is calculated by +%% shifting bits of the timestamp right. +%% +%% The resulting index is a space-filling curve that looks like +%% this in the topic-time 2D space: +%% +%% T ^ ---->------ |---->------ |---->------ +%% | --/ / --/ / --/ +%% | -<-/ | -<-/ | -<-/ +%% | -/ | -/ | -/ +%% | ---->------ | ---->------ | ---->------ +%% | --/ / --/ / --/ +%% | ---/ | ---/ | ---/ +%% | -/ ^ -/ ^ -/ +%% | ---->------ | ---->------ | ---->------ +%% | --/ / --/ / --/ +%% | -<-/ | -<-/ | -<-/ +%% | -/ | -/ | -/ +%% | ---->------| ---->------| ----------> +%% | +%% -+------------+-----------------------------> t +%% epoch +%% +%% This structure allows to quickly seek to a the first message that +%% was recorded in a certain epoch in a certain topic or a +%% group of topics matching filter like `foo/bar/#`. +%% +%% Due to its structure, for each pair of rocksdb keys K1 and K2, such +%% that K1 > K2 and topic(K1) = topic(K2), timestamp(K1) > +%% timestamp(K2). +%% That is, replay doesn't reorder messages published in each +%% individual topic. +%% +%% This property doesn't hold between different topics, but it's not deemed +%% a problem right now. +%% +%%================================================================================ + +%% API: +-export([ + make_keymapper/1, + vector_to_key/2, + bin_vector_to_key/2, + key_to_vector/2, + bin_key_to_vector/2, + key_to_bitstring/2, + bitstring_to_key/2, + make_filter/2, + ratchet/2, + bin_increment/2, + bin_checkmask/2, + bitsize/1 +]). + +-export_type([vector/0, key/0, dimension/0, offset/0, bitsize/0, bitsource/0, keymapper/0]). + +-compile( + {inline, [ + ones/1, + extract/2, + extract_inv/2 + ]} +). + +-elvis([{elvis_style, no_if_expression, disable}]). + +-ifdef(TEST). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%%================================================================================ +%% Type declarations +%%================================================================================ + +-type scalar() :: integer(). + +-type vector() :: [scalar()]. + +%% N-th coordinate of a vector: +-type dimension() :: pos_integer(). + +-type offset() :: non_neg_integer(). + +-type bitsize() :: pos_integer(). + +%% The resulting 1D key: +-type key() :: non_neg_integer(). + +-type bitsource() :: + %% Consume `_Size` bits from timestamp starting at `_Offset`th + %% bit from N-th element of the input vector: + {dimension(), offset(), bitsize()}. + +-record(scan_action, { + src_bitmask :: integer(), + src_offset :: offset(), + dst_offset :: offset() +}). + +-type scan_action() :: #scan_action{}. + +-type scanner() :: [[scan_action()]]. + +-record(keymapper, { + schema :: [bitsource()], + scanner :: scanner(), + size :: non_neg_integer(), + dim_sizeof :: [non_neg_integer()] +}). + +-opaque keymapper() :: #keymapper{}. + +-type scalar_range() :: + any | {'=', scalar() | infinity} | {'>=', scalar()} | {scalar(), '..', scalar()}. + +-include("emqx_ds_bitmask.hrl"). + +-type filter() :: #filter{}. + +%%================================================================================ +%% API functions +%%================================================================================ + +%% @doc Create a keymapper object that stores the "schema" of the +%% transformation from a list of bitsources. +%% +%% Note: Dimension is 1-based. +%% +%% Note: order of bitsources is important. First element of the list +%% is mapped to the _least_ significant bits of the key, and the last +%% element becomes most significant bits. +-spec make_keymapper([bitsource()]) -> keymapper(). +make_keymapper(Bitsources) -> + Arr0 = array:new([{fixed, false}, {default, {0, []}}]), + {Size, Arr} = fold_bitsources( + fun(DestOffset, {Dim0, Offset, Size}, Acc) -> + Dim = Dim0 - 1, + Action = #scan_action{ + src_bitmask = ones(Size), src_offset = Offset, dst_offset = DestOffset + }, + {DimSizeof, Actions} = array:get(Dim, Acc), + array:set(Dim, {DimSizeof + Size, [Action | Actions]}, Acc) + end, + Arr0, + Bitsources + ), + {DimSizeof, Scanner} = lists:unzip(array:to_list(Arr)), + #keymapper{ + schema = Bitsources, + scanner = Scanner, + size = Size, + dim_sizeof = DimSizeof + }. + +-spec bitsize(keymapper()) -> pos_integer(). +bitsize(#keymapper{size = Size}) -> + Size. + +%% @doc Map N-dimensional vector to a scalar key. +%% +%% Note: this function is not injective. +-spec vector_to_key(keymapper(), vector()) -> key(). +vector_to_key(#keymapper{scanner = []}, []) -> + 0; +vector_to_key(#keymapper{scanner = [Actions | Scanner]}, [Coord | Vector]) -> + do_vector_to_key(Actions, Scanner, Coord, Vector, 0). + +%% @doc Same as `vector_to_key', but it works with binaries, and outputs a binary. +-spec bin_vector_to_key(keymapper(), [binary()]) -> binary(). +bin_vector_to_key(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, Binaries) -> + Vec = lists:zipwith( + fun(Bin, SizeOf) -> + <> = Bin, + Int + end, + Binaries, + DimSizeof + ), + Key = vector_to_key(Keymapper, Vec), + <>. + +%% @doc Map key to a vector. +%% +%% Note: `vector_to_key(key_to_vector(K)) = K' but +%% `key_to_vector(vector_to_key(V)) = V' is not guaranteed. +-spec key_to_vector(keymapper(), key()) -> vector(). +key_to_vector(#keymapper{scanner = Scanner}, Key) -> + lists:map( + fun(Actions) -> + lists:foldl( + fun(Action, Acc) -> + Acc bor extract_inv(Key, Action) + end, + 0, + Actions + ) + end, + Scanner + ). + +%% @doc Same as `key_to_vector', but it works with binaries. +-spec bin_key_to_vector(keymapper(), binary()) -> [binary()]. +bin_key_to_vector(Keymapper = #keymapper{dim_sizeof = DimSizeof, size = Size}, BinKey) -> + <> = BinKey, + Vector = key_to_vector(Keymapper, Key), + lists:zipwith( + fun(Elem, SizeOf) -> + <> + end, + Vector, + DimSizeof + ). + +%% @doc Transform a bitstring to a key +-spec bitstring_to_key(keymapper(), bitstring()) -> key(). +bitstring_to_key(#keymapper{size = Size}, Bin) -> + case Bin of + <> -> + Key; + _ -> + error({invalid_key, Bin, Size}) + end. + +%% @doc Transform key to a fixed-size bistring +-spec key_to_bitstring(keymapper(), key()) -> bitstring(). +key_to_bitstring(#keymapper{size = Size}, Key) -> + <>. + +%% @doc Create a filter object that facilitates range scans. +-spec make_filter(keymapper(), [scalar_range()]) -> filter(). +make_filter( + KeyMapper = #keymapper{schema = Schema, dim_sizeof = DimSizeof, size = TotalSize}, Filter0 +) -> + NDim = length(DimSizeof), + %% Transform "symbolic" constraints to ranges: + Filter1 = constraints_to_ranges(KeyMapper, Filter0), + {Bitmask, Bitfilter} = make_bitfilter(KeyMapper, Filter1), + %% Calculate maximum source offset as per bitsource specification: + MaxOffset = lists:foldl( + fun({Dim, Offset, _Size}, Acc) -> + maps:update_with( + Dim, fun(OldVal) -> max(OldVal, Offset) end, maps:merge(#{Dim => 0}, Acc) + ) + end, + #{}, + Schema + ), + %% Adjust minimum and maximum values for each interval like this: + %% + %% Min: 110100|101011 -> 110100|00000 + %% Max: 110101|001011 -> 110101|11111 + %% ^ + %% | + %% max offset + %% + %% This is needed so when we increment the vector, we always scan + %% the full range of least significant bits. + Filter2 = lists:zipwith( + fun + ({Val, Val}, _Dim) -> + {Val, Val}; + ({Min0, Max0}, Dim) -> + Offset = maps:get(Dim, MaxOffset, 0), + %% Set least significant bits of Min to 0: + Min = (Min0 bsr Offset) bsl Offset, + %% Set least significant bits of Max to 1: + Max = Max0 bor ones(Offset), + {Min, Max} + end, + Filter1, + lists:seq(1, NDim) + ), + %% Project the vector into "bitsource coordinate system": + {_, Filter} = fold_bitsources( + fun(DstOffset, {Dim, SrcOffset, Size}, Acc) -> + {Min0, Max0} = lists:nth(Dim, Filter2), + Min = (Min0 bsr SrcOffset) band ones(Size), + Max = (Max0 bsr SrcOffset) band ones(Size), + Action = #filter_scan_action{ + offset = DstOffset, + size = Size, + min = Min, + max = Max + }, + [Action | Acc] + end, + [], + Schema + ), + Ranges = array:from_list(lists:reverse(Filter)), + %% Compute estimated upper and lower bounds of a _continous_ + %% interval where all keys lie: + case Filter of + [] -> + RangeMin = 0, + RangeMax = 0; + [#filter_scan_action{offset = MSBOffset, min = MSBMin, max = MSBMax} | _] -> + RangeMin = MSBMin bsl MSBOffset, + RangeMax = MSBMax bsl MSBOffset bor ones(MSBOffset) + end, + %% Final value + #filter{ + size = TotalSize, + bitmask = Bitmask, + bitfilter = Bitfilter, + bitsource_ranges = Ranges, + range_min = RangeMin, + range_max = RangeMax + }. + +%% @doc Given a filter `F' and key `K0', return the smallest key `K' +%% that satisfies the following conditions: +%% +%% 1. `K >= K0' +%% +%% 2. `K' satisfies filter `F'. +%% +%% If these conditions cannot be satisfied, return `overflow'. +%% +%% Corollary: `K' may be equal to `K0'. +-spec ratchet(filter(), key()) -> key() | overflow. +ratchet(#filter{bitsource_ranges = Ranges, range_max = Max}, Key) when Key =< Max -> + %% This function works in two steps: first, it finds the position + %% of bitsource ("pivot point") corresponding to the part of the + %% key that should be incremented (or set to the _minimum_ value + %% of the range, in case the respective part of the original key + %% is less than the minimum). It also returns "increment": value + %% that should be added to the part of the key at the pivot point. + %% Increment can be 0 or 1. + %% + %% Then it transforms the key using the following operation: + %% + %% 1. Parts of the key that are less than the pivot point are + %% reset to their minimum values. + %% + %% 2. `Increment' is added to the part of the key at the pivot + %% point. + %% + %% 3. The rest of key stays the same + NDim = array:size(Ranges), + case ratchet_scan(Ranges, NDim, Key, 0, {_Pivot0 = -1, _Increment0 = 0}, _Carry = 0) of + overflow -> + overflow; + {Pivot, Increment} -> + ratchet_do(Ranges, Key, NDim - 1, Pivot, Increment) + end; +ratchet(_, _) -> + overflow. + +%% @doc Given a binary representing a key and a filter, return the +%% next key matching the filter, or `overflow' if such key doesn't +%% exist. +-spec bin_increment(filter(), binary()) -> binary() | overflow. +bin_increment(Filter = #filter{size = Size}, <<>>) -> + Key = ratchet(Filter, 0), + <>; +bin_increment( + Filter = #filter{size = Size, bitmask = Bitmask, bitfilter = Bitfilter, range_max = RangeMax}, + KeyBin +) -> + %% The key may contain random suffix, skip it: + <> = KeyBin, + Key1 = Key0 + 1, + if + Key1 band Bitmask =:= Bitfilter, Key1 =< RangeMax -> + <>; + true -> + case ratchet(Filter, Key1) of + overflow -> + overflow; + Key -> + <> + end + end. + +%% @doc Given a filter and a binary representation of a key, return +%% `false' if the key _doesn't_ match the fitler. This function +%% returning `true' is necessary, but not sufficient condition that +%% the key satisfies the filter. +-spec bin_checkmask(filter(), binary()) -> boolean(). +bin_checkmask(#filter{size = Size, bitmask = Bitmask, bitfilter = Bitfilter}, Key) -> + case Key of + <> -> + Int band Bitmask =:= Bitfilter; + _ -> + false + end. + +%%================================================================================ +%% Internal functions +%%================================================================================ + +%% Note: this function operates in bitsource basis, scanning it from 0 +%% to NDim (i.e. from the least significant bits to the most +%% significant bits) +ratchet_scan(_Ranges, NDim, _Key, NDim, Pivot, 0) -> + %% We've reached the end: + Pivot; +ratchet_scan(_Ranges, NDim, _Key, NDim, _Pivot, 1) -> + %% We've reached the end, but key is still not large enough: + overflow; +ratchet_scan(Ranges, NDim, Key, I, Pivot0, Carry) -> + #filter_scan_action{offset = Offset, size = Size, min = Min, max = Max} = array:get(I, Ranges), + %% Extract I-th element of the vector from the original key: + Elem = ((Key bsr Offset) band ones(Size)) + Carry, + if + Elem < Min -> + %% I-th coordinate is less than the specified minimum. + %% + %% We reset this coordinate to the minimum value. It means + %% we incremented this bitposition, the less significant + %% bits have to be reset to their respective minimum + %% values: + Pivot = {I + 1, 0}, + ratchet_scan(Ranges, NDim, Key, I + 1, Pivot, 0); + Elem > Max -> + %% I-th coordinate is larger than the specified + %% minimum. We can only fix this problem by incrementing + %% the next coordinate (i.e. more significant bits). + %% + %% We reset this coordinate to the minimum value, and + %% increment the next coordinate (by setting `Carry' to + %% 1). + Pivot = {I + 1, 1}, + ratchet_scan(Ranges, NDim, Key, I + 1, Pivot, 1); + true -> + %% Coordinate is within range: + ratchet_scan(Ranges, NDim, Key, I + 1, Pivot0, 0) + end. + +%% Note: this function operates in bitsource basis, scanning it from +%% NDim to 0. It applies the transformation specified by +%% `ratchet_scan'. +ratchet_do(_Ranges, _Key, I, _Pivot, _Increment) when I < 0 -> + 0; +ratchet_do(Ranges, Key, I, Pivot, Increment) -> + #filter_scan_action{offset = Offset, size = Size, min = Min} = array:get(I, Ranges), + Mask = ones(Offset + Size) bxor ones(Offset), + Elem = + if + I > Pivot -> + Mask band Key; + I =:= Pivot -> + (Mask band Key) + (Increment bsl Offset); + true -> + Min bsl Offset + end, + %% erlang:display( + %% {ratchet_do, I, integer_to_list(Key, 16), integer_to_list(Mask, 2), + %% integer_to_list(Elem, 16)} + %% ), + Elem bor ratchet_do(Ranges, Key, I - 1, Pivot, Increment). + +-spec make_bitfilter(keymapper(), [{non_neg_integer(), non_neg_integer()}]) -> + {non_neg_integer(), non_neg_integer()}. +make_bitfilter(Keymapper = #keymapper{dim_sizeof = DimSizeof}, Ranges) -> + L = lists:zipwith( + fun + ({N, N}, Bits) -> + %% For strict equality we can employ bitmask: + {ones(Bits), N}; + (_, _) -> + {0, 0} + end, + Ranges, + DimSizeof + ), + {Bitmask, Bitfilter} = lists:unzip(L), + {vector_to_key(Keymapper, Bitmask), vector_to_key(Keymapper, Bitfilter)}. + +%% Transform constraints into a list of closed intervals that the +%% vector elements should lie in. +constraints_to_ranges(#keymapper{dim_sizeof = DimSizeof}, Filter) -> + lists:zipwith( + fun(Constraint, Bitsize) -> + Max = ones(Bitsize), + case Constraint of + any -> + {0, Max}; + {'=', infinity} -> + {Max, Max}; + {'=', Val} when Val =< Max -> + {Val, Val}; + {'>=', Val} when Val =< Max -> + {Val, Max}; + {A, '..', B} when A =< Max, B =< Max -> + {A, B} + end + end, + Filter, + DimSizeof + ). + +-spec fold_bitsources(fun((_DstOffset :: non_neg_integer(), bitsource(), Acc) -> Acc), Acc, [ + bitsource() +]) -> {bitsize(), Acc}. +fold_bitsources(Fun, InitAcc, Bitsources) -> + lists:foldl( + fun(Bitsource = {_Dim, _Offset, Size}, {DstOffset, Acc0}) -> + Acc = Fun(DstOffset, Bitsource, Acc0), + {DstOffset + Size, Acc} + end, + {0, InitAcc}, + Bitsources + ). + +do_vector_to_key([], [], _Coord, [], Acc) -> + Acc; +do_vector_to_key([], [NewActions | Scanner], _Coord, [NewCoord | Vector], Acc) -> + do_vector_to_key(NewActions, Scanner, NewCoord, Vector, Acc); +do_vector_to_key([Action | Actions], Scanner, Coord, Vector, Acc0) -> + Acc = Acc0 bor extract(Coord, Action), + do_vector_to_key(Actions, Scanner, Coord, Vector, Acc). + +-spec extract(_Source :: scalar(), scan_action()) -> integer(). +extract(Src, #scan_action{src_bitmask = SrcBitmask, src_offset = SrcOffset, dst_offset = DstOffset}) -> + ((Src bsr SrcOffset) band SrcBitmask) bsl DstOffset. + +%% extract^-1 +-spec extract_inv(_Dest :: scalar(), scan_action()) -> integer(). +extract_inv(Dest, #scan_action{ + src_bitmask = SrcBitmask, src_offset = SrcOffset, dst_offset = DestOffset +}) -> + ((Dest bsr DestOffset) band SrcBitmask) bsl SrcOffset. + +ones(Bits) -> + 1 bsl Bits - 1. + +%%================================================================================ +%% Unit tests +%%================================================================================ + +-ifdef(TEST). + +make_keymapper0_test() -> + Schema = [], + ?assertEqual( + #keymapper{ + schema = Schema, + scanner = [], + size = 0, + dim_sizeof = [] + }, + make_keymapper(Schema) + ). + +make_keymapper1_test() -> + Schema = [{1, 0, 3}, {2, 0, 5}], + ?assertEqual( + #keymapper{ + schema = Schema, + scanner = [ + [#scan_action{src_bitmask = 2#111, src_offset = 0, dst_offset = 0}], + [#scan_action{src_bitmask = 2#11111, src_offset = 0, dst_offset = 3}] + ], + size = 8, + dim_sizeof = [3, 5] + }, + make_keymapper(Schema) + ). + +make_keymapper2_test() -> + Schema = [{1, 0, 3}, {2, 0, 5}, {1, 3, 5}], + ?assertEqual( + #keymapper{ + schema = Schema, + scanner = [ + [ + #scan_action{src_bitmask = 2#11111, src_offset = 3, dst_offset = 8}, + #scan_action{src_bitmask = 2#111, src_offset = 0, dst_offset = 0} + ], + [#scan_action{src_bitmask = 2#11111, src_offset = 0, dst_offset = 3}] + ], + size = 13, + dim_sizeof = [8, 5] + }, + make_keymapper(Schema) + ). + +vector_to_key0_test() -> + Schema = [], + Vector = [], + ?assertEqual(0, vec2key(Schema, Vector)). + +vector_to_key1_test() -> + Schema = [{1, 0, 8}], + ?assertEqual(16#ff, vec2key(Schema, [16#ff])), + ?assertEqual(16#1a, vec2key(Schema, [16#1a])), + ?assertEqual(16#ff, vec2key(Schema, [16#aaff])). + +%% Test handling of source offset: +vector_to_key2_test() -> + Schema = [{1, 8, 8}], + ?assertEqual(0, vec2key(Schema, [16#ff])), + ?assertEqual(16#1a, vec2key(Schema, [16#1aff])), + ?assertEqual(16#aa, vec2key(Schema, [16#11aaff])). + +%% Basic test of 2D vector: +vector_to_key3_test() -> + Schema = [{1, 0, 8}, {2, 0, 8}], + ?assertEqual(16#aaff, vec2key(Schema, [16#ff, 16#aa])), + ?assertEqual(16#2211, vec2key(Schema, [16#aa11, 16#bb22])). + +%% Advanced test with 2D vector: +vector_to_key4_test() -> + Schema = [{1, 0, 8}, {2, 0, 8}, {1, 8, 8}, {2, 16, 8}], + ?assertEqual(16#bb112211, vec2key(Schema, [16#aa1111, 16#bb2222])). + +%% Test with binaries: +vector_to_key_bin_test() -> + Schema = [{1, 0, 8 * 4}, {2, 0, 8 * 5}, {3, 0, 8 * 5}], + Keymapper = make_keymapper(lists:reverse(Schema)), + ?assertMatch( + <<"wellhelloworld">>, bin_vector_to_key(Keymapper, [<<"well">>, <<"hello">>, <<"world">>]) + ). + +key_to_vector0_test() -> + Schema = [], + key2vec(Schema, []). + +key_to_vector1_test() -> + Schema = [{1, 0, 8}, {2, 0, 8}], + key2vec(Schema, [1, 1]), + key2vec(Schema, [255, 255]), + key2vec(Schema, [255, 1]), + key2vec(Schema, [0, 1]), + key2vec(Schema, [255, 0]). + +key_to_vector2_test() -> + Schema = [{1, 0, 3}, {2, 0, 8}, {1, 3, 5}], + key2vec(Schema, [1, 1]), + key2vec(Schema, [255, 255]), + key2vec(Schema, [255, 1]), + key2vec(Schema, [0, 1]), + key2vec(Schema, [255, 0]). + +make_bitmask0_test() -> + Keymapper = make_keymapper([]), + ?assertMatch({0, 0}, mkbmask(Keymapper, [])). + +make_bitmask1_test() -> + Keymapper = make_keymapper([{1, 0, 8}]), + ?assertEqual({0, 0}, mkbmask(Keymapper, [any])), + ?assertEqual({16#ff, 1}, mkbmask(Keymapper, [{'=', 1}])), + ?assertEqual({16#ff, 255}, mkbmask(Keymapper, [{'=', 255}])), + ?assertEqual({0, 0}, mkbmask(Keymapper, [{'>=', 0}])), + ?assertEqual({0, 0}, mkbmask(Keymapper, [{'>=', 1}])), + ?assertEqual({0, 0}, mkbmask(Keymapper, [{'>=', 16#f}])). + +make_bitmask2_test() -> + Keymapper = make_keymapper([{1, 0, 3}, {2, 0, 4}, {3, 0, 2}]), + ?assertEqual({2#00_0000_000, 2#00_0000_000}, mkbmask(Keymapper, [any, any, any])), + ?assertEqual({2#11_0000_000, 2#00_0000_000}, mkbmask(Keymapper, [any, any, {'=', 0}])), + ?assertEqual({2#00_1111_000, 2#00_0000_000}, mkbmask(Keymapper, [any, {'=', 0}, any])), + ?assertEqual({2#00_0000_111, 2#00_0000_000}, mkbmask(Keymapper, [{'=', 0}, any, any])). + +make_bitmask3_test() -> + %% Key format of type |TimeOffset|Topic|Epoch|: + Keymapper = make_keymapper([{1, 0, 8}, {2, 0, 8}, {1, 8, 8}]), + ?assertEqual({2#00000000_00000000_00000000, 16#00_00_00}, mkbmask(Keymapper, [any, any])), + ?assertEqual( + {2#11111111_11111111_11111111, 16#aa_cc_bb}, + mkbmask(Keymapper, [{'=', 16#aabb}, {'=', 16#cc}]) + ), + ?assertEqual( + {2#00000000_11111111_00000000, 16#00_bb_00}, mkbmask(Keymapper, [{'>=', 255}, {'=', 16#bb}]) + ). + +make_filter_test() -> + KeyMapper = make_keymapper([]), + Filter = [], + ?assertMatch(#filter{size = 0, bitmask = 0, bitfilter = 0}, make_filter(KeyMapper, Filter)). + +ratchet1_test() -> + Bitsources = [{1, 0, 8}], + M = make_keymapper(Bitsources), + F = make_filter(M, [any]), + #filter{bitsource_ranges = Rarr} = F, + ?assertMatch( + [ + #filter_scan_action{ + offset = 0, + size = 8, + min = 0, + max = 16#ff + } + ], + array:to_list(Rarr) + ), + ?assertEqual(0, ratchet(F, 0)), + ?assertEqual(16#fa, ratchet(F, 16#fa)), + ?assertEqual(16#ff, ratchet(F, 16#ff)), + ?assertEqual(overflow, ratchet(F, 16#100)). + +%% erlfmt-ignore +ratchet2_test() -> + Bitsources = [{1, 0, 8}, %% Static topic index + {2, 8, 8}, %% Epoch + {3, 0, 8}, %% Varying topic hash + {2, 0, 8}], %% Timestamp offset + M = make_keymapper(lists:reverse(Bitsources)), + F1 = make_filter(M, [{'=', 16#aa}, any, {'=', 16#cc}]), + ?assertEqual(16#aa00cc00, ratchet(F1, 0)), + ?assertEqual(16#aa01cc00, ratchet(F1, 16#aa00cd00)), + ?assertEqual(16#aa01cc11, ratchet(F1, 16#aa01cc11)), + ?assertEqual(16#aa11cc00, ratchet(F1, 16#aa10cd00)), + ?assertEqual(16#aa11cc00, ratchet(F1, 16#aa10dc11)), + ?assertEqual(overflow, ratchet(F1, 16#ab000000)), + F2 = make_filter(M, [{'=', 16#aa}, {'>=', 16#dddd}, {'=', 16#cc}]), + %% TODO: note that it's `16#aaddcc00` instead of + %% `16#aaddccdd'. That is because currently ratchet function + %% doesn't take LSBs of an '>=' interval if it has a hole in the + %% middle (see `make_filter/2'). This only adds extra keys to the + %% very first interval, so it's not deemed a huge problem. + ?assertEqual(16#aaddcc00, ratchet(F2, 0)), + ?assertEqual(16#aa_de_cc_00, ratchet(F2, 16#aa_dd_cd_11)). + +%% erlfmt-ignore +ratchet3_test_() -> + EpochBits = 4, + Bitsources = [{1, 0, 2}, %% Static topic index + {2, EpochBits, 4}, %% Epoch + {3, 0, 2}, %% Varying topic hash + {2, 0, EpochBits}], %% Timestamp offset + Keymapper = make_keymapper(lists:reverse(Bitsources)), + Filter1 = make_filter(Keymapper, [{'=', 2#10}, any, {'=', 2#01}]), + Filter2 = make_filter(Keymapper, [{'=', 2#01}, any, any]), + Filter3 = make_filter(Keymapper, [{'=', 2#01}, {'>=', 16#aa}, any]), + {timeout, 15, + [?_assert(test_iterate(Filter1, 0)), + ?_assert(test_iterate(Filter2, 0)), + %% Not starting from 0 here for simplicity, since the beginning + %% of a >= interval can't be properly checked with a bitmask: + ?_assert(test_iterate(Filter3, ratchet(Filter3, 1))) + ]}. + +%% Note: this function iterates through the full range of keys, so its +%% complexity grows _exponentially_ with the total size of the +%% keymapper. +test_iterate(_Filter, overflow) -> + true; +test_iterate(Filter, Key0) -> + Key = ratchet(Filter, Key0 + 1), + ?assert(ratchet_prop(Filter, Key0, Key)), + test_iterate(Filter, Key). + +ratchet_prop(#filter{bitfilter = Bitfilter, bitmask = Bitmask, size = Size}, Key0, Key) -> + %% Validate basic properties of the generated key. It must be + %% greater than the old key, and match the bitmask: + ?assert(Key =:= overflow orelse (Key band Bitmask =:= Bitfilter)), + ?assert(Key > Key0, {Key, '>=', Key0}), + IMax = ones(Size), + %% Iterate through all keys between `Key0 + 1' and `Key' and + %% validate that none of them match the bitmask. Ultimately, it + %% means that `ratchet' function doesn't skip over any valid keys: + CheckGaps = fun + F(I) when I >= Key; I > IMax -> + true; + F(I) -> + ?assertNot( + I band Bitmask =:= Bitfilter, + {found_gap, Key0, I, Key} + ), + F(I + 1) + end, + CheckGaps(Key0 + 1). + +mkbmask(Keymapper, Filter0) -> + Filter = constraints_to_ranges(Keymapper, Filter0), + make_bitfilter(Keymapper, Filter). + +key2vec(Schema, Vector) -> + Keymapper = make_keymapper(Schema), + Key = vector_to_key(Keymapper, Vector), + ?assertEqual(Vector, key_to_vector(Keymapper, Key)). + +vec2key(Schema, Vector) -> + vector_to_key(make_keymapper(Schema), Vector). + +-endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_lts.erl new file mode 100644 index 000000000..d06854fd0 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_lts.erl @@ -0,0 +1,619 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ds_lts). + +%% API: +-export([ + trie_create/1, trie_create/0, trie_restore/2, topic_key/3, match_topics/2, lookup_topic_key/2 +]). + +%% Debug: +-export([trie_next/3, trie_insert/3, dump_to_dot/2]). + +-export_type([options/0, static_key/0, trie/0]). + +-include_lib("stdlib/include/ms_transform.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-elvis([{elvis_style, variable_naming_convention, disable}]). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +%% End Of Topic +-define(EOT, []). +-define(PLUS, '+'). + +-type edge() :: binary() | ?EOT | ?PLUS. + +%% Fixed size binary +-type static_key() :: non_neg_integer(). + +-define(PREFIX, prefix). +-type state() :: static_key() | ?PREFIX. + +-type varying() :: [binary() | ?PLUS]. + +-type msg_storage_key() :: {static_key(), varying()}. + +-type threshold_fun() :: fun((non_neg_integer()) -> non_neg_integer()). + +-type persist_callback() :: fun((_Key, _Val) -> ok). + +-type options() :: + #{ + persist_callback => persist_callback(), + static_key_size => pos_integer() + }. + +-record(trie, { + persist :: persist_callback(), + static_key_size :: pos_integer(), + trie :: ets:tid(), + stats :: ets:tid() +}). + +-opaque trie() :: #trie{}. + +-record(trans, { + key :: {state(), edge()}, + next :: state() +}). + +%%================================================================================ +%% API funcions +%%================================================================================ + +%% @doc Create an empty trie +-spec trie_create(options()) -> trie(). +trie_create(UserOpts) -> + Defaults = #{ + persist_callback => fun(_, _) -> ok end, + static_key_size => 8 + }, + #{ + persist_callback := Persist, + static_key_size := StaticKeySize + } = maps:merge(Defaults, UserOpts), + Trie = ets:new(trie, [{keypos, #trans.key}, set, public]), + Stats = ets:new(stats, [{keypos, 1}, set, public]), + #trie{ + persist = Persist, + static_key_size = StaticKeySize, + trie = Trie, + stats = Stats + }. + +-spec trie_create() -> trie(). +trie_create() -> + trie_create(#{}). + +%% @doc Restore trie from a dump +-spec trie_restore(options(), [{_Key, _Val}]) -> trie(). +trie_restore(Options, Dump) -> + Trie = trie_create(Options), + lists:foreach( + fun({{StateFrom, Token}, StateTo}) -> + trie_insert(Trie, StateFrom, Token, StateTo) + end, + Dump + ), + Trie. + +%% @doc Lookup the topic key. Create a new one, if not found. +-spec topic_key(trie(), threshold_fun(), [binary()]) -> msg_storage_key(). +topic_key(Trie, ThresholdFun, Tokens) -> + do_topic_key(Trie, ThresholdFun, 0, ?PREFIX, Tokens, []). + +%% @doc Return an exisiting topic key if it exists. +-spec lookup_topic_key(trie(), [binary()]) -> {ok, msg_storage_key()} | undefined. +lookup_topic_key(Trie, Tokens) -> + do_lookup_topic_key(Trie, ?PREFIX, Tokens, []). + +%% @doc Return list of keys of topics that match a given topic filter +-spec match_topics(trie(), [binary() | '+' | '#']) -> + [msg_storage_key()]. +match_topics(Trie, TopicFilter) -> + do_match_topics(Trie, ?PREFIX, [], TopicFilter). + +%% @doc Dump trie to graphviz format for debugging +-spec dump_to_dot(trie(), file:filename()) -> ok. +dump_to_dot(#trie{trie = Trie, stats = Stats}, Filename) -> + L = ets:tab2list(Trie), + {Nodes0, Edges} = + lists:foldl( + fun(#trans{key = {From, Label}, next = To}, {AccN, AccEdge}) -> + Edge = {From, To, Label}, + {[From, To] ++ AccN, [Edge | AccEdge]} + end, + {[], []}, + L + ), + Nodes = + lists:map( + fun(Node) -> + case ets:lookup(Stats, Node) of + [{_, NChildren}] -> ok; + [] -> NChildren = 0 + end, + {Node, NChildren} + end, + lists:usort(Nodes0) + ), + {ok, FD} = file:open(Filename, [write]), + Print = fun + (?PREFIX) -> "prefix"; + (NodeId) -> integer_to_binary(NodeId, 16) + end, + io:format(FD, "digraph {~n", []), + lists:foreach( + fun({Node, NChildren}) -> + Id = Print(Node), + io:format(FD, " \"~s\" [label=\"~s : ~p\"];~n", [Id, Id, NChildren]) + end, + Nodes + ), + lists:foreach( + fun({From, To, Label}) -> + io:format(FD, " \"~s\" -> \"~s\" [label=\"~s\"];~n", [Print(From), Print(To), Label]) + end, + Edges + ), + io:format(FD, "}~n", []), + file:close(FD). + +%%================================================================================ +%% Internal exports +%%================================================================================ + +-spec trie_next(trie(), state(), binary() | ?EOT) -> {Wildcard, state()} | undefined when + Wildcard :: boolean(). +trie_next(#trie{trie = Trie}, State, ?EOT) -> + case ets:lookup(Trie, {State, ?EOT}) of + [#trans{next = Next}] -> {false, Next}; + [] -> undefined + end; +trie_next(#trie{trie = Trie}, State, Token) -> + case ets:lookup(Trie, {State, Token}) of + [#trans{next = Next}] -> + {false, Next}; + [] -> + case ets:lookup(Trie, {State, ?PLUS}) of + [#trans{next = Next}] -> {true, Next}; + [] -> undefined + end + end. + +-spec trie_insert(trie(), state(), edge()) -> {Updated, state()} when + NChildren :: non_neg_integer(), + Updated :: false | NChildren. +trie_insert(Trie, State, Token) -> + trie_insert(Trie, State, Token, get_id_for_key(Trie, State, Token)). + +%%================================================================================ +%% Internal functions +%%================================================================================ + +-spec trie_insert(trie(), state(), edge(), state()) -> {Updated, state()} when + NChildren :: non_neg_integer(), + Updated :: false | NChildren. +trie_insert(#trie{trie = Trie, stats = Stats, persist = Persist}, State, Token, NewState) -> + Key = {State, Token}, + Rec = #trans{ + key = Key, + next = NewState + }, + case ets:insert_new(Trie, Rec) of + true -> + ok = Persist(Key, NewState), + Inc = + case Token of + ?EOT -> 0; + ?PLUS -> 0; + _ -> 1 + end, + NChildren = ets:update_counter(Stats, State, {2, Inc}, {State, 0}), + {NChildren, NewState}; + false -> + [#trans{next = NextState}] = ets:lookup(Trie, Key), + {false, NextState} + end. + +-spec get_id_for_key(trie(), state(), edge()) -> static_key(). +get_id_for_key(#trie{static_key_size = Size}, _State, _Token) -> + %% Requirements for the return value: + %% + %% It should be globally unique for the `{State, Token}` pair. Other + %% than that, there's no requirements. The return value doesn't even + %% have to be deterministic, since the states are saved in the trie. + %% + %% The generated value becomes the ID of the topic in the durable + %% storage. Its size should be relatively small to reduce the + %% overhead of storing messages. + %% + %% If we want to impress computer science crowd, sorry, I mean to + %% minimize storage requirements, we can even employ Huffman coding + %% based on the frequency of messages. + <> = crypto:strong_rand_bytes(Size), + Int. + +%% erlfmt-ignore +-spec do_match_topics(trie(), state(), [binary() | '+'], [binary() | '+' | '#']) -> + list(). +do_match_topics(Trie, State, Varying, []) -> + case trie_next(Trie, State, ?EOT) of + {false, Static} -> [{Static, lists:reverse(Varying)}]; + undefined -> [] + end; +do_match_topics(Trie, State, Varying, ['#']) -> + Emanating = emanating(Trie, State, ?PLUS), + lists:flatmap( + fun + ({?EOT, Static}) -> + [{Static, lists:reverse(Varying)}]; + ({?PLUS, NextState}) -> + do_match_topics(Trie, NextState, [?PLUS | Varying], ['#']); + ({_, NextState}) -> + do_match_topics(Trie, NextState, Varying, ['#']) + end, + Emanating + ); +do_match_topics(Trie, State, Varying, [Level | Rest]) -> + Emanating = emanating(Trie, State, Level), + lists:flatmap( + fun + ({?EOT, _NextState}) -> + []; + ({?PLUS, NextState}) -> + do_match_topics(Trie, NextState, [Level | Varying], Rest); + ({_, NextState}) -> + do_match_topics(Trie, NextState, Varying, Rest) + end, + Emanating + ). + +-spec do_lookup_topic_key(trie(), state(), [binary()], [binary()]) -> + {ok, msg_storage_key()} | undefined. +do_lookup_topic_key(Trie, State, [], Varying) -> + case trie_next(Trie, State, ?EOT) of + {false, Static} -> + {ok, {Static, lists:reverse(Varying)}}; + undefined -> + undefined + end; +do_lookup_topic_key(Trie, State, [Tok | Rest], Varying) -> + case trie_next(Trie, State, Tok) of + {true, NextState} -> + do_lookup_topic_key(Trie, NextState, Rest, [Tok | Varying]); + {false, NextState} -> + do_lookup_topic_key(Trie, NextState, Rest, Varying); + undefined -> + undefined + end. + +do_topic_key(Trie, _, _, State, [], Varying) -> + %% We reached the end of topic. Assert: Trie node that corresponds + %% to EOT cannot be a wildcard. + {_, false, Static} = trie_next_(Trie, State, ?EOT), + {Static, lists:reverse(Varying)}; +do_topic_key(Trie, ThresholdFun, Depth, State, [Tok | Rest], Varying0) -> + % TODO: it's not necessary to call it every time. + Threshold = ThresholdFun(Depth), + Varying = + case trie_next_(Trie, State, Tok) of + {NChildren, _, NextState} when is_integer(NChildren), NChildren >= Threshold -> + %% Number of children for the trie node reached the + %% threshold, we need to insert wildcard here. + {_, _WildcardState} = trie_insert(Trie, State, ?PLUS), + Varying0; + {_, false, NextState} -> + Varying0; + {_, true, NextState} -> + %% This topic level is marked as wildcard in the trie, + %% we need to add it to the varying part of the key: + [Tok | Varying0] + end, + do_topic_key(Trie, ThresholdFun, Depth + 1, NextState, Rest, Varying). + +%% @doc Has side effects! Inserts missing elements +-spec trie_next_(trie(), state(), binary() | ?EOT) -> {New, Wildcard, state()} when + New :: false | non_neg_integer(), + Wildcard :: boolean(). +trie_next_(Trie, State, Token) -> + case trie_next(Trie, State, Token) of + {Wildcard, NextState} -> + {false, Wildcard, NextState}; + undefined -> + {Updated, NextState} = trie_insert(Trie, State, Token), + {Updated, false, NextState} + end. + +%% @doc Return all edges emanating from a node: +%% erlfmt-ignore +-spec emanating(trie(), state(), edge()) -> [{edge(), state()}]. +emanating(#trie{trie = Tab}, State, ?PLUS) -> + ets:select( + Tab, + ets:fun2ms( + fun(#trans{key = {S, Edge}, next = Next}) when S == State -> + {Edge, Next} + end + ) + ); +emanating(#trie{trie = Tab}, State, ?EOT) -> + case ets:lookup(Tab, {State, ?EOT}) of + [#trans{next = Next}] -> [{?EOT, Next}]; + [] -> [] + end; +emanating(#trie{trie = Tab}, State, Bin) when is_binary(Bin) -> + [ + {Edge, Next} + || #trans{key = {_, Edge}, next = Next} <- + ets:lookup(Tab, {State, ?PLUS}) ++ + ets:lookup(Tab, {State, Bin}) + ]. + +%%================================================================================ +%% Tests +%%================================================================================ + +-ifdef(TEST). + +trie_basic_test() -> + T = trie_create(), + ?assertMatch(undefined, trie_next(T, ?PREFIX, <<"foo">>)), + {1, S1} = trie_insert(T, ?PREFIX, <<"foo">>), + ?assertMatch({false, S1}, trie_insert(T, ?PREFIX, <<"foo">>)), + ?assertMatch({false, S1}, trie_next(T, ?PREFIX, <<"foo">>)), + + ?assertMatch(undefined, trie_next(T, ?PREFIX, <<"bar">>)), + {2, S2} = trie_insert(T, ?PREFIX, <<"bar">>), + ?assertMatch({false, S2}, trie_insert(T, ?PREFIX, <<"bar">>)), + + ?assertMatch(undefined, trie_next(T, S1, <<"foo">>)), + ?assertMatch(undefined, trie_next(T, S1, <<"bar">>)), + {1, S11} = trie_insert(T, S1, <<"foo">>), + {2, S12} = trie_insert(T, S1, <<"bar">>), + ?assertMatch({false, S11}, trie_next(T, S1, <<"foo">>)), + ?assertMatch({false, S12}, trie_next(T, S1, <<"bar">>)), + + ?assertMatch(undefined, trie_next(T, S11, <<"bar">>)), + {1, S111} = trie_insert(T, S11, <<"bar">>), + ?assertMatch({false, S111}, trie_next(T, S11, <<"bar">>)). + +lookup_key_test() -> + T = trie_create(), + {_, S1} = trie_insert(T, ?PREFIX, <<"foo">>), + {_, S11} = trie_insert(T, S1, <<"foo">>), + %% Topics don't match until we insert ?EOT: + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"foo">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"foo">>, <<"foo">>]) + ), + {_, S10} = trie_insert(T, S1, ?EOT), + {_, S110} = trie_insert(T, S11, ?EOT), + ?assertMatch( + {ok, {S10, []}}, + lookup_topic_key(T, [<<"foo">>]) + ), + ?assertMatch( + {ok, {S110, []}}, + lookup_topic_key(T, [<<"foo">>, <<"foo">>]) + ), + %% The rest of keys still don't match: + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"bar">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"bar">>, <<"foo">>]) + ). + +wildcard_lookup_test() -> + T = trie_create(), + {1, S1} = trie_insert(T, ?PREFIX, <<"foo">>), + %% Plus doesn't increase the number of children + {0, S11} = trie_insert(T, S1, ?PLUS), + {1, S111} = trie_insert(T, S11, <<"foo">>), + %% ?EOT doesn't increase the number of children + {0, S1110} = trie_insert(T, S111, ?EOT), + ?assertMatch( + {ok, {S1110, [<<"bar">>]}}, + lookup_topic_key(T, [<<"foo">>, <<"bar">>, <<"foo">>]) + ), + ?assertMatch( + {ok, {S1110, [<<"quux">>]}}, + lookup_topic_key(T, [<<"foo">>, <<"quux">>, <<"foo">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"foo">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"foo">>, <<"bar">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"foo">>, <<"bar">>, <<"bar">>]) + ), + ?assertMatch( + undefined, + lookup_topic_key(T, [<<"bar">>, <<"foo">>, <<"foo">>]) + ), + {_, S10} = trie_insert(T, S1, ?EOT), + ?assertMatch( + {ok, {S10, []}}, + lookup_topic_key(T, [<<"foo">>]) + ). + +%% erlfmt-ignore +topic_key_test() -> + T = trie_create(), + try + Threshold = 4, + ThresholdFun = fun(0) -> 1000; + (_) -> Threshold + end, + %% Test that bottom layer threshold is high: + lists:foreach( + fun(I) -> + {_, []} = test_key(T, ThresholdFun, [I, 99999, 999999, 99999]) + end, + lists:seq(1, 10)), + %% Test adding children on the 2nd level: + lists:foreach( + fun(I) -> + case test_key(T, ThresholdFun, [1, I, 1]) of + {_, []} -> + ?assert(I < Threshold, {I, '<', Threshold}), + ok; + {_, [Var]} -> + ?assert(I >= Threshold, {I, '>=', Threshold}), + ?assertEqual(Var, integer_to_binary(I)) + end + end, + lists:seq(1, 100)), + %% This doesn't affect 2nd level with a different prefix: + ?assertMatch({_, []}, test_key(T, ThresholdFun, [2, 1, 1])), + ?assertMatch({_, []}, test_key(T, ThresholdFun, [2, 10, 1])), + %% This didn't retroactively change the indexes that were + %% created prior to reaching the threshold: + ?assertMatch({_, []}, test_key(T, ThresholdFun, [1, 1, 1])), + ?assertMatch({_, []}, test_key(T, ThresholdFun, [1, 2, 1])), + %% Now create another level of +: + lists:foreach( + fun(I) -> + case test_key(T, ThresholdFun, [1, 42, 1, I, 42]) of + {_, [<<"42">>]} when I =< Threshold -> %% TODO: off by 1 error + ok; + {_, [<<"42">>, Var]} -> + ?assertEqual(Var, integer_to_binary(I)); + Ret -> + error({Ret, I}) + end + end, + lists:seq(1, 100)) + after + dump_to_dot(T, filename:join("_build", atom_to_list(?FUNCTION_NAME) ++ ".dot")) + end. + +%% erlfmt-ignore +topic_match_test() -> + T = trie_create(), + try + Threshold = 2, + ThresholdFun = fun(0) -> 1000; + (_) -> Threshold + end, + {S1, []} = test_key(T, ThresholdFun, [1]), + {S11, []} = test_key(T, ThresholdFun, [1, 1]), + {S12, []} = test_key(T, ThresholdFun, [1, 2]), + {S111, []} = test_key(T, ThresholdFun, [1, 1, 1]), + %% Match concrete topics: + assert_match_topics(T, [1], [{S1, []}]), + assert_match_topics(T, [1, 1], [{S11, []}]), + assert_match_topics(T, [1, 1, 1], [{S111, []}]), + %% Match topics with +: + assert_match_topics(T, [1, '+'], [{S11, []}, {S12, []}]), + assert_match_topics(T, [1, '+', 1], [{S111, []}]), + %% Match topics with #: + assert_match_topics(T, [1, '#'], + [{S1, []}, + {S11, []}, {S12, []}, + {S111, []}]), + assert_match_topics(T, [1, 1, '#'], + [{S11, []}, + {S111, []}]), + %% Now add learned wildcards: + {S21, []} = test_key(T, ThresholdFun, [2, 1]), + {S22, []} = test_key(T, ThresholdFun, [2, 2]), + {S2_, [<<"3">>]} = test_key(T, ThresholdFun, [2, 3]), + {S2_11, [<<"3">>]} = test_key(T, ThresholdFun, [2, 3, 1, 1]), + {S2_12, [<<"4">>]} = test_key(T, ThresholdFun, [2, 4, 1, 2]), + {S2_1_, [<<"3">>, <<"3">>]} = test_key(T, ThresholdFun, [2, 3, 1, 3]), + %% %% Check matching: + assert_match_topics(T, [2, 2], + [{S22, []}, {S2_, [<<"2">>]}]), + assert_match_topics(T, [2, '+'], + [{S22, []}, {S21, []}, {S2_, ['+']}]), + assert_match_topics(T, [2, '#'], + [{S21, []}, {S22, []}, + {S2_, ['+']}, + {S2_11, ['+']}, {S2_12, ['+']}, {S2_1_, ['+', '+']}]), + ok + after + dump_to_dot(T, filename:join("_build", atom_to_list(?FUNCTION_NAME) ++ ".dot")) + end. + +-define(keys_history, topic_key_history). + +%% erlfmt-ignore +assert_match_topics(Trie, Filter0, Expected) -> + Filter = lists:map(fun(I) when is_integer(I) -> integer_to_binary(I); + (I) -> I + end, + Filter0), + Matched = match_topics(Trie, Filter), + ?assertMatch( #{missing := [], unexpected := []} + , #{ missing => Expected -- Matched + , unexpected => Matched -- Expected + } + , Filter + ). + +%% erlfmt-ignore +test_key(Trie, Threshold, Topic0) -> + Topic = [integer_to_binary(I) || I <- Topic0], + Ret = topic_key(Trie, Threshold, Topic), + %% Test idempotency: + Ret1 = topic_key(Trie, Threshold, Topic), + ?assertEqual(Ret, Ret1, Topic), + %% Add new key to the history: + case get(?keys_history) of + undefined -> OldHistory = #{}; + OldHistory -> ok + end, + %% Test that the generated keys are always unique for the topic: + History = maps:update_with( + Ret, + fun(Old) -> + case Old =:= Topic of + true -> Old; + false -> error(#{ '$msg' => "Duplicate key!" + , key => Ret + , old_topic => Old + , new_topic => Topic + }) + end + end, + Topic, + OldHistory), + put(?keys_history, History), + {ok, Ret} = lookup_topic_key(Trie, Topic), + Ret. + +-endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl b/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl deleted file mode 100644 index 7b141b202..000000000 --- a/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl +++ /dev/null @@ -1,742 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ds_message_storage_bitmask). - -%%================================================================================ -%% @doc Description of the schema -%% -%% Let us assume that `T' is a topic and `t' is time. These are the two -%% dimensions used to index messages. They can be viewed as -%% "coordinates" of an MQTT message in a 2D space. -%% -%% Oftentimes, when wildcard subscription is used, keys must be -%% scanned in both dimensions simultaneously. -%% -%% Rocksdb allows to iterate over sorted keys very fast. This means we -%% need to map our two-dimentional keys to a single index that is -%% sorted in a way that helps to iterate over both time and topic -%% without having to do a lot of random seeks. -%% -%% == Mapping of 2D keys to rocksdb keys == -%% -%% We use "zigzag" pattern to store messages, where rocksdb key is -%% composed like like this: -%% -%% |ttttt|TTTTTTTTT|tttt| -%% ^ ^ ^ -%% | | | -%% +-------+ | +---------+ -%% | | | -%% most significant topic hash least significant -%% bits of timestamp bits of timestamp -%% (a.k.a epoch) (a.k.a time offset) -%% -%% Topic hash is level-aware: each topic level is hashed separately -%% and the resulting hashes are bitwise-concatentated. This allows us -%% to map topics to fixed-length bitstrings while keeping some degree -%% of information about the hierarchy. -%% -%% Next important concept is what we call "epoch". Duration of the -%% epoch is determined by maximum time offset. Epoch is calculated by -%% shifting bits of the timestamp right. -%% -%% The resulting index is a space-filling curve that looks like -%% this in the topic-time 2D space: -%% -%% T ^ ---->------ |---->------ |---->------ -%% | --/ / --/ / --/ -%% | -<-/ | -<-/ | -<-/ -%% | -/ | -/ | -/ -%% | ---->------ | ---->------ | ---->------ -%% | --/ / --/ / --/ -%% | ---/ | ---/ | ---/ -%% | -/ ^ -/ ^ -/ -%% | ---->------ | ---->------ | ---->------ -%% | --/ / --/ / --/ -%% | -<-/ | -<-/ | -<-/ -%% | -/ | -/ | -/ -%% | ---->------| ---->------| ----------> -%% | -%% -+------------+-----------------------------> t -%% epoch -%% -%% This structure allows to quickly seek to a the first message that -%% was recorded in a certain epoch in a certain topic or a -%% group of topics matching filter like `foo/bar/#`. -%% -%% Due to its structure, for each pair of rocksdb keys K1 and K2, such -%% that K1 > K2 and topic(K1) = topic(K2), timestamp(K1) > -%% timestamp(K2). -%% That is, replay doesn't reorder messages published in each -%% individual topic. -%% -%% This property doesn't hold between different topics, but it's not deemed -%% a problem right now. -%% -%%================================================================================ - --behaviour(emqx_ds_storage_layer). - -%% API: --export([create_new/3, open/5]). --export([make_keymapper/1]). - --export([store/5]). --export([delete/4]). --export([make_iterator/2]). --export([make_iterator/3]). --export([next/1]). - --export([preserve_iterator/1]). --export([restore_iterator/3]). --export([refresh_iterator/1]). - -%% Debug/troubleshooting: -%% Keymappers --export([ - keymapper_info/1, - compute_bitstring/3, - compute_topic_bitmask/2, - compute_time_bitmask/1, - hash/2 -]). - -%% Keyspace filters --export([ - make_keyspace_filter/2, - compute_initial_seek/1, - compute_next_seek/2, - compute_time_seek/3, - compute_topic_seek/4 -]). - --export_type([db/0, iterator/0, schema/0]). - --export_type([options/0]). --export_type([iteration_options/0]). - --compile( - {inline, [ - bitwise_concat/3, - ones/1, - successor/1, - topic_hash_matches/3, - time_matches/3 - ]} -). - -%%================================================================================ -%% Type declarations -%%================================================================================ - --type topic() :: emqx_ds:topic(). --type topic_filter() :: emqx_ds:topic_filter(). --type time() :: emqx_ds:time(). - -%% Number of bits --type bits() :: non_neg_integer(). - -%% Key of a RocksDB record. --type key() :: binary(). - -%% Distribution of entropy among topic levels. -%% Example: [4, 8, 16] means that level 1 gets 4 bits, level 2 gets 8 bits, -%% and _rest of levels_ (if any) get 16 bits. --type bits_per_level() :: [bits(), ...]. - --type options() :: #{ - %% Number of bits in a message timestamp. - timestamp_bits := bits(), - %% Number of bits in a key allocated to each level in a message topic. - topic_bits_per_level := bits_per_level(), - %% Maximum granularity of iteration over time. - epoch := time(), - - iteration => iteration_options(), - - cf_options => emqx_ds_storage_layer:db_cf_options() -}. - --type iteration_options() :: #{ - %% Request periodic iterator refresh. - %% This might be helpful during replays taking a lot of time (e.g. tens of seconds). - %% Note that `{every, 1000}` means 1000 _operations_ with the iterator which is not - %% the same as 1000 replayed messages. - iterator_refresh => {every, _NumOperations :: pos_integer()} -}. - -%% Persistent configuration of the generation, it is used to create db -%% record when the database is reopened --record(schema, {keymapper :: keymapper()}). - --opaque schema() :: #schema{}. - --record(db, { - shard :: emqx_ds:shard(), - handle :: rocksdb:db_handle(), - cf :: rocksdb:cf_handle(), - keymapper :: keymapper(), - write_options = [{sync, true}] :: emqx_ds_storage_layer:db_write_options(), - read_options = [] :: emqx_ds_storage_layer:db_read_options() -}). - --record(it, { - handle :: rocksdb:itr_handle(), - filter :: keyspace_filter(), - cursor :: binary() | undefined, - next_action :: {seek, binary()} | next, - refresh_counter :: {non_neg_integer(), pos_integer()} | undefined -}). - --record(filter, { - keymapper :: keymapper(), - topic_filter :: topic_filter(), - start_time :: integer(), - hash_bitfilter :: integer(), - hash_bitmask :: integer(), - time_bitfilter :: integer(), - time_bitmask :: integer() -}). - -% NOTE -% Keymapper decides how to map messages into RocksDB column family keyspace. --record(keymapper, { - source :: [bitsource(), ...], - bitsize :: bits(), - epoch :: non_neg_integer() -}). - --type bitsource() :: - %% Consume `_Size` bits from timestamp starting at `_Offset`th bit. - %% TODO consistency - {timestamp, _Offset :: bits(), _Size :: bits()} - %% Consume next topic level (either one or all of them) and compute `_Size` bits-wide hash. - | {hash, level | levels, _Size :: bits()}. - --opaque db() :: #db{}. --opaque iterator() :: #it{}. --type keymapper() :: #keymapper{}. --type keyspace_filter() :: #filter{}. - -%%================================================================================ -%% API funcions -%%================================================================================ - -%% Create a new column family for the generation and a serializable representation of the schema --spec create_new(rocksdb:db_handle(), emqx_ds_storage_layer:gen_id(), options()) -> - {schema(), emqx_ds_storage_layer:cf_refs()}. -create_new(DBHandle, GenId, Options) -> - CFName = data_cf(GenId), - CFOptions = maps:get(cf_options, Options, []), - {ok, CFHandle} = rocksdb:create_column_family(DBHandle, CFName, CFOptions), - Schema = #schema{keymapper = make_keymapper(Options)}, - {Schema, [{CFName, CFHandle}]}. - -%% Reopen the database --spec open( - emqx_ds:shard(), - rocksdb:db_handle(), - emqx_ds_storage_layer:gen_id(), - emqx_ds_storage_layer:cf_refs(), - schema() -) -> - db(). -open(Shard, DBHandle, GenId, CFs, #schema{keymapper = Keymapper}) -> - {value, {_, CFHandle}} = lists:keysearch(data_cf(GenId), 1, CFs), - #db{ - shard = Shard, - handle = DBHandle, - cf = CFHandle, - keymapper = Keymapper - }. - --spec make_keymapper(options()) -> keymapper(). -make_keymapper(#{ - timestamp_bits := TimestampBits, - topic_bits_per_level := BitsPerLevel, - epoch := MaxEpoch -}) -> - TimestampLSBs = min(TimestampBits, floor(math:log2(MaxEpoch))), - TimestampMSBs = TimestampBits - TimestampLSBs, - NLevels = length(BitsPerLevel), - {LevelBits, [TailLevelsBits]} = lists:split(NLevels - 1, BitsPerLevel), - Source = lists:flatten([ - [{timestamp, TimestampLSBs, TimestampMSBs} || TimestampMSBs > 0], - [{hash, level, Bits} || Bits <- LevelBits], - {hash, levels, TailLevelsBits}, - [{timestamp, 0, TimestampLSBs} || TimestampLSBs > 0] - ]), - #keymapper{ - source = Source, - bitsize = lists:sum([S || {_, _, S} <- Source]), - epoch = 1 bsl TimestampLSBs - }. - --spec store(db(), emqx_guid:guid(), emqx_ds:time(), topic(), binary()) -> - ok | {error, _TODO}. -store(DB = #db{handle = DBHandle, cf = CFHandle}, MessageID, PublishedAt, Topic, MessagePayload) -> - Key = make_message_key(Topic, PublishedAt, MessageID, DB#db.keymapper), - Value = make_message_value(Topic, MessagePayload), - rocksdb:put(DBHandle, CFHandle, Key, Value, DB#db.write_options). - --spec delete(db(), emqx_guid:guid(), emqx_ds:time(), topic()) -> - ok | {error, _TODO}. -delete(DB = #db{handle = DBHandle, cf = CFHandle}, MessageID, PublishedAt, Topic) -> - Key = make_message_key(Topic, PublishedAt, MessageID, DB#db.keymapper), - rocksdb:delete(DBHandle, CFHandle, Key, DB#db.write_options). - --spec make_iterator(db(), emqx_ds:replay()) -> - {ok, iterator()} | {error, _TODO}. -make_iterator(DB, Replay) -> - {Keyspace, _ShardId} = DB#db.shard, - Options = emqx_ds_conf:iteration_options(Keyspace), - make_iterator(DB, Replay, Options). - --spec make_iterator(db(), emqx_ds:replay(), iteration_options()) -> - % {error, invalid_start_time}? might just start from the beginning of time - % and call it a day: client violated the contract anyway. - {ok, iterator()} | {error, _TODO}. -make_iterator(DB = #db{handle = DBHandle, cf = CFHandle}, Replay, Options) -> - case rocksdb:iterator(DBHandle, CFHandle, DB#db.read_options) of - {ok, ITHandle} -> - Filter = make_keyspace_filter(Replay, DB#db.keymapper), - InitialSeek = combine(compute_initial_seek(Filter), <<>>, DB#db.keymapper), - RefreshCounter = make_refresh_counter(maps:get(iterator_refresh, Options, undefined)), - {ok, #it{ - handle = ITHandle, - filter = Filter, - next_action = {seek, InitialSeek}, - refresh_counter = RefreshCounter - }}; - Err -> - Err - end. - --spec next(iterator()) -> {value, binary(), iterator()} | none | {error, closed}. -next(It0 = #it{filter = #filter{keymapper = Keymapper}}) -> - It = maybe_refresh_iterator(It0), - case rocksdb:iterator_move(It#it.handle, It#it.next_action) of - % spec says `{ok, Key}` is also possible but the implementation says it's not - {ok, Key, Value} -> - % Preserve last seen key in the iterator so it could be restored / refreshed later. - ItNext = It#it{cursor = Key}, - Bitstring = extract(Key, Keymapper), - case match_next(Bitstring, Value, It#it.filter) of - {_Topic, Payload} -> - {value, Payload, ItNext#it{next_action = next}}; - next -> - next(ItNext#it{next_action = next}); - NextBitstring when is_integer(NextBitstring) -> - NextSeek = combine(NextBitstring, <<>>, Keymapper), - next(ItNext#it{next_action = {seek, NextSeek}}); - none -> - stop_iteration(ItNext) - end; - {error, invalid_iterator} -> - stop_iteration(It); - {error, iterator_closed} -> - {error, closed} - end. - --spec preserve_iterator(iterator()) -> binary(). -preserve_iterator(#it{cursor = Cursor}) -> - State = #{ - v => 1, - cursor => Cursor - }, - term_to_binary(State). - --spec restore_iterator(db(), emqx_ds:replay(), binary()) -> - {ok, iterator()} | {error, _TODO}. -restore_iterator(DB, Replay, Serial) when is_binary(Serial) -> - State = binary_to_term(Serial), - restore_iterator(DB, Replay, State); -restore_iterator(DB, Replay, #{ - v := 1, - cursor := Cursor -}) -> - case make_iterator(DB, Replay) of - {ok, It} when Cursor == undefined -> - % Iterator was preserved right after it has been made. - {ok, It}; - {ok, It} -> - % Iterator was preserved mid-replay, seek right past the last seen key. - {ok, It#it{cursor = Cursor, next_action = {seek, successor(Cursor)}}}; - Err -> - Err - end. - --spec refresh_iterator(iterator()) -> iterator(). -refresh_iterator(It = #it{handle = Handle, cursor = Cursor, next_action = Action}) -> - case rocksdb:iterator_refresh(Handle) of - ok when Action =:= next -> - % Now the underlying iterator is invalid, need to seek instead. - It#it{next_action = {seek, successor(Cursor)}}; - ok -> - % Now the underlying iterator is invalid, but will seek soon anyway. - It; - {error, _} -> - % Implementation could in theory return an {error, ...} tuple. - % Supposedly our best bet is to ignore it. - % TODO logging? - It - end. - -%%================================================================================ -%% Internal exports -%%================================================================================ - --spec keymapper_info(keymapper()) -> - #{source := [bitsource()], bitsize := bits(), epoch := time()}. -keymapper_info(#keymapper{source = Source, bitsize = Bitsize, epoch = Epoch}) -> - #{source => Source, bitsize => Bitsize, epoch => Epoch}. - -make_message_key(Topic, PublishedAt, MessageID, Keymapper) -> - combine(compute_bitstring(Topic, PublishedAt, Keymapper), MessageID, Keymapper). - -make_message_value(Topic, MessagePayload) -> - term_to_binary({Topic, MessagePayload}). - -unwrap_message_value(Binary) -> - binary_to_term(Binary). - --spec combine(_Bitstring :: integer(), emqx_guid:guid() | <<>>, keymapper()) -> - key(). -combine(Bitstring, MessageID, #keymapper{bitsize = Size}) -> - <>. - --spec extract(key(), keymapper()) -> - _Bitstring :: integer(). -extract(Key, #keymapper{bitsize = Size}) -> - <> = Key, - Bitstring. - --spec compute_bitstring(topic_filter(), time(), keymapper()) -> integer(). -compute_bitstring(TopicFilter, Timestamp, #keymapper{source = Source}) -> - compute_bitstring(TopicFilter, Timestamp, Source, 0). - --spec compute_topic_bitmask(topic_filter(), keymapper()) -> integer(). -compute_topic_bitmask(TopicFilter, #keymapper{source = Source}) -> - compute_topic_bitmask(TopicFilter, Source, 0). - --spec compute_time_bitmask(keymapper()) -> integer(). -compute_time_bitmask(#keymapper{source = Source}) -> - compute_time_bitmask(Source, 0). - --spec hash(term(), bits()) -> integer(). -hash(Input, Bits) -> - % at most 32 bits - erlang:phash2(Input, 1 bsl Bits). - --spec make_keyspace_filter(emqx_ds:replay(), keymapper()) -> keyspace_filter(). -make_keyspace_filter({TopicFilter, StartTime}, Keymapper) -> - Bitstring = compute_bitstring(TopicFilter, StartTime, Keymapper), - HashBitmask = compute_topic_bitmask(TopicFilter, Keymapper), - TimeBitmask = compute_time_bitmask(Keymapper), - HashBitfilter = Bitstring band HashBitmask, - TimeBitfilter = Bitstring band TimeBitmask, - #filter{ - keymapper = Keymapper, - topic_filter = TopicFilter, - start_time = StartTime, - hash_bitfilter = HashBitfilter, - hash_bitmask = HashBitmask, - time_bitfilter = TimeBitfilter, - time_bitmask = TimeBitmask - }. - --spec compute_initial_seek(keyspace_filter()) -> integer(). -compute_initial_seek(#filter{hash_bitfilter = HashBitfilter, time_bitfilter = TimeBitfilter}) -> - % Should be the same as `compute_initial_seek(0, Filter)`. - HashBitfilter bor TimeBitfilter. - --spec compute_next_seek(integer(), keyspace_filter()) -> integer(). -compute_next_seek( - Bitstring, - Filter = #filter{ - hash_bitfilter = HashBitfilter, - hash_bitmask = HashBitmask, - time_bitfilter = TimeBitfilter, - time_bitmask = TimeBitmask - } -) -> - HashMatches = topic_hash_matches(Bitstring, HashBitfilter, HashBitmask), - TimeMatches = time_matches(Bitstring, TimeBitfilter, TimeBitmask), - compute_next_seek(HashMatches, TimeMatches, Bitstring, Filter). - -%%================================================================================ -%% Internal functions -%%================================================================================ - -compute_bitstring(Topic, Timestamp, [{timestamp, Offset, Size} | Rest], Acc) -> - I = (Timestamp bsr Offset) band ones(Size), - compute_bitstring(Topic, Timestamp, Rest, bitwise_concat(Acc, I, Size)); -compute_bitstring([], Timestamp, [{hash, level, Size} | Rest], Acc) -> - I = hash(<<"/">>, Size), - compute_bitstring([], Timestamp, Rest, bitwise_concat(Acc, I, Size)); -compute_bitstring([Level | Tail], Timestamp, [{hash, level, Size} | Rest], Acc) -> - I = hash(Level, Size), - compute_bitstring(Tail, Timestamp, Rest, bitwise_concat(Acc, I, Size)); -compute_bitstring(Tail, Timestamp, [{hash, levels, Size} | Rest], Acc) -> - I = hash(Tail, Size), - compute_bitstring(Tail, Timestamp, Rest, bitwise_concat(Acc, I, Size)); -compute_bitstring(_, _, [], Acc) -> - Acc. - -compute_topic_bitmask(Filter, [{timestamp, _, Size} | Rest], Acc) -> - compute_topic_bitmask(Filter, Rest, bitwise_concat(Acc, 0, Size)); -compute_topic_bitmask(['#'], [{hash, _, Size} | Rest], Acc) -> - compute_topic_bitmask(['#'], Rest, bitwise_concat(Acc, 0, Size)); -compute_topic_bitmask(['+' | Tail], [{hash, _, Size} | Rest], Acc) -> - compute_topic_bitmask(Tail, Rest, bitwise_concat(Acc, 0, Size)); -compute_topic_bitmask([], [{hash, level, Size} | Rest], Acc) -> - compute_topic_bitmask([], Rest, bitwise_concat(Acc, ones(Size), Size)); -compute_topic_bitmask([_ | Tail], [{hash, level, Size} | Rest], Acc) -> - compute_topic_bitmask(Tail, Rest, bitwise_concat(Acc, ones(Size), Size)); -compute_topic_bitmask(Tail, [{hash, levels, Size} | Rest], Acc) -> - Mask = - case lists:member('+', Tail) orelse lists:member('#', Tail) of - true -> 0; - false -> ones(Size) - end, - compute_topic_bitmask([], Rest, bitwise_concat(Acc, Mask, Size)); -compute_topic_bitmask(_, [], Acc) -> - Acc. - -compute_time_bitmask([{timestamp, _, Size} | Rest], Acc) -> - compute_time_bitmask(Rest, bitwise_concat(Acc, ones(Size), Size)); -compute_time_bitmask([{hash, _, Size} | Rest], Acc) -> - compute_time_bitmask(Rest, bitwise_concat(Acc, 0, Size)); -compute_time_bitmask([], Acc) -> - Acc. - -bitwise_concat(Acc, Item, ItemSize) -> - (Acc bsl ItemSize) bor Item. - -ones(Bits) -> - 1 bsl Bits - 1. - --spec successor(key()) -> key(). -successor(Key) -> - <>. - -%% |123|345|678| -%% foo bar baz - -%% |123|000|678| - |123|fff|678| - -%% foo + baz - -%% |fff|000|fff| - -%% |123|000|678| - -%% |123|056|678| & |fff|000|fff| = |123|000|678|. - -match_next( - Bitstring, - Value, - Filter = #filter{ - topic_filter = TopicFilter, - hash_bitfilter = HashBitfilter, - hash_bitmask = HashBitmask, - time_bitfilter = TimeBitfilter, - time_bitmask = TimeBitmask - } -) -> - HashMatches = topic_hash_matches(Bitstring, HashBitfilter, HashBitmask), - TimeMatches = time_matches(Bitstring, TimeBitfilter, TimeBitmask), - case HashMatches and TimeMatches of - true -> - Message = {Topic, _Payload} = unwrap_message_value(Value), - case emqx_topic:match(Topic, TopicFilter) of - true -> - Message; - false -> - next - end; - false -> - compute_next_seek(HashMatches, TimeMatches, Bitstring, Filter) - end. - -%% `Bitstring` is out of the hash space defined by `HashBitfilter`. -compute_next_seek( - _HashMatches = false, - _TimeMatches, - Bitstring, - Filter = #filter{ - keymapper = Keymapper, - hash_bitfilter = HashBitfilter, - hash_bitmask = HashBitmask, - time_bitfilter = TimeBitfilter, - time_bitmask = TimeBitmask - } -) -> - NextBitstring = compute_topic_seek(Bitstring, HashBitfilter, HashBitmask, Keymapper), - case NextBitstring of - none -> - none; - _ -> - TimeMatches = time_matches(NextBitstring, TimeBitfilter, TimeBitmask), - compute_next_seek(true, TimeMatches, NextBitstring, Filter) - end; -%% `Bitstring` is out of the time range defined by `TimeBitfilter`. -compute_next_seek( - _HashMatches = true, - _TimeMatches = false, - Bitstring, - #filter{ - time_bitfilter = TimeBitfilter, - time_bitmask = TimeBitmask - } -) -> - compute_time_seek(Bitstring, TimeBitfilter, TimeBitmask); -compute_next_seek(true, true, Bitstring, _It) -> - Bitstring. - -topic_hash_matches(Bitstring, HashBitfilter, HashBitmask) -> - (Bitstring band HashBitmask) == HashBitfilter. - -time_matches(Bitstring, TimeBitfilter, TimeBitmask) -> - (Bitstring band TimeBitmask) >= TimeBitfilter. - -compute_time_seek(Bitstring, TimeBitfilter, TimeBitmask) -> - % Replace the bits of the timestamp in `Bistring` with bits from `Timebitfilter`. - (Bitstring band (bnot TimeBitmask)) bor TimeBitfilter. - -%% Find the closest bitstring which is: -%% * greater than `Bitstring`, -%% * and falls into the hash space defined by `HashBitfilter`. -%% Note that the result can end up "back" in time and out of the time range. -compute_topic_seek(Bitstring, HashBitfilter, HashBitmask, Keymapper) -> - Sources = Keymapper#keymapper.source, - Size = Keymapper#keymapper.bitsize, - compute_topic_seek(Bitstring, HashBitfilter, HashBitmask, Sources, Size). - -compute_topic_seek(Bitstring, HashBitfilter, HashBitmask, Sources, Size) -> - % NOTE - % We're iterating through `Substring` here, in lockstep with `HashBitfilter` - % and `HashBitmask`, starting from least signigicant bits. Each bitsource in - % `Sources` has a bitsize `S` and, accordingly, gives us a sub-bitstring `S` - % bits long which we interpret as a "digit". There are 2 flavors of those - % "digits": - % * regular digit with 2^S possible values, - % * degenerate digit with exactly 1 possible value U (represented with 0). - % Our goal here is to find a successor of `Bistring` and perform a kind of - % digit-by-digit addition operation with carry propagation. - NextSeek = zipfoldr3( - fun(Source, Substring, Filter, LBitmask, Offset, Acc) -> - case Source of - {hash, _, S} when LBitmask =:= 0 -> - % Regular case - bitwise_add_digit(Substring, Acc, S, Offset); - {hash, _, _} when LBitmask =/= 0, Substring < Filter -> - % Degenerate case, I_digit < U, no overflow. - % Successor is `U bsl Offset` which is equivalent to 0. - 0; - {hash, _, S} when LBitmask =/= 0, Substring > Filter -> - % Degenerate case, I_digit > U, overflow. - % Successor is `(1 bsl Size + U) bsl Offset`. - overflow_digit(S, Offset); - {hash, _, S} when LBitmask =/= 0 -> - % Degenerate case, I_digit = U - % Perform digit addition with I_digit = 0, assuming "digit" has - % 0 bits of information (but is `S` bits long at the same time). - % This will overflow only if the result of previous iteration - % was an overflow. - bitwise_add_digit(0, Acc, 0, S, Offset); - {timestamp, _, S} -> - % Regular case - bitwise_add_digit(Substring, Acc, S, Offset) - end - end, - 0, - Bitstring, - HashBitfilter, - HashBitmask, - Size, - Sources - ), - case NextSeek bsr Size of - _Carry = 0 -> - % Found the successor. - % We need to recover values of those degenerate digits which we - % represented with 0 during digit-by-digit iteration. - NextSeek bor (HashBitfilter band HashBitmask); - _Carry = 1 -> - % We got "carried away" past the range, time to stop iteration. - none - end. - -bitwise_add_digit(Digit, Number, Width, Offset) -> - bitwise_add_digit(Digit, Number, Width, Width, Offset). - -%% Add "digit" (represented with integer `Digit`) to the `Number` assuming -%% this digit starts at `Offset` bits in `Number` and is `Width` bits long. -%% Perform an overflow if the result of addition would not fit into `Bits` -%% bits. -bitwise_add_digit(Digit, Number, Bits, Width, Offset) -> - Sum = (Digit bsl Offset) + Number, - case (Sum bsr Offset) < (1 bsl Bits) of - true -> Sum; - false -> overflow_digit(Width, Offset) - end. - -%% Constuct a number which denotes an overflow of digit that starts at -%% `Offset` bits and is `Width` bits long. -overflow_digit(Width, Offset) -> - (1 bsl Width) bsl Offset. - -%% Iterate through sub-bitstrings of 3 integers in lockstep, starting from least -%% significant bits first. -%% -%% Each integer is assumed to be `Size` bits long. Lengths of sub-bitstring are -%% specified in `Sources` list, in order from most significant bits to least -%% significant. Each iteration calls `FoldFun` with: -%% * bitsource that was used to extract sub-bitstrings, -%% * 3 sub-bitstrings in integer representation, -%% * bit offset into integers, -%% * current accumulator. --spec zipfoldr3(FoldFun, Acc, integer(), integer(), integer(), _Size :: bits(), [bitsource()]) -> - Acc -when - FoldFun :: fun((bitsource(), integer(), integer(), integer(), _Offset :: bits(), Acc) -> Acc). -zipfoldr3(_FoldFun, Acc, _, _, _, 0, []) -> - Acc; -zipfoldr3(FoldFun, Acc, I1, I2, I3, Offset, [Source = {_, _, S} | Rest]) -> - OffsetNext = Offset - S, - AccNext = zipfoldr3(FoldFun, Acc, I1, I2, I3, OffsetNext, Rest), - FoldFun( - Source, - substring(I1, OffsetNext, S), - substring(I2, OffsetNext, S), - substring(I3, OffsetNext, S), - OffsetNext, - AccNext - ). - -substring(I, Offset, Size) -> - (I bsr Offset) band ones(Size). - -%% @doc Generate a column family ID for the MQTT messages --spec data_cf(emqx_ds_storage_layer:gen_id()) -> [char()]. -data_cf(GenId) -> - ?MODULE_STRING ++ integer_to_list(GenId). - -make_refresh_counter({every, N}) when is_integer(N), N > 0 -> - {0, N}; -make_refresh_counter(undefined) -> - undefined. - -maybe_refresh_iterator(It = #it{refresh_counter = {N, N}}) -> - refresh_iterator(It#it{refresh_counter = {0, N}}); -maybe_refresh_iterator(It = #it{refresh_counter = {M, N}}) -> - It#it{refresh_counter = {M + 1, N}}; -maybe_refresh_iterator(It = #it{refresh_counter = undefined}) -> - It. - -stop_iteration(It) -> - ok = rocksdb:iterator_close(It#it.handle), - none. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl new file mode 100644 index 000000000..a06af104d --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl @@ -0,0 +1,245 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Replication layer for DS backends that don't support +%% replication on their own. +-module(emqx_ds_replication_layer). + +-behaviour(emqx_ds). + +-export([ + list_shards/1, + open_db/2, + drop_db/1, + store_batch/3, + get_streams/3, + make_iterator/4, + next/3 +]). + +%% internal exports: +-export([ + do_open_shard_v1/3, + do_drop_shard_v1/2, + do_store_batch_v1/4, + do_get_streams_v1/4, + do_make_iterator_v1/5, + do_next_v1/4 +]). + +-export_type([shard_id/0, stream/0, iterator/0, message_id/0]). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +%% # "Record" integer keys. We use maps with integer keys to avoid persisting and sending +%% records over the wire. + +%% tags: +-define(STREAM, 1). +-define(IT, 2). + +%% keys: +-define(tag, 1). +-define(shard, 2). +-define(enc, 3). + +-type shard_id() :: atom(). + +%% This enapsulates the stream entity from the replication level. +%% +%% TODO: currently the stream is hardwired to only support the +%% internal rocksdb storage. In the future we want to add another +%% implementations for emqx_ds, so this type has to take this into +%% account. +-opaque stream() :: + #{ + ?tag := ?STREAM, + ?shard := emqx_ds_replication_layer:shard_id(), + ?enc := emqx_ds_storage_layer:stream() + }. + +-opaque iterator() :: + #{ + ?tag := ?IT, + ?shard := emqx_ds_replication_layer:shard_id(), + ?enc := emqx_ds_storage_layer:iterator() + }. + +-type message_id() :: emqx_ds_storage_layer:message_id(). + +%%================================================================================ +%% API functions +%%================================================================================ + +-spec list_shards(emqx_ds:db()) -> [shard_id()]. +list_shards(_DB) -> + %% TODO: milestone 5 + list_nodes(). + +-spec open_db(emqx_ds:db(), emqx_ds:create_db_opts()) -> ok | {error, _}. +open_db(DB, Opts) -> + %% TODO: improve error reporting, don't just crash + lists:foreach( + fun(Shard) -> + Node = node_of_shard(DB, Shard), + ok = emqx_ds_proto_v1:open_shard(Node, DB, Shard, Opts) + end, + list_shards(DB) + ). + +-spec drop_db(emqx_ds:db()) -> ok | {error, _}. +drop_db(DB) -> + lists:foreach( + fun(Shard) -> + Node = node_of_shard(DB, Shard), + ok = emqx_ds_proto_v1:drop_shard(Node, DB, Shard) + end, + list_shards(DB) + ). + +-spec store_batch(emqx_ds:db(), [emqx_types:message()], emqx_ds:message_store_opts()) -> + emqx_ds:store_batch_result(). +store_batch(DB, Batch, Opts) -> + %% TODO: Currently we store messages locally. + Shard = node(), + Node = node_of_shard(DB, Shard), + emqx_ds_proto_v1:store_batch(Node, DB, Shard, Batch, Opts). + +-spec get_streams(emqx_ds:db(), emqx_ds:topic_filter(), emqx_ds:time()) -> + [{emqx_ds:stream_rank(), stream()}]. +get_streams(DB, TopicFilter, StartTime) -> + Shards = list_shards(DB), + lists:flatmap( + fun(Shard) -> + Node = node_of_shard(DB, Shard), + Streams = emqx_ds_proto_v1:get_streams(Node, DB, Shard, TopicFilter, StartTime), + lists:map( + fun({RankY, Stream}) -> + RankX = Shard, + Rank = {RankX, RankY}, + {Rank, #{ + ?tag => ?STREAM, + ?shard => Shard, + ?enc => Stream + }} + end, + Streams + ) + end, + Shards + ). + +-spec make_iterator(emqx_ds:db(), stream(), emqx_ds:topic_filter(), emqx_ds:time()) -> + emqx_ds:make_iterator_result(iterator()). +make_iterator(DB, Stream, TopicFilter, StartTime) -> + #{?tag := ?STREAM, ?shard := Shard, ?enc := StorageStream} = Stream, + Node = node_of_shard(DB, Shard), + case emqx_ds_proto_v1:make_iterator(Node, DB, Shard, StorageStream, TopicFilter, StartTime) of + {ok, Iter} -> + {ok, #{?tag => ?IT, ?shard => Shard, ?enc => Iter}}; + Err = {error, _} -> + Err + end. + +-spec next(emqx_ds:db(), iterator(), pos_integer()) -> emqx_ds:next_result(iterator()). +next(DB, Iter0, BatchSize) -> + #{?tag := ?IT, ?shard := Shard, ?enc := StorageIter0} = Iter0, + Node = node_of_shard(DB, Shard), + %% TODO: iterator can contain information that is useful for + %% reconstructing messages sent over the network. For example, + %% when we send messages with the learned topic index, we could + %% send the static part of topic once, and append it to the + %% messages on the receiving node, hence saving some network. + %% + %% This kind of trickery should be probably done here in the + %% replication layer. Or, perhaps, in the logic layer. + case emqx_ds_proto_v1:next(Node, DB, Shard, StorageIter0, BatchSize) of + {ok, StorageIter, Batch} -> + Iter = Iter0#{?enc := StorageIter}, + {ok, Iter, Batch}; + Other -> + Other + end. + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +%%================================================================================ +%% Internal exports (RPC targets) +%%================================================================================ + +-spec do_open_shard_v1( + emqx_ds:db(), emqx_ds_replication_layer:shard_id(), emqx_ds:create_db_opts() +) -> + ok | {error, _}. +do_open_shard_v1(DB, Shard, Opts) -> + emqx_ds_storage_layer:open_shard({DB, Shard}, Opts). + +-spec do_drop_shard_v1(emqx_ds:db(), emqx_ds_replication_layer:shard_id()) -> ok | {error, _}. +do_drop_shard_v1(DB, Shard) -> + emqx_ds_storage_layer:drop_shard({DB, Shard}). + +-spec do_store_batch_v1( + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + [emqx_types:message()], + emqx_ds:message_store_opts() +) -> + emqx_ds:store_batch_result(). +do_store_batch_v1(DB, Shard, Batch, Options) -> + emqx_ds_storage_layer:store_batch({DB, Shard}, Batch, Options). + +-spec do_get_streams_v1( + emqx_ds:db(), emqx_ds_replicationi_layer:shard_id(), emqx_ds:topic_filter(), emqx_ds:time() +) -> + [{integer(), emqx_ds_storage_layer:stream()}]. +do_get_streams_v1(DB, Shard, TopicFilter, StartTime) -> + emqx_ds_storage_layer:get_streams({DB, Shard}, TopicFilter, StartTime). + +-spec do_make_iterator_v1( + emqx_ds:db(), + emqx_ds_storage_layer:shard_id(), + emqx_ds_storage_layer:stream(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _}. +do_make_iterator_v1(DB, Shard, Stream, TopicFilter, StartTime) -> + emqx_ds_storage_layer:make_iterator({DB, Shard}, Stream, TopicFilter, StartTime). + +-spec do_next_v1( + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:iterator(), + pos_integer() +) -> + emqx_ds:next_result(emqx_ds_storage_layer:iterator()). +do_next_v1(DB, Shard, Iter, BatchSize) -> + emqx_ds_storage_layer:next({DB, Shard}, Iter, BatchSize). + +%%================================================================================ +%% Internal functions +%%================================================================================ + +-spec node_of_shard(emqx_ds:db(), shard_id()) -> node(). +node_of_shard(_DB, Node) -> + Node. + +list_nodes() -> + mria:running_nodes(). diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl new file mode 100644 index 000000000..2d4949919 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl @@ -0,0 +1,454 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc A storage layout based on learned topic structure and using +%% bitfield mapping for the varying topic layers. +-module(emqx_ds_storage_bitfield_lts). + +-behaviour(emqx_ds_storage_layer). + +%% API: +-export([]). + +%% behavior callbacks: +-export([create/4, open/5, store_batch/4, get_streams/4, make_iterator/5, next/4]). + +%% internal exports: +-export([format_key/2]). + +-export_type([options/0]). + +-include_lib("emqx_utils/include/emqx_message.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +%% # "Record" integer keys. We use maps with integer keys to avoid persisting and sending +%% records over the wire. + +%% tags: +-define(STREAM, 1). +-define(IT, 2). + +%% keys: +-define(tag, 1). +-define(topic_filter, 2). +-define(start_time, 3). +-define(storage_key, 4). +-define(last_seen_key, 5). + +-type options() :: + #{ + bits_per_wildcard_level => pos_integer(), + topic_index_bytes => pos_integer(), + epoch_bits => non_neg_integer() + }. + +%% Permanent state: +-type schema() :: + #{ + bits_per_wildcard_level := pos_integer(), + topic_index_bytes := pos_integer(), + ts_bits := non_neg_integer(), + ts_offset_bits := non_neg_integer() + }. + +%% Runtime state: +-record(s, { + db :: rocksdb:db_handle(), + data :: rocksdb:cf_handle(), + trie :: emqx_ds_lts:trie(), + keymappers :: array:array(emqx_ds_bitmask_keymapper:keymapper()), + ts_offset :: non_neg_integer() +}). + +-type s() :: #s{}. + +-type stream() :: + #{ + ?tag := ?STREAM, + ?storage_key := emqx_ds_lts:msg_storage_key() + }. + +-type iterator() :: + #{ + ?tag := ?IT, + ?topic_filter := emqx_ds:topic_filter(), + ?start_time := emqx_ds:time(), + ?storage_key := emqx_ds_lts:msg_storage_key(), + ?last_seen_key := binary() + }. + +-define(COUNTER, emqx_ds_storage_bitfield_lts_counter). + +%% Limit on the number of wildcard levels in the learned topic trie: +-define(WILDCARD_LIMIT, 10). + +-include("emqx_ds_bitmask.hrl"). + +%%================================================================================ +%% API funcions +%%================================================================================ + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +-spec create( + emqx_ds_storage_layer:shard_id(), + rocksdb:db_handle(), + emqx_ds_storage_layer:gen_id(), + options() +) -> + {schema(), emqx_ds_storage_layer:cf_refs()}. +create(_ShardId, DBHandle, GenId, Options) -> + %% Get options: + BitsPerTopicLevel = maps:get(bits_per_wildcard_level, Options, 64), + TopicIndexBytes = maps:get(topic_index_bytes, Options, 4), + %% 10 bits -> 1024 ms -> ~1 sec + TSOffsetBits = maps:get(epoch_bits, Options, 10), + %% Create column families: + DataCFName = data_cf(GenId), + TrieCFName = trie_cf(GenId), + {ok, DataCFHandle} = rocksdb:create_column_family(DBHandle, DataCFName, []), + {ok, TrieCFHandle} = rocksdb:create_column_family(DBHandle, TrieCFName, []), + %% Create schema: + Schema = #{ + bits_per_wildcard_level => BitsPerTopicLevel, + topic_index_bytes => TopicIndexBytes, + ts_bits => 64, + ts_offset_bits => TSOffsetBits + }, + {Schema, [{DataCFName, DataCFHandle}, {TrieCFName, TrieCFHandle}]}. + +-spec open( + emqx_ds_storage_layer:shard_id(), + rocksdb:db_handle(), + emqx_ds_storage_layer:gen_id(), + emqx_ds_storage_layer:cf_refs(), + schema() +) -> + s(). +open(_Shard, DBHandle, GenId, CFRefs, Schema) -> + #{ + bits_per_wildcard_level := BitsPerTopicLevel, + topic_index_bytes := TopicIndexBytes, + ts_bits := TSBits, + ts_offset_bits := TSOffsetBits + } = Schema, + {_, DataCF} = lists:keyfind(data_cf(GenId), 1, CFRefs), + {_, TrieCF} = lists:keyfind(trie_cf(GenId), 1, CFRefs), + Trie = restore_trie(TopicIndexBytes, DBHandle, TrieCF), + %% If user's topics have more than learned 10 wildcard levels + %% (more than 2, really), then it's total carnage; learned topic + %% structure won't help. + MaxWildcardLevels = ?WILDCARD_LIMIT, + KeymapperCache = array:from_list( + [ + make_keymapper(TopicIndexBytes, BitsPerTopicLevel, TSBits, TSOffsetBits, N) + || N <- lists:seq(0, MaxWildcardLevels) + ] + ), + #s{ + db = DBHandle, + data = DataCF, + trie = Trie, + keymappers = KeymapperCache, + ts_offset = TSOffsetBits + }. + +-spec store_batch( + emqx_ds_storage_layer:shard_id(), s(), [emqx_types:message()], emqx_ds:message_store_opts() +) -> + emqx_ds:store_batch_result(). +store_batch(_ShardId, S = #s{db = DB, data = Data}, Messages, _Options) -> + lists:foreach( + fun(Msg) -> + {Key, _} = make_key(S, Msg), + Val = serialize(Msg), + rocksdb:put(DB, Data, Key, Val, []) + end, + Messages + ). + +-spec get_streams( + emqx_ds_storage_layer:shard_id(), + s(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> [stream()]. +get_streams(_Shard, #s{trie = Trie}, TopicFilter, _StartTime) -> + Indexes = emqx_ds_lts:match_topics(Trie, TopicFilter), + [#{?tag => ?STREAM, ?storage_key => I} || I <- Indexes]. + +-spec make_iterator( + emqx_ds_storage_layer:shard_id(), + s(), + stream(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> {ok, iterator()}. +make_iterator( + _Shard, _Data, #{?tag := ?STREAM, ?storage_key := StorageKey}, TopicFilter, StartTime +) -> + %% Note: it's a good idea to keep the iterator structure lean, + %% since it can be stored on a remote node that could update its + %% code independently from us. + {ok, #{ + ?tag => ?IT, + ?topic_filter => TopicFilter, + ?start_time => StartTime, + ?storage_key => StorageKey, + ?last_seen_key => <<>> + }}. + +next(_Shard, Schema = #s{ts_offset = TSOffset}, It, BatchSize) -> + %% Compute safe cutoff time. + %% It's the point in time where the last complete epoch ends, so we need to know + %% the current time to compute it. + Now = emqx_message:timestamp_now(), + SafeCutoffTime = (Now bsr TSOffset) bsl TSOffset, + next_until(Schema, It, SafeCutoffTime, BatchSize). + +next_until(_Schema, It = #{?tag := ?IT, ?start_time := StartTime}, SafeCutoffTime, _BatchSize) when + StartTime >= SafeCutoffTime +-> + %% We're in the middle of the current epoch, so we can't yet iterate over it. + %% It would be unsafe otherwise: messages can be stored in the current epoch + %% concurrently with iterating over it. They can end up earlier (in the iteration + %% order) due to the nature of keymapping, potentially causing us to miss them. + {ok, It, []}; +next_until(#s{db = DB, data = CF, keymappers = Keymappers}, It, SafeCutoffTime, BatchSize) -> + #{ + ?tag := ?IT, + ?start_time := StartTime, + ?storage_key := {TopicIndex, Varying} + } = It, + %% Make filter: + Inequations = [ + {'=', TopicIndex}, + {StartTime, '..', SafeCutoffTime - 1}, + %% Unique integer: + any + %% Varying topic levels: + | lists:map( + fun + ('+') -> + any; + (TopicLevel) when is_binary(TopicLevel) -> + {'=', hash_topic_level(TopicLevel)} + end, + Varying + ) + ], + %% Obtain a keymapper for the current number of varying levels. + NVarying = length(Varying), + %% Assert: + NVarying =< ?WILDCARD_LIMIT orelse + error({too_many_varying_topic_levels, NVarying}), + Keymapper = array:get(NVarying, Keymappers), + Filter = + #filter{range_min = LowerBound, range_max = UpperBound} = emqx_ds_bitmask_keymapper:make_filter( + Keymapper, Inequations + ), + {ok, ITHandle} = rocksdb:iterator(DB, CF, [ + {iterate_lower_bound, emqx_ds_bitmask_keymapper:key_to_bitstring(Keymapper, LowerBound)}, + {iterate_upper_bound, emqx_ds_bitmask_keymapper:key_to_bitstring(Keymapper, UpperBound + 1)} + ]), + try + put(?COUNTER, 0), + next_loop(ITHandle, Keymapper, Filter, SafeCutoffTime, It, [], BatchSize) + after + rocksdb:iterator_close(ITHandle), + erase(?COUNTER) + end. + +%%================================================================================ +%% Internal functions +%%================================================================================ + +next_loop(_ITHandle, _KeyMapper, _Filter, _Cutoff, It, Acc, 0) -> + {ok, It, lists:reverse(Acc)}; +next_loop(ITHandle, KeyMapper, Filter, Cutoff, It0, Acc0, N0) -> + inc_counter(), + #{?tag := ?IT, ?last_seen_key := Key0} = It0, + case emqx_ds_bitmask_keymapper:bin_increment(Filter, Key0) of + overflow -> + {ok, It0, lists:reverse(Acc0)}; + Key1 -> + %% assert + true = Key1 > Key0, + case rocksdb:iterator_move(ITHandle, {seek, Key1}) of + {ok, Key, Val} -> + {N, It, Acc} = + traverse_interval(ITHandle, Filter, Cutoff, Key, Val, It0, Acc0, N0), + next_loop(ITHandle, KeyMapper, Filter, Cutoff, It, Acc, N); + {error, invalid_iterator} -> + {ok, It0, lists:reverse(Acc0)} + end + end. + +traverse_interval(ITHandle, Filter, Cutoff, Key, Val, It0, Acc0, N) -> + It = It0#{?last_seen_key := Key}, + case emqx_ds_bitmask_keymapper:bin_checkmask(Filter, Key) of + true -> + Msg = deserialize(Val), + case check_message(Cutoff, It, Msg) of + true -> + Acc = [Msg | Acc0], + traverse_interval(ITHandle, Filter, Cutoff, It, Acc, N - 1); + false -> + traverse_interval(ITHandle, Filter, Cutoff, It, Acc0, N); + overflow -> + {0, It0, Acc0} + end; + false -> + {N, It, Acc0} + end. + +traverse_interval(_ITHandle, _Filter, _Cutoff, It, Acc, 0) -> + {0, It, Acc}; +traverse_interval(ITHandle, Filter, Cutoff, It, Acc, N) -> + inc_counter(), + case rocksdb:iterator_move(ITHandle, next) of + {ok, Key, Val} -> + traverse_interval(ITHandle, Filter, Cutoff, Key, Val, It, Acc, N); + {error, invalid_iterator} -> + {0, It, Acc} + end. + +-spec check_message(emqx_ds:time(), iterator(), emqx_types:message()) -> + true | false | overflow. +check_message( + Cutoff, + _It, + #message{timestamp = Timestamp} +) when Timestamp >= Cutoff -> + %% We hit the current epoch, we can't continue iterating over it yet. + %% It would be unsafe otherwise: messages can be stored in the current epoch + %% concurrently with iterating over it. They can end up earlier (in the iteration + %% order) due to the nature of keymapping, potentially causing us to miss them. + overflow; +check_message( + _Cutoff, + #{?tag := ?IT, ?start_time := StartTime, ?topic_filter := TopicFilter}, + #message{timestamp = Timestamp, topic = Topic} +) when Timestamp >= StartTime -> + emqx_topic:match(emqx_topic:words(Topic), TopicFilter); +check_message(_Cutoff, _It, _Msg) -> + false. + +format_key(KeyMapper, Key) -> + Vec = [integer_to_list(I, 16) || I <- emqx_ds_bitmask_keymapper:key_to_vector(KeyMapper, Key)], + lists:flatten(io_lib:format("~.16B (~s)", [Key, string:join(Vec, ",")])). + +-spec make_key(s(), emqx_types:message()) -> {binary(), [binary()]}. +make_key(#s{keymappers = KeyMappers, trie = Trie}, #message{timestamp = Timestamp, topic = TopicBin}) -> + Tokens = emqx_topic:tokens(TopicBin), + {TopicIndex, Varying} = emqx_ds_lts:topic_key(Trie, fun threshold_fun/1, Tokens), + VaryingHashes = [hash_topic_level(I) || I <- Varying], + KeyMapper = array:get(length(Varying), KeyMappers), + KeyBin = make_key(KeyMapper, TopicIndex, Timestamp, VaryingHashes), + {KeyBin, Varying}. + +-spec make_key(emqx_ds_bitmask_keymapper:keymapper(), emqx_ds_lts:static_key(), emqx_ds:time(), [ + non_neg_integer() +]) -> + binary(). +make_key(KeyMapper, TopicIndex, Timestamp, Varying) -> + UniqueInteger = erlang:unique_integer([monotonic, positive]), + emqx_ds_bitmask_keymapper:key_to_bitstring( + KeyMapper, + emqx_ds_bitmask_keymapper:vector_to_key(KeyMapper, [ + TopicIndex, Timestamp, UniqueInteger | Varying + ]) + ). + +%% TODO: don't hardcode the thresholds +threshold_fun(0) -> + 100; +threshold_fun(_) -> + 20. + +hash_topic_level(TopicLevel) -> + <> = erlang:md5(TopicLevel), + Int. + +serialize(Msg) -> + term_to_binary(Msg). + +deserialize(Blob) -> + binary_to_term(Blob). + +-define(BYTE_SIZE, 8). + +%% erlfmt-ignore +make_keymapper(TopicIndexBytes, BitsPerTopicLevel, TSBits, TSOffsetBits, N) -> + Bitsources = + %% Dimension Offset Bitsize + [{1, 0, TopicIndexBytes * ?BYTE_SIZE}, %% Topic index + {2, TSOffsetBits, TSBits - TSOffsetBits }] ++ %% Timestamp epoch + [{3 + I, 0, BitsPerTopicLevel } %% Varying topic levels + || I <- lists:seq(1, N)] ++ + [{2, 0, TSOffsetBits }, %% Timestamp offset + {3, 0, 64 }], %% Unique integer + Keymapper = emqx_ds_bitmask_keymapper:make_keymapper(lists:reverse(Bitsources)), + %% Assert: + case emqx_ds_bitmask_keymapper:bitsize(Keymapper) rem 8 of + 0 -> + ok; + _ -> + error(#{'$msg' => "Non-even key size", bitsources => Bitsources}) + end, + Keymapper. + +-spec restore_trie(pos_integer(), rocksdb:db_handle(), rocksdb:cf_handle()) -> emqx_ds_lts:trie(). +restore_trie(TopicIndexBytes, DB, CF) -> + PersistCallback = fun(Key, Val) -> + rocksdb:put(DB, CF, term_to_binary(Key), term_to_binary(Val), []) + end, + {ok, IT} = rocksdb:iterator(DB, CF, []), + try + Dump = read_persisted_trie(IT, rocksdb:iterator_move(IT, first)), + TrieOpts = #{persist_callback => PersistCallback, static_key_size => TopicIndexBytes}, + emqx_ds_lts:trie_restore(TrieOpts, Dump) + after + rocksdb:iterator_close(IT) + end. + +read_persisted_trie(IT, {ok, KeyB, ValB}) -> + [ + {binary_to_term(KeyB), binary_to_term(ValB)} + | read_persisted_trie(IT, rocksdb:iterator_move(IT, next)) + ]; +read_persisted_trie(_IT, {error, invalid_iterator}) -> + []. + +inc_counter() -> + N = get(?COUNTER), + put(?COUNTER, N + 1). + +%% @doc Generate a column family ID for the MQTT messages +-spec data_cf(emqx_ds_storage_layer:gen_id()) -> [char()]. +data_cf(GenId) -> + "emqx_ds_storage_bitfield_lts_data" ++ integer_to_list(GenId). + +%% @doc Generate a column family ID for the trie +-spec trie_cf(emqx_ds_storage_layer:gen_id()) -> [char()]. +trie_cf(GenId) -> + "emqx_ds_storage_bitfield_lts_trie" ++ integer_to_list(GenId). diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl index 6137a1ed7..0fe719dbc 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl @@ -1,277 +1,269 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_ds_storage_layer). -behaviour(gen_server). -%% API: --export([start_link/2]). --export([create_generation/3]). +%% Replication layer API: +-export([open_shard/2, drop_shard/1, store_batch/3, get_streams/3, make_iterator/4, next/3]). --export([store/5]). --export([delete/4]). +%% gen_server +-export([start_link/2, init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). --export([make_iterator/2, next/1]). +%% internal exports: +-export([db_dir/1]). --export([ - preserve_iterator/2, - restore_iterator/2, - discard_iterator/2, - ensure_iterator/3, - discard_iterator_prefix/2, - list_iterator_prefix/2, - foldl_iterator_prefix/4 -]). +-export_type([gen_id/0, generation/0, cf_refs/0, stream/0, iterator/0]). -%% behaviour callbacks: --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). - --export_type([cf_refs/0, gen_id/0, options/0, state/0, iterator/0]). --export_type([db_options/0, db_write_options/0, db_read_options/0]). - --compile({inline, [meta_lookup/2]}). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %%================================================================================ %% Type declarations %%================================================================================ --type options() :: #{ - dir => file:filename() -}. +%% # "Record" integer keys. We use maps with integer keys to avoid persisting and sending +%% records over the wire. -%% see rocksdb:db_options() --type db_options() :: proplists:proplist(). -%% see rocksdb:write_options() --type db_write_options() :: proplists:proplist(). -%% see rocksdb:read_options() --type db_read_options() :: proplists:proplist(). +%% tags: +-define(STREAM, 1). +-define(IT, 2). + +%% keys: +-define(tag, 1). +-define(generation, 2). +-define(enc, 3). + +-type prototype() :: + {emqx_ds_storage_reference, emqx_ds_storage_reference:options()} + | {emqx_ds_storage_bitfield_lts, emqx_ds_storage_bitfield_lts:options()}. + +-type shard_id() :: {emqx_ds:db(), emqx_ds_replication_layer:shard_id()}. -type cf_refs() :: [{string(), rocksdb:cf_handle()}]. -%% Message storage generation -%% Keep in mind that instances of this type are persisted in long-term storage. --type generation() :: #{ - %% Module that handles data for the generation +-type gen_id() :: 0..16#ffff. + +%% Note: this might be stored permanently on a remote node. +-opaque stream() :: + #{ + ?tag := ?STREAM, + ?generation := gen_id(), + ?enc := term() + }. + +%% Note: this might be stored permanently on a remote node. +-opaque iterator() :: + #{ + ?tag := ?IT, + ?generation := gen_id(), + ?enc := term() + }. + +%%%% Generation: + +-type generation(Data) :: #{ + %% Module that handles data for the generation: module := module(), - %% Module-specific data defined at generation creation time - data := term(), + %% Module-specific data defined at generation creation time: + data := Data, %% When should this generation become active? %% This generation should only contain messages timestamped no earlier than that. %% The very first generation will have `since` equal 0. - since := emqx_ds:time() + since := emqx_ds:time(), + until := emqx_ds:time() | undefined }. +%% Schema for a generation. Persistent term. +-type generation_schema() :: generation(term()). + +%% Runtime view of generation: +-type generation() :: generation(term()). + +%%%% Shard: + +-type shard(GenData) :: #{ + %% ID of the current generation (where the new data is written): + current_generation := gen_id(), + %% This data is used to create new generation: + prototype := prototype(), + %% Generations: + {generation, gen_id()} => GenData +}. + +%% Shard schema (persistent): +-type shard_schema() :: shard(generation_schema()). + +%% Shard (runtime): +-type shard() :: shard(generation()). + +%%================================================================================ +%% Generation callbacks +%%================================================================================ + +%% Create the new schema given generation id and the options. +%% Create rocksdb column families. +-callback create(shard_id(), rocksdb:db_handle(), gen_id(), _Options) -> + {_Schema, cf_refs()}. + +%% Open the existing schema +-callback open(shard_id(), rocsdb:db_handle(), gen_id(), cf_refs(), _Schema) -> + _Data. + +-callback store_batch(shard_id(), _Data, [emqx_types:message()], emqx_ds:message_store_opts()) -> + emqx_ds:store_batch_result(). + +-callback get_streams(shard_id(), _Data, emqx_ds:topic_filter(), emqx_ds:time()) -> + [_Stream]. + +-callback make_iterator(shard_id(), _Data, _Stream, emqx_ds:topic_filter(), emqx_ds:time()) -> + emqx_ds:make_iterator_result(_Iterator). + +-callback next(shard_id(), _Data, Iter, pos_integer()) -> + {ok, Iter, [emqx_types:message()]} | {error, _}. + +%%================================================================================ +%% API for the replication layer +%%================================================================================ + +-spec open_shard(shard_id(), emqx_ds:builtin_db_opts()) -> ok. +open_shard(Shard, Options) -> + emqx_ds_storage_layer_sup:ensure_shard(Shard, Options). + +-spec drop_shard(shard_id()) -> ok. +drop_shard(Shard) -> + catch emqx_ds_storage_layer_sup:stop_shard(Shard), + ok = rocksdb:destroy(db_dir(Shard), []). + +-spec store_batch(shard_id(), [emqx_types:message()], emqx_ds:message_store_opts()) -> + emqx_ds:store_batch_result(). +store_batch(Shard, Messages, Options) -> + %% We always store messages in the current generation: + GenId = generation_current(Shard), + #{module := Mod, data := GenData} = generation_get(Shard, GenId), + Mod:store_batch(Shard, GenData, Messages, Options). + +-spec get_streams(shard_id(), emqx_ds:topic_filter(), emqx_ds:time()) -> + [{integer(), stream()}]. +get_streams(Shard, TopicFilter, StartTime) -> + Gens = generations_since(Shard, StartTime), + lists:flatmap( + fun(GenId) -> + #{module := Mod, data := GenData} = generation_get(Shard, GenId), + Streams = Mod:get_streams(Shard, GenData, TopicFilter, StartTime), + [ + {GenId, #{ + ?tag => ?STREAM, + ?generation => GenId, + ?enc => Stream + }} + || Stream <- Streams + ] + end, + Gens + ). + +-spec make_iterator(shard_id(), stream(), emqx_ds:topic_filter(), emqx_ds:time()) -> + emqx_ds:make_iterator_result(iterator()). +make_iterator( + Shard, #{?tag := ?STREAM, ?generation := GenId, ?enc := Stream}, TopicFilter, StartTime +) -> + #{module := Mod, data := GenData} = generation_get(Shard, GenId), + case Mod:make_iterator(Shard, GenData, Stream, TopicFilter, StartTime) of + {ok, Iter} -> + {ok, #{ + ?tag => ?IT, + ?generation => GenId, + ?enc => Iter + }}; + {error, _} = Err -> + Err + end. + +-spec next(shard_id(), iterator(), pos_integer()) -> + emqx_ds:next_result(iterator()). +next(Shard, Iter = #{?tag := ?IT, ?generation := GenId, ?enc := GenIter0}, BatchSize) -> + #{module := Mod, data := GenData} = generation_get(Shard, GenId), + Current = generation_current(Shard), + case Mod:next(Shard, GenData, GenIter0, BatchSize) of + {ok, _GenIter, []} when GenId < Current -> + %% This is a past generation. Storage layer won't write + %% any more messages here. The iterator reached the end: + %% the stream has been fully replayed. + {ok, end_of_stream}; + {ok, GenIter, Batch} -> + {ok, Iter#{?enc := GenIter}, Batch}; + Error = {error, _} -> + Error + end. + +%%================================================================================ +%% gen_server for the shard +%%================================================================================ + +-define(REF(ShardId), {via, gproc, {n, l, {?MODULE, ShardId}}}). + +-spec start_link(shard_id(), emqx_ds:builtin_db_opts()) -> + {ok, pid()}. +start_link(Shard = {_, _}, Options) -> + gen_server:start_link(?REF(Shard), ?MODULE, {Shard, Options}, []). + -record(s, { - shard :: emqx_ds:shard(), + shard_id :: emqx_ds:shard_id(), db :: rocksdb:db_handle(), - cf_iterator :: rocksdb:cf_handle(), - cf_generations :: cf_refs() + cf_refs :: cf_refs(), + schema :: shard_schema(), + shard :: shard() }). --record(it, { - shard :: emqx_ds:shard(), - gen :: gen_id(), - replay :: emqx_ds:replay(), - module :: module(), - data :: term() -}). +%% Note: we specify gen_server requests as records to make use of Dialyzer: +-record(call_create_generation, {since :: emqx_ds:time()}). --type gen_id() :: 0..16#ffff. - --opaque state() :: #s{}. --opaque iterator() :: #it{}. - -%% Contents of the default column family: -%% -%% [{<<"genNN">>, #generation{}}, ..., -%% {<<"current">>, GenID}] +-type server_state() :: #s{}. -define(DEFAULT_CF, "default"). -define(DEFAULT_CF_OPTS, []). --define(ITERATOR_CF, "$iterators"). - -%% TODO -%% 1. CuckooTable might be of use here / `OptimizeForPointLookup(...)`. -%% 2. Supposedly might be compressed _very_ effectively. -%% 3. `inplace_update_support`? --define(ITERATOR_CF_OPTS, []). - --define(REF(Keyspace, ShardId), {via, gproc, {n, l, {?MODULE, Keyspace, ShardId}}}). - -%%================================================================================ -%% Callbacks -%%================================================================================ - --callback create_new(rocksdb:db_handle(), gen_id(), _Options :: term()) -> - {_Schema, cf_refs()}. - --callback open( - emqx_ds:shard(), - rocksdb:db_handle(), - gen_id(), - cf_refs(), - _Schema -) -> - term(). - --callback store( - _Schema, - _MessageID :: binary(), - emqx_ds:time(), - emqx_ds:topic(), - _Payload :: binary() -) -> - ok | {error, _}. - --callback delete(_Schema, _MessageID :: binary(), emqx_ds:time(), emqx_ds:topic()) -> - ok | {error, _}. - --callback make_iterator(_Schema, emqx_ds:replay()) -> - {ok, _It} | {error, _}. - --callback restore_iterator(_Schema, emqx_ds:replay(), binary()) -> {ok, _It} | {error, _}. - --callback preserve_iterator(_It) -> term(). - --callback next(It) -> {value, binary(), It} | none | {error, closed}. - -%%================================================================================ -%% API funcions -%%================================================================================ - --spec start_link(emqx_ds:shard(), emqx_ds_storage_layer:options()) -> - {ok, pid()}. -start_link(Shard = {Keyspace, ShardId}, Options) -> - gen_server:start_link(?REF(Keyspace, ShardId), ?MODULE, {Shard, Options}, []). - --spec create_generation( - emqx_ds:shard(), emqx_ds:time(), emqx_ds_conf:backend_config() -) -> - {ok, gen_id()} | {error, nonmonotonic}. -create_generation({Keyspace, ShardId}, Since, Config = {_Module, _Options}) -> - gen_server:call(?REF(Keyspace, ShardId), {create_generation, Since, Config}). - --spec store(emqx_ds:shard(), emqx_guid:guid(), emqx_ds:time(), emqx_ds:topic(), binary()) -> - ok | {error, _}. -store(Shard, GUID, Time, Topic, Msg) -> - {_GenId, #{module := Mod, data := Data}} = meta_lookup_gen(Shard, Time), - Mod:store(Data, GUID, Time, Topic, Msg). - --spec delete(emqx_ds:shard(), emqx_guid:guid(), emqx_ds:time(), emqx_ds:topic()) -> - ok | {error, _}. -delete(Shard, GUID, Time, Topic) -> - {_GenId, #{module := Mod, data := Data}} = meta_lookup_gen(Shard, Time), - Mod:delete(Data, GUID, Time, Topic). - --spec make_iterator(emqx_ds:shard(), emqx_ds:replay()) -> - {ok, iterator()} | {error, _TODO}. -make_iterator(Shard, Replay = {_, StartTime}) -> - {GenId, Gen} = meta_lookup_gen(Shard, StartTime), - open_iterator(Gen, #it{ - shard = Shard, - gen = GenId, - replay = Replay - }). - --spec next(iterator()) -> {value, binary(), iterator()} | none | {error, closed}. -next(It = #it{module = Mod, data = ItData}) -> - case Mod:next(ItData) of - {value, Val, ItDataNext} -> - {value, Val, It#it{data = ItDataNext}}; - {error, _} = Error -> - Error; - none -> - case open_next_iterator(It) of - {ok, ItNext} -> - next(ItNext); - {error, _} = Error -> - Error; - none -> - none - end - end. - --spec preserve_iterator(iterator(), emqx_ds:iterator_id()) -> - ok | {error, _TODO}. -preserve_iterator(It = #it{}, IteratorID) -> - iterator_put_state(IteratorID, It). - --spec restore_iterator(emqx_ds:shard(), emqx_ds:replay_id()) -> - {ok, iterator()} | {error, _TODO}. -restore_iterator(Shard, ReplayID) -> - case iterator_get_state(Shard, ReplayID) of - {ok, Serial} -> - restore_iterator_state(Shard, Serial); - not_found -> - {error, not_found}; - {error, _Reason} = Error -> - Error - end. - --spec ensure_iterator(emqx_ds:shard(), emqx_ds:iterator_id(), emqx_ds:replay()) -> - {ok, iterator()} | {error, _TODO}. -ensure_iterator(Shard, IteratorID, Replay = {_TopicFilter, _StartMS}) -> - case restore_iterator(Shard, IteratorID) of - {ok, It} -> - {ok, It}; - {error, not_found} -> - {ok, It} = make_iterator(Shard, Replay), - ok = emqx_ds_storage_layer:preserve_iterator(It, IteratorID), - {ok, It}; - Error -> - Error - end. - --spec discard_iterator(emqx_ds:shard(), emqx_ds:replay_id()) -> - ok | {error, _TODO}. -discard_iterator(Shard, ReplayID) -> - iterator_delete(Shard, ReplayID). - --spec discard_iterator_prefix(emqx_ds:shard(), binary()) -> - ok | {error, _TODO}. -discard_iterator_prefix(Shard, KeyPrefix) -> - case do_discard_iterator_prefix(Shard, KeyPrefix) of - {ok, _} -> ok; - Error -> Error - end. - --spec list_iterator_prefix( - emqx_ds:shard(), - binary() -) -> {ok, [emqx_ds:iterator_id()]} | {error, _TODO}. -list_iterator_prefix(Shard, KeyPrefix) -> - do_list_iterator_prefix(Shard, KeyPrefix). - --spec foldl_iterator_prefix( - emqx_ds:shard(), - binary(), - fun((_Key :: binary(), _Value :: binary(), Acc) -> Acc), - Acc -) -> {ok, Acc} | {error, _TODO} when - Acc :: term(). -foldl_iterator_prefix(Shard, KeyPrefix, Fn, Acc) -> - do_foldl_iterator_prefix(Shard, KeyPrefix, Fn, Acc). - -%%================================================================================ -%% behaviour callbacks -%%================================================================================ - -init({Shard, Options}) -> +init({ShardId, Options}) -> process_flag(trap_exit, true), - {ok, S0} = open_db(Shard, Options), - S = ensure_current_generation(S0), - ok = populate_metadata(S), + logger:set_process_metadata(#{shard_id => ShardId, domain => [ds, storage_layer, shard]}), + erase_schema_runtime(ShardId), + {ok, DB, CFRefs0} = rocksdb_open(ShardId, Options), + {Schema, CFRefs} = + case get_schema_persistent(DB) of + not_found -> + Prototype = maps:get(storage, Options), + create_new_shard_schema(ShardId, DB, CFRefs0, Prototype); + Scm -> + {Scm, CFRefs0} + end, + Shard = open_shard(ShardId, DB, CFRefs, Schema), + S = #s{ + shard_id = ShardId, + db = DB, + cf_refs = CFRefs, + schema = Schema, + shard = Shard + }, + commit_metadata(S), {ok, S}. -handle_call({create_generation, Since, Config}, _From, S) -> - case create_new_gen(Since, Config, S) of - {ok, GenId, NS} -> - {reply, {ok, GenId}, NS}; - {error, _} = Error -> - {reply, Error, S} - end; +handle_call(#call_create_generation{since = Since}, _From, S0) -> + S = add_generation(S0, Since), + commit_metadata(S), + {reply, ok, S}; handle_call(_Call, _From, S) -> {reply, {error, unknown_call}, S}. @@ -281,346 +273,182 @@ handle_cast(_Cast, S) -> handle_info(_Info, S) -> {noreply, S}. -terminate(_Reason, #s{db = DB, shard = Shard}) -> - meta_erase(Shard), +terminate(_Reason, #s{db = DB, shard_id = ShardId}) -> + erase_schema_runtime(ShardId), ok = rocksdb:close(DB). +%%================================================================================ +%% Internal exports +%%================================================================================ + %%================================================================================ %% Internal functions %%================================================================================ --record(db, {handle :: rocksdb:db_handle(), cf_iterator :: rocksdb:cf_handle()}). +-spec open_shard(shard_id(), rocksdb:db_handle(), cf_refs(), shard_schema()) -> + shard(). +open_shard(ShardId, DB, CFRefs, ShardSchema) -> + %% Transform generation schemas to generation runtime data: + maps:map( + fun + ({generation, GenId}, GenSchema) -> + open_generation(ShardId, DB, CFRefs, GenId, GenSchema); + (_K, Val) -> + Val + end, + ShardSchema + ). --spec populate_metadata(state()) -> ok. -populate_metadata(S = #s{shard = Shard, db = DBHandle, cf_iterator = CFIterator}) -> - ok = meta_put(Shard, db, #db{handle = DBHandle, cf_iterator = CFIterator}), - Current = schema_get_current(DBHandle), - lists:foreach(fun(GenId) -> populate_metadata(GenId, S) end, lists:seq(0, Current)). +-spec add_generation(server_state(), emqx_ds:time()) -> server_state(). +add_generation(S0, Since) -> + #s{shard_id = ShardId, db = DB, schema = Schema0, shard = Shard0, cf_refs = CFRefs0} = S0, + {GenId, Schema, NewCFRefs} = new_generation(ShardId, DB, Schema0, Since), + CFRefs = NewCFRefs ++ CFRefs0, + Key = {generation, GenId}, + Generation = open_generation(ShardId, DB, CFRefs, GenId, maps:get(Key, Schema)), + Shard = Shard0#{Key => Generation}, + S0#s{ + cf_refs = CFRefs, + schema = Schema, + shard = Shard + }. --spec populate_metadata(gen_id(), state()) -> ok. -populate_metadata(GenId, S = #s{shard = Shard, db = DBHandle}) -> - Gen = open_gen(GenId, schema_get_gen(DBHandle, GenId), S), - meta_register_gen(Shard, GenId, Gen). +-spec open_generation(shard_id(), rocksdb:db_handle(), cf_refs(), gen_id(), generation_schema()) -> + generation(). +open_generation(ShardId, DB, CFRefs, GenId, GenSchema) -> + ?tp(debug, ds_open_generation, #{gen_id => GenId, schema => GenSchema}), + #{module := Mod, data := Schema} = GenSchema, + RuntimeData = Mod:open(ShardId, DB, GenId, CFRefs, Schema), + GenSchema#{data => RuntimeData}. --spec ensure_current_generation(state()) -> state(). -ensure_current_generation(S = #s{shard = {Keyspace, _ShardId}, db = DBHandle}) -> - case schema_get_current(DBHandle) of - undefined -> - Config = emqx_ds_conf:keyspace_config(Keyspace), - {ok, _, NS} = create_new_gen(0, Config, S), - NS; - _GenId -> - S - end. - --spec create_new_gen(emqx_ds:time(), emqx_ds_conf:backend_config(), state()) -> - {ok, gen_id(), state()} | {error, nonmonotonic}. -create_new_gen(Since, Config, S = #s{shard = Shard, db = DBHandle}) -> - GenId = get_next_id(meta_get_current(Shard)), - GenId = get_next_id(schema_get_current(DBHandle)), - case is_gen_valid(Shard, GenId, Since) of - ok -> - {ok, Gen, NS} = create_gen(GenId, Since, Config, S), - %% TODO: Transaction? Column family creation can't be transactional, anyway. - ok = schema_put_gen(DBHandle, GenId, Gen), - ok = schema_put_current(DBHandle, GenId), - ok = meta_register_gen(Shard, GenId, open_gen(GenId, Gen, NS)), - {ok, GenId, NS}; - {error, _} = Error -> - Error - end. - --spec create_gen(gen_id(), emqx_ds:time(), emqx_ds_conf:backend_config(), state()) -> - {ok, generation(), state()}. -create_gen(GenId, Since, {Module, Options}, S = #s{db = DBHandle, cf_generations = CFs}) -> - % TODO: Backend implementation should ensure idempotency. - {Schema, NewCFs} = Module:create_new(DBHandle, GenId, Options), - Gen = #{ - module => Module, - data => Schema, - since => Since +-spec create_new_shard_schema(shard_id(), rocksdb:db_handle(), cf_refs(), prototype()) -> + {shard_schema(), cf_refs()}. +create_new_shard_schema(ShardId, DB, CFRefs, Prototype) -> + ?tp(notice, ds_create_new_shard_schema, #{shard => ShardId, prototype => Prototype}), + %% TODO: read prototype from options/config + Schema0 = #{ + current_generation => 0, + prototype => Prototype }, - {ok, Gen, S#s{cf_generations = NewCFs ++ CFs}}. + {_NewGenId, Schema, NewCFRefs} = new_generation(ShardId, DB, Schema0, _Since = 0), + {Schema, NewCFRefs ++ CFRefs}. --spec open_db(emqx_ds:shard(), options()) -> {ok, state()} | {error, _TODO}. -open_db(Shard = {Keyspace, ShardId}, Options) -> - DefaultDir = filename:join([atom_to_binary(Keyspace), ShardId]), - DBDir = unicode:characters_to_list(maps:get(dir, Options, DefaultDir)), +-spec new_generation(shard_id(), rocksdb:db_handle(), shard_schema(), emqx_ds:time()) -> + {gen_id(), shard_schema(), cf_refs()}. +new_generation(ShardId, DB, Schema0, Since) -> + #{current_generation := PrevGenId, prototype := {Mod, ModConf}} = Schema0, + GenId = PrevGenId + 1, + {GenData, NewCFRefs} = Mod:create(ShardId, DB, GenId, ModConf), + GenSchema = #{module => Mod, data => GenData, since => Since, until => undefined}, + Schema = Schema0#{ + current_generation => GenId, + {generation, GenId} => GenSchema + }, + {GenId, Schema, NewCFRefs}. + +%% @doc Commit current state of the server to both rocksdb and the persistent term +-spec commit_metadata(server_state()) -> ok. +commit_metadata(#s{shard_id = ShardId, schema = Schema, shard = Runtime, db = DB}) -> + ok = put_schema_persistent(DB, Schema), + put_schema_runtime(ShardId, Runtime). + +-spec rocksdb_open(shard_id(), emqx_ds:builtin_db_opts()) -> + {ok, rocksdb:db_handle(), cf_refs()} | {error, _TODO}. +rocksdb_open(Shard, Options) -> DBOptions = [ {create_if_missing, true}, {create_missing_column_families, true} - | emqx_ds_conf:db_options(Keyspace) + | maps:get(db_options, Options, []) ], + DBDir = db_dir(Shard), _ = filelib:ensure_dir(DBDir), ExistingCFs = case rocksdb:list_column_families(DBDir, DBOptions) of {ok, CFs} -> - [{Name, []} || Name <- CFs, Name /= ?DEFAULT_CF, Name /= ?ITERATOR_CF]; + [{Name, []} || Name <- CFs, Name /= ?DEFAULT_CF]; % DB is not present. First start {error, {db_open, _}} -> [] end, ColumnFamilies = [ - {?DEFAULT_CF, ?DEFAULT_CF_OPTS}, - {?ITERATOR_CF, ?ITERATOR_CF_OPTS} + {?DEFAULT_CF, ?DEFAULT_CF_OPTS} | ExistingCFs ], case rocksdb:open(DBDir, DBOptions, ColumnFamilies) of - {ok, DBHandle, [_CFDefault, CFIterator | CFRefs]} -> + {ok, DBHandle, [_CFDefault | CFRefs]} -> {CFNames, _} = lists:unzip(ExistingCFs), - {ok, #s{ - shard = Shard, - db = DBHandle, - cf_iterator = CFIterator, - cf_generations = lists:zip(CFNames, CFRefs) - }}; + {ok, DBHandle, lists:zip(CFNames, CFRefs)}; Error -> Error end. --spec open_gen(gen_id(), generation(), state()) -> generation(). -open_gen( - GenId, - Gen = #{module := Mod, data := Data}, - #s{shard = Shard, db = DBHandle, cf_generations = CFs} -) -> - DB = Mod:open(Shard, DBHandle, GenId, CFs, Data), - Gen#{data := DB}. +-spec db_dir(shard_id()) -> file:filename(). +db_dir({DB, ShardId}) -> + filename:join([emqx:data_dir(), atom_to_list(DB), atom_to_list(ShardId)]). --spec open_next_iterator(iterator()) -> {ok, iterator()} | {error, _Reason} | none. -open_next_iterator(It = #it{shard = Shard, gen = GenId}) -> - open_next_iterator(meta_get_gen(Shard, GenId + 1), It#it{gen = GenId + 1}). +%%-------------------------------------------------------------------------------- +%% Schema access +%%-------------------------------------------------------------------------------- -open_next_iterator(undefined, _It) -> - none; -open_next_iterator(Gen = #{}, It) -> - open_iterator(Gen, It). +-spec generation_current(shard_id()) -> gen_id(). +generation_current(Shard) -> + #{current_generation := Current} = get_schema_runtime(Shard), + Current. --spec open_iterator(generation(), iterator()) -> {ok, iterator()} | {error, _Reason}. -open_iterator(#{module := Mod, data := Data}, It = #it{}) -> - case Mod:make_iterator(Data, It#it.replay) of - {ok, ItData} -> - {ok, It#it{module = Mod, data = ItData}}; - Err -> - Err - end. +-spec generation_get(shard_id(), gen_id()) -> generation(). +generation_get(Shard, GenId) -> + #{{generation, GenId} := GenData} = get_schema_runtime(Shard), + GenData. --spec open_restore_iterator(generation(), iterator(), binary()) -> - {ok, iterator()} | {error, _Reason}. -open_restore_iterator(#{module := Mod, data := Data}, It = #it{replay = Replay}, Serial) -> - case Mod:restore_iterator(Data, Replay, Serial) of - {ok, ItData} -> - {ok, It#it{module = Mod, data = ItData}}; - Err -> - Err - end. - -%% - --define(KEY_REPLAY_STATE(IteratorId), <<(IteratorId)/binary, "rs">>). --define(KEY_REPLAY_STATE_PAT(KeyReplayState), begin - <> = (KeyReplayState), - IteratorId -end). - --define(ITERATION_WRITE_OPTS, []). --define(ITERATION_READ_OPTS, []). - -iterator_get_state(Shard, ReplayID) -> - #db{handle = Handle, cf_iterator = CF} = meta_lookup(Shard, db), - rocksdb:get(Handle, CF, ?KEY_REPLAY_STATE(ReplayID), ?ITERATION_READ_OPTS). - -iterator_put_state(ID, It = #it{shard = Shard}) -> - #db{handle = Handle, cf_iterator = CF} = meta_lookup(Shard, db), - Serial = preserve_iterator_state(It), - rocksdb:put(Handle, CF, ?KEY_REPLAY_STATE(ID), Serial, ?ITERATION_WRITE_OPTS). - -iterator_delete(Shard, ID) -> - #db{handle = Handle, cf_iterator = CF} = meta_lookup(Shard, db), - rocksdb:delete(Handle, CF, ?KEY_REPLAY_STATE(ID), ?ITERATION_WRITE_OPTS). - -preserve_iterator_state(#it{ - gen = Gen, - replay = {TopicFilter, StartTime}, - module = Mod, - data = ItData -}) -> - term_to_binary(#{ - v => 1, - gen => Gen, - filter => TopicFilter, - start => StartTime, - st => Mod:preserve_iterator(ItData) - }). - -restore_iterator_state(Shard, Serial) when is_binary(Serial) -> - restore_iterator_state(Shard, binary_to_term(Serial)); -restore_iterator_state( - Shard, - #{ - v := 1, - gen := Gen, - filter := TopicFilter, - start := StartTime, - st := State - } -) -> - It = #it{shard = Shard, gen = Gen, replay = {TopicFilter, StartTime}}, - open_restore_iterator(meta_get_gen(Shard, Gen), It, State). - -do_list_iterator_prefix(Shard, KeyPrefix) -> - Fn = fun(K0, _V, Acc) -> - K = ?KEY_REPLAY_STATE_PAT(K0), - [K | Acc] - end, - do_foldl_iterator_prefix(Shard, KeyPrefix, Fn, []). - -do_discard_iterator_prefix(Shard, KeyPrefix) -> - #db{handle = DBHandle, cf_iterator = CF} = meta_lookup(Shard, db), - Fn = fun(K, _V, _Acc) -> ok = rocksdb:delete(DBHandle, CF, K, ?ITERATION_WRITE_OPTS) end, - do_foldl_iterator_prefix(Shard, KeyPrefix, Fn, ok). - -do_foldl_iterator_prefix(Shard, KeyPrefix, Fn, Acc) -> - #db{handle = Handle, cf_iterator = CF} = meta_lookup(Shard, db), - case rocksdb:iterator(Handle, CF, ?ITERATION_READ_OPTS) of - {ok, It} -> - NextAction = {seek, KeyPrefix}, - do_foldl_iterator_prefix(Handle, CF, It, KeyPrefix, NextAction, Fn, Acc); - Error -> - Error - end. - -do_foldl_iterator_prefix(DBHandle, CF, It, KeyPrefix, NextAction, Fn, Acc) -> - case rocksdb:iterator_move(It, NextAction) of - {ok, K = <>, V} -> - NewAcc = Fn(K, V, Acc), - do_foldl_iterator_prefix(DBHandle, CF, It, KeyPrefix, next, Fn, NewAcc); - {ok, _K, _V} -> - ok = rocksdb:iterator_close(It), - {ok, Acc}; - {error, invalid_iterator} -> - ok = rocksdb:iterator_close(It), - {ok, Acc}; - Error -> - ok = rocksdb:iterator_close(It), - Error - end. - -%% Functions for dealing with the metadata stored persistently in rocksdb - --define(CURRENT_GEN, <<"current">>). --define(SCHEMA_WRITE_OPTS, []). --define(SCHEMA_READ_OPTS, []). - --spec schema_get_gen(rocksdb:db_handle(), gen_id()) -> generation(). -schema_get_gen(DBHandle, GenId) -> - {ok, Bin} = rocksdb:get(DBHandle, schema_gen_key(GenId), ?SCHEMA_READ_OPTS), - binary_to_term(Bin). - --spec schema_put_gen(rocksdb:db_handle(), gen_id(), generation()) -> ok | {error, _}. -schema_put_gen(DBHandle, GenId, Gen) -> - rocksdb:put(DBHandle, schema_gen_key(GenId), term_to_binary(Gen), ?SCHEMA_WRITE_OPTS). - --spec schema_get_current(rocksdb:db_handle()) -> gen_id() | undefined. -schema_get_current(DBHandle) -> - case rocksdb:get(DBHandle, ?CURRENT_GEN, ?SCHEMA_READ_OPTS) of - {ok, Bin} -> - binary_to_integer(Bin); - not_found -> - undefined - end. - --spec schema_put_current(rocksdb:db_handle(), gen_id()) -> ok | {error, _}. -schema_put_current(DBHandle, GenId) -> - rocksdb:put(DBHandle, ?CURRENT_GEN, integer_to_binary(GenId), ?SCHEMA_WRITE_OPTS). - --spec schema_gen_key(integer()) -> binary(). -schema_gen_key(N) -> - <<"gen", N:32>>. - --undef(CURRENT_GEN). --undef(SCHEMA_WRITE_OPTS). --undef(SCHEMA_READ_OPTS). - -%% Functions for dealing with the runtime shard metadata: - --define(PERSISTENT_TERM(SHARD, GEN), {?MODULE, SHARD, GEN}). - --spec meta_register_gen(emqx_ds:shard(), gen_id(), generation()) -> ok. -meta_register_gen(Shard, GenId, Gen) -> - Gs = - case GenId > 0 of - true -> meta_lookup(Shard, GenId - 1); - false -> [] +-spec generations_since(shard_id(), emqx_ds:time()) -> [gen_id()]. +generations_since(Shard, Since) -> + Schema = get_schema_runtime(Shard), + maps:fold( + fun + ({generation, GenId}, #{until := Until}, Acc) when Until >= Since -> + [GenId | Acc]; + (_K, _V, Acc) -> + Acc end, - ok = meta_put(Shard, GenId, [Gen | Gs]), - ok = meta_put(Shard, current, GenId). + [], + Schema + ). --spec meta_lookup_gen(emqx_ds:shard(), emqx_ds:time()) -> {gen_id(), generation()}. -meta_lookup_gen(Shard, Time) -> - % TODO - % Is cheaper persistent term GC on update here worth extra lookup? I'm leaning - % towards a "no". - Current = meta_lookup(Shard, current), - Gens = meta_lookup(Shard, Current), - find_gen(Time, Current, Gens). +-define(PERSISTENT_TERM(SHARD), {emqx_ds_storage_layer, SHARD}). -find_gen(Time, GenId, [Gen = #{since := Since} | _]) when Time >= Since -> - {GenId, Gen}; -find_gen(Time, GenId, [_Gen | Rest]) -> - find_gen(Time, GenId - 1, Rest). +-spec get_schema_runtime(shard_id()) -> shard(). +get_schema_runtime(Shard = {_, _}) -> + persistent_term:get(?PERSISTENT_TERM(Shard)). --spec meta_get_gen(emqx_ds:shard(), gen_id()) -> generation() | undefined. -meta_get_gen(Shard, GenId) -> - case meta_lookup(Shard, GenId, []) of - [Gen | _Older] -> Gen; - [] -> undefined - end. +-spec put_schema_runtime(shard_id(), shard()) -> ok. +put_schema_runtime(Shard = {_, _}, RuntimeSchema) -> + persistent_term:put(?PERSISTENT_TERM(Shard), RuntimeSchema), + ok. --spec meta_get_current(emqx_ds:shard()) -> gen_id() | undefined. -meta_get_current(Shard) -> - meta_lookup(Shard, current, undefined). - --spec meta_lookup(emqx_ds:shard(), _K) -> _V. -meta_lookup(Shard, K) -> - persistent_term:get(?PERSISTENT_TERM(Shard, K)). - --spec meta_lookup(emqx_ds:shard(), _K, Default) -> _V | Default. -meta_lookup(Shard, K, Default) -> - persistent_term:get(?PERSISTENT_TERM(Shard, K), Default). - --spec meta_put(emqx_ds:shard(), _K, _V) -> ok. -meta_put(Shard, K, V) -> - persistent_term:put(?PERSISTENT_TERM(Shard, K), V). - --spec meta_erase(emqx_ds:shard()) -> ok. -meta_erase(Shard) -> - [ - persistent_term:erase(K) - || {K = ?PERSISTENT_TERM(Z, _), _} <- persistent_term:get(), Z =:= Shard - ], +-spec erase_schema_runtime(shard_id()) -> ok. +erase_schema_runtime(Shard) -> + persistent_term:erase(?PERSISTENT_TERM(Shard)), ok. -undef(PERSISTENT_TERM). -get_next_id(undefined) -> 0; -get_next_id(GenId) -> GenId + 1. +-define(ROCKSDB_SCHEMA_KEY, <<"schema_v1">>). -is_gen_valid(Shard, GenId, Since) when GenId > 0 -> - [GenPrev | _] = meta_lookup(Shard, GenId - 1), - case GenPrev of - #{since := SincePrev} when Since > SincePrev -> - ok; - #{} -> - {error, nonmonotonic} - end; -is_gen_valid(_Shard, 0, 0) -> - ok. +-spec get_schema_persistent(rocksdb:db_handle()) -> shard_schema() | not_found. +get_schema_persistent(DB) -> + case rocksdb:get(DB, ?ROCKSDB_SCHEMA_KEY, []) of + {ok, Blob} -> + Schema = binary_to_term(Blob), + %% Sanity check: + #{current_generation := _, prototype := _} = Schema, + Schema; + not_found -> + not_found + end. -%% -spec store_cfs(rocksdb:db_handle(), [{string(), rocksdb:cf_handle()}]) -> ok. -%% store_cfs(DBHandle, CFRefs) -> -%% lists:foreach( -%% fun({CFName, CFRef}) -> -%% persistent_term:put({self(), CFName}, {DBHandle, CFRef}) -%% end, -%% CFRefs). +-spec put_schema_persistent(rocksdb:db_handle(), shard_schema()) -> ok. +put_schema_persistent(DB, Schema) -> + Blob = term_to_binary(Schema), + rocksdb:put(DB, ?ROCKSDB_SCHEMA_KEY, Blob, []). + +-undef(ROCKSDB_SCHEMA_KEY). diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl index 56c8c760a..c2eee8dcb 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl @@ -6,7 +6,7 @@ -behaviour(supervisor). %% API: --export([start_link/0, start_shard/2, stop_shard/1]). +-export([start_link/0, start_shard/2, stop_shard/1, ensure_shard/2]). %% behaviour callbacks: -export([init/1]). @@ -25,7 +25,7 @@ start_link() -> supervisor:start_link({local, ?SUP}, ?MODULE, []). --spec start_shard(emqx_ds:shard(), emqx_ds_storage_layer:options()) -> +-spec start_shard(emqx_ds_storage_layer:shard_id(), emqx_ds:create_db_opts()) -> supervisor:startchild_ret(). start_shard(Shard, Options) -> supervisor:start_child(?SUP, shard_child_spec(Shard, Options)). @@ -35,6 +35,18 @@ stop_shard(Shard) -> ok = supervisor:terminate_child(?SUP, Shard), ok = supervisor:delete_child(?SUP, Shard). +-spec ensure_shard(emqx_ds_storage_layer:shard_id(), emqx_ds_storage_layer:options()) -> + ok | {error, _Reason}. +ensure_shard(Shard, Options) -> + case start_shard(Shard, Options) of + {ok, _Pid} -> + ok; + {error, {already_started, _Pid}} -> + ok; + {error, Reason} -> + {error, Reason} + end. + %%================================================================================ %% behaviour callbacks %%================================================================================ @@ -52,7 +64,7 @@ init([]) -> %% Internal functions %%================================================================================ --spec shard_child_spec(emqx_ds:shard(), emqx_ds_storage_layer:options()) -> +-spec shard_child_spec(emqx_ds_storage_layer:shard_id(), emqx_ds:create_db_opts()) -> supervisor:child_spec(). shard_child_spec(Shard, Options) -> #{ diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl new file mode 100644 index 000000000..6676faf88 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl @@ -0,0 +1,139 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Reference implementation of the storage. +%% +%% Trivial, extremely slow and inefficient. It also doesn't handle +%% restart of the Erlang node properly, so obviously it's only to be +%% used for testing. +-module(emqx_ds_storage_reference). + +-behaviour(emqx_ds_storage_layer). + +%% API: +-export([]). + +%% behavior callbacks: +-export([create/4, open/5, store_batch/4, get_streams/4, make_iterator/5, next/4]). + +%% internal exports: +-export([]). + +-export_type([options/0]). + +-include_lib("emqx_utils/include/emqx_message.hrl"). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +-type options() :: #{}. + +%% Permanent state: +-record(schema, {}). + +%% Runtime state: +-record(s, { + db :: rocksdb:db_handle(), + cf :: rocksdb:cf_handle() +}). + +-record(stream, {}). + +-record(it, { + topic_filter :: emqx_ds:topic_filter(), + start_time :: emqx_ds:time(), + last_seen_message_key = first :: binary() | first +}). + +%%================================================================================ +%% API funcions +%%================================================================================ + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +create(_ShardId, DBHandle, GenId, _Options) -> + CFName = data_cf(GenId), + {ok, CFHandle} = rocksdb:create_column_family(DBHandle, CFName, []), + Schema = #schema{}, + {Schema, [{CFName, CFHandle}]}. + +open(_Shard, DBHandle, GenId, CFRefs, #schema{}) -> + {_, CF} = lists:keyfind(data_cf(GenId), 1, CFRefs), + #s{db = DBHandle, cf = CF}. + +store_batch(_ShardId, #s{db = DB, cf = CF}, Messages, _Options) -> + lists:foreach( + fun(Msg) -> + Id = erlang:unique_integer([monotonic]), + Key = <>, + Val = term_to_binary(Msg), + rocksdb:put(DB, CF, Key, Val, []) + end, + Messages + ). + +get_streams(_Shard, _Data, _TopicFilter, _StartTime) -> + [#stream{}]. + +make_iterator(_Shard, _Data, #stream{}, TopicFilter, StartTime) -> + {ok, #it{ + topic_filter = TopicFilter, + start_time = StartTime + }}. + +next(_Shard, #s{db = DB, cf = CF}, It0, BatchSize) -> + #it{topic_filter = TopicFilter, start_time = StartTime, last_seen_message_key = Key0} = It0, + {ok, ITHandle} = rocksdb:iterator(DB, CF, []), + Action = + case Key0 of + first -> + first; + _ -> + _ = rocksdb:iterator_move(ITHandle, Key0), + next + end, + {Key, Messages} = do_next(TopicFilter, StartTime, ITHandle, Action, BatchSize, Key0, []), + rocksdb:iterator_close(ITHandle), + It = It0#it{last_seen_message_key = Key}, + {ok, It, lists:reverse(Messages)}. + +%%================================================================================ +%% Internal functions +%%================================================================================ + +do_next(_, _, _, _, 0, Key, Acc) -> + {Key, Acc}; +do_next(TopicFilter, StartTime, IT, Action, NLeft, Key0, Acc) -> + case rocksdb:iterator_move(IT, Action) of + {ok, Key, Blob} -> + Msg = #message{topic = Topic, timestamp = TS} = binary_to_term(Blob), + case emqx_topic:match(Topic, TopicFilter) andalso TS >= StartTime of + true -> + do_next(TopicFilter, StartTime, IT, next, NLeft - 1, Key, [Msg | Acc]); + false -> + do_next(TopicFilter, StartTime, IT, next, NLeft, Key, Acc) + end; + {error, invalid_iterator} -> + {Key0, Acc} + end. + +%% @doc Generate a column family ID for the MQTT messages +-spec data_cf(emqx_ds_storage_layer:gen_id()) -> [char()]. +data_cf(GenId) -> + "emqx_ds_storage_reference" ++ integer_to_list(GenId). diff --git a/apps/emqx_durable_storage/src/emqx_ds_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_sup.erl index ca939e892..d371a2346 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_sup.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_sup.erl @@ -30,7 +30,7 @@ start_link() -> %%================================================================================ init([]) -> - Children = [shard_sup()], + Children = [storage_layer_sup()], SupFlags = #{ strategy => one_for_all, intensity => 0, @@ -42,7 +42,7 @@ init([]) -> %% Internal functions %%================================================================================ -shard_sup() -> +storage_layer_sup() -> #{ id => local_store_shard_sup, start => {emqx_ds_storage_layer_sup, start_link, []}, diff --git a/apps/emqx_durable_storage/src/emqx_durable_storage.app.src b/apps/emqx_durable_storage/src/emqx_durable_storage.app.src index 6edbfda9b..2bce4ff8e 100644 --- a/apps/emqx_durable_storage/src/emqx_durable_storage.app.src +++ b/apps/emqx_durable_storage/src/emqx_durable_storage.app.src @@ -2,10 +2,10 @@ {application, emqx_durable_storage, [ {description, "Message persistence and subscription replays for EMQX"}, % strict semver, bump manually! - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib, rocksdb, gproc, mria]}, + {applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]}, {mod, {emqx_ds_app, []}}, {env, []} ]}. diff --git a/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl new file mode 100644 index 000000000..10d1ed7a5 --- /dev/null +++ b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v1.erl @@ -0,0 +1,100 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ds_proto_v1). + +-behavior(emqx_bpapi). + +-include_lib("emqx_utils/include/bpapi.hrl"). +%% API: +-export([open_shard/4, drop_shard/3, store_batch/5, get_streams/5, make_iterator/6, next/5]). + +%% behavior callbacks: +-export([introduced_in/0]). + +%%================================================================================ +%% API funcions +%%================================================================================ + +-spec open_shard( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds:create_db_opts() +) -> + ok. +open_shard(Node, DB, Shard, Opts) -> + erpc:call(Node, emqx_ds_replication_layer, do_open_shard_v1, [DB, Shard, Opts]). + +-spec drop_shard(node(), emqx_ds:db(), emqx_ds_replication_layer:shard_id()) -> + ok. +drop_shard(Node, DB, Shard) -> + erpc:call(Node, emqx_ds_replication_layer, do_drop_shard_v1, [DB, Shard]). + +-spec get_streams( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> + [{integer(), emqx_ds_storage_layer:stream()}]. +get_streams(Node, DB, Shard, TopicFilter, Time) -> + erpc:call(Node, emqx_ds_replication_layer, do_get_streams_v1, [DB, Shard, TopicFilter, Time]). + +-spec make_iterator( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:stream(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _}. +make_iterator(Node, DB, Shard, Stream, TopicFilter, StartTime) -> + erpc:call(Node, emqx_ds_replication_layer, do_make_iterator_v1, [ + DB, Shard, Stream, TopicFilter, StartTime + ]). + +-spec next( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:iterator(), + pos_integer() +) -> + {ok, emqx_ds_storage_layer:iterator(), [emqx_types:messages()]} + | {ok, end_of_stream} + | {error, _}. +next(Node, DB, Shard, Iter, BatchSize) -> + erpc:call(Node, emqx_ds_replication_layer, do_next_v1, [DB, Shard, Iter, BatchSize]). + +-spec store_batch( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + [emqx_types:message()], + emqx_ds:message_store_opts() +) -> + emqx_ds:store_batch_result(). +store_batch(Node, DB, Shard, Batch, Options) -> + erpc:call(Node, emqx_ds_replication_layer, do_store_batch_v1, [DB, Shard, Batch, Options]). + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +introduced_in() -> + "5.4.0". diff --git a/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl new file mode 100644 index 000000000..9b74e3227 --- /dev/null +++ b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl @@ -0,0 +1,146 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ds_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +opts() -> + #{ + backend => builtin, + storage => {emqx_ds_storage_reference, #{}} + }. + +%% A simple smoke test that verifies that opening/closing the DB +%% doesn't crash, and not much else +t_00_smoke_open_drop(_Config) -> + DB = 'DB', + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + ?assertMatch(ok, emqx_ds:drop_db(DB)). + +%% A simple smoke test that verifies that storing the messages doesn't +%% crash +t_01_smoke_store(_Config) -> + DB = default, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + Msg = message(<<"foo/bar">>, <<"foo">>, 0), + ?assertMatch(ok, emqx_ds:store_batch(DB, [Msg])). + +%% A simple smoke test that verifies that getting the list of streams +%% doesn't crash and that iterators can be opened. +t_02_smoke_get_streams_start_iter(_Config) -> + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + StartTime = 0, + TopicFilter = ['#'], + [{Rank, Stream}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + ?assertMatch({_, _}, Rank), + ?assertMatch({ok, _Iter}, emqx_ds:make_iterator(DB, Stream, TopicFilter, StartTime)). + +%% A simple smoke test that verifies that it's possible to iterate +%% over messages. +t_03_smoke_iterate(_Config) -> + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + StartTime = 0, + TopicFilter = ['#'], + Msgs = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo">>, <<"2">>, 1), + message(<<"bar/bar">>, <<"3">>, 2) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs)), + [{_, Stream}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + {ok, Iter0} = emqx_ds:make_iterator(DB, Stream, TopicFilter, StartTime), + {ok, Iter, Batch} = iterate(DB, Iter0, 1), + ?assertEqual(Msgs, Batch, {Iter0, Iter}). + +%% Verify that iterators survive restart of the application. This is +%% an important property, since the lifetime of the iterators is tied +%% to the external resources, such as clients' sessions, and they +%% should always be able to continue replaying the topics from where +%% they are left off. +t_04_restart(_Config) -> + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + TopicFilter = ['#'], + StartTime = 0, + Msgs = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo">>, <<"2">>, 1), + message(<<"bar/bar">>, <<"3">>, 2) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs)), + [{_, Stream}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + {ok, Iter0} = emqx_ds:make_iterator(DB, Stream, TopicFilter, StartTime), + %% Restart the application: + ?tp(warning, emqx_ds_SUITE_restart_app, #{}), + ok = application:stop(emqx_durable_storage), + {ok, _} = application:ensure_all_started(emqx_durable_storage), + ok = emqx_ds:open_db(DB, opts()), + %% The old iterator should be still operational: + {ok, Iter, Batch} = iterate(DB, Iter0, 1), + ?assertEqual(Msgs, Batch, {Iter0, Iter}). + +message(Topic, Payload, PublishedAt) -> + #message{ + topic = Topic, + payload = Payload, + timestamp = PublishedAt, + id = emqx_guid:gen() + }. + +iterate(DB, It, BatchSize) -> + iterate(DB, It, BatchSize, []). + +iterate(DB, It0, BatchSize, Acc) -> + case emqx_ds:next(DB, It0, BatchSize) of + {ok, It, []} -> + {ok, It, Acc}; + {ok, It, Msgs} -> + iterate(DB, It, BatchSize, Acc ++ Msgs); + Ret -> + Ret + end. + +%% CT callbacks + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [mria, emqx_durable_storage], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_TC, Config) -> + %% snabbkaffe:fix_ct_logging(), + application:ensure_all_started(emqx_durable_storage), + Config. + +end_per_testcase(_TC, _Config) -> + ok = application:stop(emqx_durable_storage). diff --git a/apps/emqx_durable_storage/test/emqx_ds_message_storage_bitmask_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_message_storage_bitmask_SUITE.erl deleted file mode 100644 index 599bd6c7b..000000000 --- a/apps/emqx_durable_storage/test/emqx_ds_message_storage_bitmask_SUITE.erl +++ /dev/null @@ -1,188 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_ds_message_storage_bitmask_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("stdlib/include/assert.hrl"). - --import(emqx_ds_message_storage_bitmask, [ - make_keymapper/1, - keymapper_info/1, - compute_topic_bitmask/2, - compute_time_bitmask/1, - compute_topic_seek/4 -]). - -all() -> emqx_common_test_helpers:all(?MODULE). - -t_make_keymapper(_) -> - ?assertMatch( - #{ - source := [ - {timestamp, 9, 23}, - {hash, level, 2}, - {hash, level, 4}, - {hash, levels, 8}, - {timestamp, 0, 9} - ], - bitsize := 46, - epoch := 512 - }, - keymapper_info( - make_keymapper(#{ - timestamp_bits => 32, - topic_bits_per_level => [2, 4, 8], - epoch => 1000 - }) - ) - ). - -t_make_keymapper_single_hash_level(_) -> - ?assertMatch( - #{ - source := [ - {timestamp, 0, 32}, - {hash, levels, 16} - ], - bitsize := 48, - epoch := 1 - }, - keymapper_info( - make_keymapper(#{ - timestamp_bits => 32, - topic_bits_per_level => [16], - epoch => 1 - }) - ) - ). - -t_make_keymapper_no_timestamp(_) -> - ?assertMatch( - #{ - source := [ - {hash, level, 4}, - {hash, level, 8}, - {hash, levels, 16} - ], - bitsize := 28, - epoch := 1 - }, - keymapper_info( - make_keymapper(#{ - timestamp_bits => 0, - topic_bits_per_level => [4, 8, 16], - epoch => 42 - }) - ) - ). - -t_compute_topic_bitmask(_) -> - KM = make_keymapper(#{topic_bits_per_level => [3, 4, 5, 2], timestamp_bits => 0, epoch => 1}), - ?assertEqual( - 2#111_1111_11111_11, - compute_topic_bitmask([<<"foo">>, <<"bar">>], KM) - ), - ?assertEqual( - 2#111_0000_11111_11, - compute_topic_bitmask([<<"foo">>, '+'], KM) - ), - ?assertEqual( - 2#111_0000_00000_11, - compute_topic_bitmask([<<"foo">>, '+', '+'], KM) - ), - ?assertEqual( - 2#111_0000_11111_00, - compute_topic_bitmask([<<"foo">>, '+', <<"bar">>, '+'], KM) - ). - -t_compute_topic_bitmask_wildcard(_) -> - KM = make_keymapper(#{topic_bits_per_level => [3, 4, 5, 2], timestamp_bits => 0, epoch => 1}), - ?assertEqual( - 2#000_0000_00000_00, - compute_topic_bitmask(['#'], KM) - ), - ?assertEqual( - 2#111_0000_00000_00, - compute_topic_bitmask([<<"foo">>, '#'], KM) - ), - ?assertEqual( - 2#111_1111_11111_00, - compute_topic_bitmask([<<"foo">>, <<"bar">>, <<"baz">>, '#'], KM) - ). - -t_compute_topic_bitmask_wildcard_long_tail(_) -> - KM = make_keymapper(#{topic_bits_per_level => [3, 4, 5, 2], timestamp_bits => 0, epoch => 1}), - ?assertEqual( - 2#111_1111_11111_11, - compute_topic_bitmask([<<"foo">>, <<"bar">>, <<"baz">>, <<>>, <<"xyzzy">>], KM) - ), - ?assertEqual( - 2#111_1111_11111_00, - compute_topic_bitmask([<<"foo">>, <<"bar">>, <<"baz">>, <<>>, '#'], KM) - ). - -t_compute_time_bitmask(_) -> - KM = make_keymapper(#{topic_bits_per_level => [1, 2, 3], timestamp_bits => 10, epoch => 200}), - ?assertEqual(2#111_000000_1111111, compute_time_bitmask(KM)). - -t_compute_time_bitmask_epoch_only(_) -> - KM = make_keymapper(#{topic_bits_per_level => [1, 2, 3], timestamp_bits => 10, epoch => 1}), - ?assertEqual(2#1111111111_000000, compute_time_bitmask(KM)). - -%% Filter = |123|***|678|***| -%% Mask = |123|***|678|***| -%% Key1 = |123|011|108|121| → Seek = 0 |123|011|678|000| -%% Key2 = |123|011|679|919| → Seek = 0 |123|012|678|000| -%% Key3 = |123|999|679|001| → Seek = 1 |123|000|678|000| → eos -%% Key4 = |125|011|179|017| → Seek = 1 |123|000|678|000| → eos - -t_compute_next_topic_seek(_) -> - KM = make_keymapper(#{topic_bits_per_level => [8, 8, 16, 12], timestamp_bits => 0, epoch => 1}), - ?assertMatch( - none, - compute_topic_seek( - 16#FD_42_4242_043, - 16#FD_42_4242_042, - 16#FF_FF_FFFF_FFF, - KM - ) - ), - ?assertMatch( - 16#FD_11_0678_000, - compute_topic_seek( - 16#FD_11_0108_121, - 16#FD_00_0678_000, - 16#FF_00_FFFF_000, - KM - ) - ), - ?assertMatch( - 16#FD_12_0678_000, - compute_topic_seek( - 16#FD_11_0679_919, - 16#FD_00_0678_000, - 16#FF_00_FFFF_000, - KM - ) - ), - ?assertMatch( - none, - compute_topic_seek( - 16#FD_FF_0679_001, - 16#FD_00_0678_000, - 16#FF_00_FFFF_000, - KM - ) - ), - ?assertMatch( - none, - compute_topic_seek( - 16#FE_11_0179_017, - 16#FD_00_0678_000, - 16#FF_00_FFFF_000, - KM - ) - ). diff --git a/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl new file mode 100644 index 000000000..6dc24a269 --- /dev/null +++ b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl @@ -0,0 +1,396 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ds_storage_bitfield_lts_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(SHARD, shard(?FUNCTION_NAME)). + +-define(DEFAULT_CONFIG, #{ + backend => builtin, + storage => {emqx_ds_storage_bitfield_lts, #{}} +}). + +-define(COMPACT_CONFIG, #{ + backend => builtin, + storage => + {emqx_ds_storage_bitfield_lts, #{ + bits_per_wildcard_level => 8 + }} +}). + +%% Smoke test for opening and reopening the database +t_open(_Config) -> + ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), + {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}). + +%% Smoke test of store function +t_store(_Config) -> + MessageID = emqx_guid:gen(), + PublishedAt = 1000, + Topic = <<"foo/bar">>, + Payload = <<"message">>, + Msg = #message{ + id = MessageID, + topic = Topic, + payload = Payload, + timestamp = PublishedAt + }, + ?assertMatch(ok, emqx_ds_storage_layer:store_batch(?SHARD, [Msg], #{})). + +%% Smoke test for iteration through a concrete topic +t_iterate(_Config) -> + %% Prepare data: + Topics = [<<"foo/bar">>, <<"foo/bar/baz">>, <<"a">>], + Timestamps = lists:seq(1, 10), + Batch = [ + make_message(PublishedAt, Topic, integer_to_binary(PublishedAt)) + || Topic <- Topics, PublishedAt <- Timestamps + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, Batch, []), + %% Iterate through individual topics: + [ + begin + [{_Rank, Stream}] = emqx_ds_storage_layer:get_streams(?SHARD, parse_topic(Topic), 0), + {ok, It} = emqx_ds_storage_layer:make_iterator(?SHARD, Stream, parse_topic(Topic), 0), + {ok, NextIt, Messages} = emqx_ds_storage_layer:next(?SHARD, It, 100), + ?assertEqual( + lists:map(fun integer_to_binary/1, Timestamps), + payloads(Messages) + ), + {ok, _, []} = emqx_ds_storage_layer:next(?SHARD, NextIt, 100) + end + || Topic <- Topics + ], + ok. + +-define(assertSameSet(A, B), ?assertEqual(lists:sort(A), lists:sort(B))). + +%% Smoke test that verifies that concrete topics are mapped to +%% individual streams, unless there's too many of them. +t_get_streams(_Config) -> + %% Prepare data (without wildcards): + Topics = [<<"foo/bar">>, <<"foo/bar/baz">>, <<"a">>], + Timestamps = lists:seq(1, 10), + Batch = [ + make_message(PublishedAt, Topic, integer_to_binary(PublishedAt)) + || Topic <- Topics, PublishedAt <- Timestamps + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, Batch, []), + GetStream = fun(Topic) -> + StartTime = 0, + emqx_ds_storage_layer:get_streams(?SHARD, parse_topic(Topic), StartTime) + end, + %% Get streams for individual topics to use as a reference for later: + [FooBar = {_, _}] = GetStream(<<"foo/bar">>), + [FooBarBaz] = GetStream(<<"foo/bar/baz">>), + [A] = GetStream(<<"a">>), + %% Restart shard to make sure trie is persisted and restored: + ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), + {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}), + %% Verify that there are no "ghost streams" for topics that don't + %% have any messages: + [] = GetStream(<<"bar/foo">>), + %% Test some wildcard patterns: + ?assertEqual([FooBar], GetStream("+/+")), + ?assertSameSet([FooBar, FooBarBaz], GetStream(<<"foo/#">>)), + ?assertSameSet([FooBar, FooBarBaz, A], GetStream(<<"#">>)), + %% Now insert a bunch of messages with different topics to create wildcards: + NewBatch = [ + begin + B = integer_to_binary(I), + make_message(100, <<"foo/bar/", B/binary>>, <<"filler", B/binary>>) + end + || I <- lists:seq(1, 200) + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, NewBatch, []), + %% Check that "foo/bar/baz" topic now appears in two streams: + %% "foo/bar/baz" and "foo/bar/+": + NewStreams = lists:sort(GetStream("foo/bar/baz")), + ?assertMatch([_, _], NewStreams), + ?assert(lists:member(FooBarBaz, NewStreams)), + %% Verify that size of the trie is still relatively small, even + %% after processing 200+ topics: + AllStreams = GetStream("#"), + NTotal = length(AllStreams), + ?assert(NTotal < 30, {NTotal, '<', 30}), + ?assert(lists:member(FooBar, AllStreams)), + ?assert(lists:member(FooBarBaz, AllStreams)), + ?assert(lists:member(A, AllStreams)), + ok. + +t_replay(_Config) -> + %% Create concrete topics: + Topics = [<<"foo/bar">>, <<"foo/bar/baz">>], + Timestamps = lists:seq(1, 10_000, 100), + Batch1 = [ + make_message(PublishedAt, Topic, integer_to_binary(PublishedAt)) + || Topic <- Topics, PublishedAt <- Timestamps + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, Batch1, []), + %% Create wildcard topics `wildcard/+/suffix/foo' and `wildcard/+/suffix/bar': + Batch2 = [ + begin + B = integer_to_binary(I), + make_message( + TS, <<"wildcard/", B/binary, "/suffix/", Suffix/binary>>, integer_to_binary(TS) + ) + end + || I <- lists:seq(1, 200), TS <- Timestamps, Suffix <- [<<"foo">>, <<"bar">>] + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, Batch2, []), + %% Check various topic filters: + Messages = Batch1 ++ Batch2, + %% Missing topics (no ghost messages): + ?assertNot(check(?SHARD, <<"missing/foo/bar">>, 0, Messages)), + %% Regular topics: + ?assert(check(?SHARD, <<"foo/bar">>, 0, Messages)), + ?assert(check(?SHARD, <<"foo/bar/baz">>, 0, Messages)), + ?assert(check(?SHARD, <<"foo/#">>, 0, Messages)), + ?assert(check(?SHARD, <<"foo/+">>, 0, Messages)), + ?assert(check(?SHARD, <<"foo/+/+">>, 0, Messages)), + ?assert(check(?SHARD, <<"+/+/+">>, 0, Messages)), + ?assert(check(?SHARD, <<"+/+/baz">>, 0, Messages)), + %% Restart shard to make sure trie is persisted and restored: + ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), + {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}), + %% Learned wildcard topics: + ?assertNot(check(?SHARD, <<"wildcard/1000/suffix/foo">>, 0, [])), + ?assert(check(?SHARD, <<"wildcard/1/suffix/foo">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/100/suffix/foo">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/+/suffix/foo">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/1/suffix/+">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/100/suffix/+">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/#">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/1/#">>, 0, Messages)), + ?assert(check(?SHARD, <<"wildcard/100/#">>, 0, Messages)), + ?assert(check(?SHARD, <<"#">>, 0, Messages)), + ok. + +check(Shard, TopicFilter, StartTime, ExpectedMessages) -> + ExpectedFiltered = lists:filter( + fun(#message{topic = Topic, timestamp = TS}) -> + emqx_topic:match(Topic, TopicFilter) andalso TS >= StartTime + end, + ExpectedMessages + ), + ?check_trace( + #{timetrap => 10_000}, + begin + Dump = dump_messages(Shard, TopicFilter, StartTime), + verify_dump(TopicFilter, StartTime, Dump), + Missing = ExpectedFiltered -- Dump, + Extras = Dump -- ExpectedFiltered, + ?assertMatch( + #{missing := [], unexpected := []}, + #{ + missing => Missing, + unexpected => Extras, + topic_filter => TopicFilter, + start_time => StartTime + } + ) + end, + [] + ), + length(ExpectedFiltered) > 0. + +verify_dump(TopicFilter, StartTime, Dump) -> + lists:foldl( + fun(#message{topic = Topic, timestamp = TS}, Acc) -> + %% Verify that the topic of the message returned by the + %% iterator matches the expected topic filter: + ?assert(emqx_topic:match(Topic, TopicFilter), {unexpected_topic, Topic, TopicFilter}), + %% Verify that timestamp of the message is greater than + %% the StartTime of the iterator: + ?assert(TS >= StartTime, {start_time, TopicFilter, TS, StartTime}), + %% Verify that iterator didn't reorder messages + %% (timestamps for each topic are growing): + LastTopicTs = maps:get(Topic, Acc, -1), + ?assert(TS >= LastTopicTs, {topic_ts_reordering, Topic, TS, LastTopicTs}), + Acc#{Topic => TS} + end, + #{}, + Dump + ). + +dump_messages(Shard, TopicFilter, StartTime) -> + Streams = emqx_ds_storage_layer:get_streams(Shard, parse_topic(TopicFilter), StartTime), + lists:flatmap( + fun({_Rank, Stream}) -> + dump_stream(Shard, Stream, TopicFilter, StartTime) + end, + Streams + ). + +dump_stream(Shard, Stream, TopicFilter, StartTime) -> + BatchSize = 100, + {ok, Iterator} = emqx_ds_storage_layer:make_iterator( + Shard, Stream, parse_topic(TopicFilter), StartTime + ), + Loop = fun + F(It, 0) -> + error({too_many_iterations, It}); + F(It, N) -> + case emqx_ds_storage_layer:next(Shard, It, BatchSize) of + end_of_stream -> + []; + {ok, _NextIt, []} -> + []; + {ok, NextIt, Batch} -> + Batch ++ F(NextIt, N - 1) + end + end, + MaxIterations = 1000000, + Loop(Iterator, MaxIterations). + +%% t_create_gen(_Config) -> +%% {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 5, ?DEFAULT_CONFIG), +%% ?assertEqual( +%% {error, nonmonotonic}, +%% emqx_ds_storage_layer:create_generation(?SHARD, 1, ?DEFAULT_CONFIG) +%% ), +%% ?assertEqual( +%% {error, nonmonotonic}, +%% emqx_ds_storage_layer:create_generation(?SHARD, 5, ?DEFAULT_CONFIG) +%% ), +%% {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), +%% Topics = ["foo/bar", "foo/bar/baz"], +%% Timestamps = lists:seq(1, 100), +%% [ +%% ?assertMatch({ok, [_]}, store(?SHARD, PublishedAt, Topic, <<>>)) +%% || Topic <- Topics, PublishedAt <- Timestamps +%% ]. + +%% t_iterate_multigen(_Config) -> +%% {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), +%% {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 50, ?DEFAULT_CONFIG), +%% {ok, 3} = emqx_ds_storage_layer:create_generation(?SHARD, 1000, ?DEFAULT_CONFIG), +%% Topics = ["foo/bar", "foo/bar/baz", "a", "a/bar"], +%% Timestamps = lists:seq(1, 100), +%% _ = [ +%% store(?SHARD, PublishedAt, Topic, term_to_binary({Topic, PublishedAt})) +%% || Topic <- Topics, PublishedAt <- Timestamps +%% ], +%% ?assertEqual( +%% lists:sort([ +%% {Topic, PublishedAt} +%% || Topic <- ["foo/bar", "foo/bar/baz"], PublishedAt <- Timestamps +%% ]), +%% lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "foo/#", 0)]) +%% ), +%% ?assertEqual( +%% lists:sort([ +%% {Topic, PublishedAt} +%% || Topic <- ["a", "a/bar"], PublishedAt <- lists:seq(60, 100) +%% ]), +%% lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "a/#", 60)]) +%% ). + +%% t_iterate_multigen_preserve_restore(_Config) -> +%% ReplayID = atom_to_binary(?FUNCTION_NAME), +%% {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), +%% {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 50, ?DEFAULT_CONFIG), +%% {ok, 3} = emqx_ds_storage_layer:create_generation(?SHARD, 100, ?DEFAULT_CONFIG), +%% Topics = ["foo/bar", "foo/bar/baz", "a/bar"], +%% Timestamps = lists:seq(1, 100), +%% TopicFilter = "foo/#", +%% TopicsMatching = ["foo/bar", "foo/bar/baz"], +%% _ = [ +%% store(?SHARD, TS, Topic, term_to_binary({Topic, TS})) +%% || Topic <- Topics, TS <- Timestamps +%% ], +%% It0 = iterator(?SHARD, TopicFilter, 0), +%% {It1, Res10} = iterate(It0, 10), +%% % preserve mid-generation +%% ok = emqx_ds_storage_layer:preserve_iterator(It1, ReplayID), +%% {ok, It2} = emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID), +%% {It3, Res100} = iterate(It2, 88), +%% % preserve on the generation boundary +%% ok = emqx_ds_storage_layer:preserve_iterator(It3, ReplayID), +%% {ok, It4} = emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID), +%% {It5, Res200} = iterate(It4, 1000), +%% ?assertEqual({end_of_stream, []}, iterate(It5, 1)), +%% ?assertEqual( +%% lists:sort([{Topic, TS} || Topic <- TopicsMatching, TS <- Timestamps]), +%% lists:sort([binary_to_term(Payload) || Payload <- Res10 ++ Res100 ++ Res200]) +%% ), +%% ?assertEqual( +%% ok, +%% emqx_ds_storage_layer:discard_iterator(?SHARD, ReplayID) +%% ), +%% ?assertEqual( +%% {error, not_found}, +%% emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID) +%% ). + +make_message(PublishedAt, Topic, Payload) when is_list(Topic) -> + make_message(PublishedAt, list_to_binary(Topic), Payload); +make_message(PublishedAt, Topic, Payload) when is_binary(Topic) -> + ID = emqx_guid:gen(), + #message{ + id = ID, + topic = Topic, + timestamp = PublishedAt, + payload = Payload + }. + +store(Shard, PublishedAt, TopicL, Payload) when is_list(TopicL) -> + store(Shard, PublishedAt, list_to_binary(TopicL), Payload); +store(Shard, PublishedAt, Topic, Payload) -> + ID = emqx_guid:gen(), + Msg = #message{ + id = ID, + topic = Topic, + timestamp = PublishedAt, + payload = Payload + }, + emqx_ds_storage_layer:message_store(Shard, [Msg], #{}). + +payloads(Messages) -> + lists:map( + fun(#message{payload = P}) -> + P + end, + Messages + ). + +parse_topic(Topic = [L | _]) when is_binary(L); is_atom(L) -> + Topic; +parse_topic(Topic) -> + emqx_topic:words(iolist_to_binary(Topic)). + +%% CT callbacks + +all() -> emqx_common_test_helpers:all(?MODULE). +suite() -> [{timetrap, {seconds, 20}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_durable_storage), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_durable_storage). + +init_per_testcase(TC, Config) -> + {ok, _} = emqx_ds_storage_layer_sup:start_shard(shard(TC), ?DEFAULT_CONFIG), + Config. + +end_per_testcase(TC, _Config) -> + ok = emqx_ds_storage_layer_sup:stop_shard(shard(TC)). + +shard(TC) -> + {?MODULE, TC}. + +keyspace(TC) -> + TC. + +set_keyspace_config(Keyspace, Config) -> + ok = application:set_env(emqx_ds, keyspace_config, #{Keyspace => Config}). diff --git a/apps/emqx_durable_storage/test/emqx_ds_storage_layer_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_storage_layer_SUITE.erl deleted file mode 100644 index 3a872934f..000000000 --- a/apps/emqx_durable_storage/test/emqx_ds_storage_layer_SUITE.erl +++ /dev/null @@ -1,282 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_ds_storage_layer_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("common_test/include/ct.hrl"). --include_lib("stdlib/include/assert.hrl"). - --define(SHARD, shard(?FUNCTION_NAME)). - --define(DEFAULT_CONFIG, - {emqx_ds_message_storage_bitmask, #{ - timestamp_bits => 64, - topic_bits_per_level => [8, 8, 32, 16], - epoch => 5, - iteration => #{ - iterator_refresh => {every, 5} - } - }} -). - --define(COMPACT_CONFIG, - {emqx_ds_message_storage_bitmask, #{ - timestamp_bits => 16, - topic_bits_per_level => [16, 16], - epoch => 10 - }} -). - -%% Smoke test for opening and reopening the database -t_open(_Config) -> - ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), - {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}). - -%% Smoke test of store function -t_store(_Config) -> - MessageID = emqx_guid:gen(), - PublishedAt = 1000, - Topic = [<<"foo">>, <<"bar">>], - Payload = <<"message">>, - ?assertMatch(ok, emqx_ds_storage_layer:store(?SHARD, MessageID, PublishedAt, Topic, Payload)). - -%% Smoke test for iteration through a concrete topic -t_iterate(_Config) -> - %% Prepare data: - Topics = [[<<"foo">>, <<"bar">>], [<<"foo">>, <<"bar">>, <<"baz">>], [<<"a">>]], - Timestamps = lists:seq(1, 10), - [ - emqx_ds_storage_layer:store( - ?SHARD, - emqx_guid:gen(), - PublishedAt, - Topic, - integer_to_binary(PublishedAt) - ) - || Topic <- Topics, PublishedAt <- Timestamps - ], - %% Iterate through individual topics: - [ - begin - {ok, It} = emqx_ds_storage_layer:make_iterator(?SHARD, {Topic, 0}), - Values = iterate(It), - ?assertEqual(lists:map(fun integer_to_binary/1, Timestamps), Values) - end - || Topic <- Topics - ], - ok. - -%% Smoke test for iteration with wildcard topic filter -t_iterate_wildcard(_Config) -> - %% Prepare data: - Topics = ["foo/bar", "foo/bar/baz", "a", "a/bar"], - Timestamps = lists:seq(1, 10), - _ = [ - store(?SHARD, PublishedAt, Topic, term_to_binary({Topic, PublishedAt})) - || Topic <- Topics, PublishedAt <- Timestamps - ], - ?assertEqual( - lists:sort([{Topic, PublishedAt} || Topic <- Topics, PublishedAt <- Timestamps]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "#", 0)]) - ), - ?assertEqual( - [], - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "#", 10 + 1)]) - ), - ?assertEqual( - lists:sort([{Topic, PublishedAt} || Topic <- Topics, PublishedAt <- lists:seq(5, 10)]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "#", 5)]) - ), - ?assertEqual( - lists:sort([ - {Topic, PublishedAt} - || Topic <- ["foo/bar", "foo/bar/baz"], PublishedAt <- Timestamps - ]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "foo/#", 0)]) - ), - ?assertEqual( - lists:sort([{"foo/bar", PublishedAt} || PublishedAt <- Timestamps]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "foo/+", 0)]) - ), - ?assertEqual( - [], - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "foo/+/bar", 0)]) - ), - ?assertEqual( - lists:sort([ - {Topic, PublishedAt} - || Topic <- ["foo/bar", "foo/bar/baz", "a/bar"], PublishedAt <- Timestamps - ]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "+/bar/#", 0)]) - ), - ?assertEqual( - lists:sort([{Topic, PublishedAt} || Topic <- ["a", "a/bar"], PublishedAt <- Timestamps]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "a/#", 0)]) - ), - ?assertEqual( - [], - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "a/+/+", 0)]) - ), - ok. - -t_iterate_long_tail_wildcard(_Config) -> - Topic = "b/c/d/e/f/g", - TopicFilter = "b/c/d/e/+/+", - Timestamps = lists:seq(1, 100), - _ = [ - store(?SHARD, PublishedAt, Topic, term_to_binary({Topic, PublishedAt})) - || PublishedAt <- Timestamps - ], - ?assertEqual( - lists:sort([{"b/c/d/e/f/g", PublishedAt} || PublishedAt <- lists:seq(50, 100)]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, TopicFilter, 50)]) - ). - -t_create_gen(_Config) -> - {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 5, ?DEFAULT_CONFIG), - ?assertEqual( - {error, nonmonotonic}, - emqx_ds_storage_layer:create_generation(?SHARD, 1, ?DEFAULT_CONFIG) - ), - ?assertEqual( - {error, nonmonotonic}, - emqx_ds_storage_layer:create_generation(?SHARD, 5, ?DEFAULT_CONFIG) - ), - {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), - Topics = ["foo/bar", "foo/bar/baz"], - Timestamps = lists:seq(1, 100), - [ - ?assertEqual(ok, store(?SHARD, PublishedAt, Topic, <<>>)) - || Topic <- Topics, PublishedAt <- Timestamps - ]. - -t_iterate_multigen(_Config) -> - {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), - {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 50, ?DEFAULT_CONFIG), - {ok, 3} = emqx_ds_storage_layer:create_generation(?SHARD, 1000, ?DEFAULT_CONFIG), - Topics = ["foo/bar", "foo/bar/baz", "a", "a/bar"], - Timestamps = lists:seq(1, 100), - _ = [ - store(?SHARD, PublishedAt, Topic, term_to_binary({Topic, PublishedAt})) - || Topic <- Topics, PublishedAt <- Timestamps - ], - ?assertEqual( - lists:sort([ - {Topic, PublishedAt} - || Topic <- ["foo/bar", "foo/bar/baz"], PublishedAt <- Timestamps - ]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "foo/#", 0)]) - ), - ?assertEqual( - lists:sort([ - {Topic, PublishedAt} - || Topic <- ["a", "a/bar"], PublishedAt <- lists:seq(60, 100) - ]), - lists:sort([binary_to_term(Payload) || Payload <- iterate(?SHARD, "a/#", 60)]) - ). - -t_iterate_multigen_preserve_restore(_Config) -> - ReplayID = atom_to_binary(?FUNCTION_NAME), - {ok, 1} = emqx_ds_storage_layer:create_generation(?SHARD, 10, ?COMPACT_CONFIG), - {ok, 2} = emqx_ds_storage_layer:create_generation(?SHARD, 50, ?DEFAULT_CONFIG), - {ok, 3} = emqx_ds_storage_layer:create_generation(?SHARD, 100, ?DEFAULT_CONFIG), - Topics = ["foo/bar", "foo/bar/baz", "a/bar"], - Timestamps = lists:seq(1, 100), - TopicFilter = "foo/#", - TopicsMatching = ["foo/bar", "foo/bar/baz"], - _ = [ - store(?SHARD, TS, Topic, term_to_binary({Topic, TS})) - || Topic <- Topics, TS <- Timestamps - ], - It0 = iterator(?SHARD, TopicFilter, 0), - {It1, Res10} = iterate(It0, 10), - % preserve mid-generation - ok = emqx_ds_storage_layer:preserve_iterator(It1, ReplayID), - {ok, It2} = emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID), - {It3, Res100} = iterate(It2, 88), - % preserve on the generation boundary - ok = emqx_ds_storage_layer:preserve_iterator(It3, ReplayID), - {ok, It4} = emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID), - {It5, Res200} = iterate(It4, 1000), - ?assertEqual(none, It5), - ?assertEqual( - lists:sort([{Topic, TS} || Topic <- TopicsMatching, TS <- Timestamps]), - lists:sort([binary_to_term(Payload) || Payload <- Res10 ++ Res100 ++ Res200]) - ), - ?assertEqual( - ok, - emqx_ds_storage_layer:discard_iterator(?SHARD, ReplayID) - ), - ?assertEqual( - {error, not_found}, - emqx_ds_storage_layer:restore_iterator(?SHARD, ReplayID) - ). - -store(Shard, PublishedAt, Topic, Payload) -> - ID = emqx_guid:gen(), - emqx_ds_storage_layer:store(Shard, ID, PublishedAt, parse_topic(Topic), Payload). - -iterate(DB, TopicFilter, StartTime) -> - iterate(iterator(DB, TopicFilter, StartTime)). - -iterate(It) -> - case emqx_ds_storage_layer:next(It) of - {value, Payload, ItNext} -> - [Payload | iterate(ItNext)]; - none -> - [] - end. - -iterate(It, 0) -> - {It, []}; -iterate(It, N) -> - case emqx_ds_storage_layer:next(It) of - {value, Payload, ItNext} -> - {ItFinal, Ps} = iterate(ItNext, N - 1), - {ItFinal, [Payload | Ps]}; - none -> - {none, []} - end. - -iterator(DB, TopicFilter, StartTime) -> - {ok, It} = emqx_ds_storage_layer:make_iterator(DB, {parse_topic(TopicFilter), StartTime}), - It. - -parse_topic(Topic = [L | _]) when is_binary(L); is_atom(L) -> - Topic; -parse_topic(Topic) -> - emqx_topic:words(iolist_to_binary(Topic)). - -%% CT callbacks - -all() -> emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - {ok, _} = application:ensure_all_started(emqx_durable_storage), - Config. - -end_per_suite(_Config) -> - ok = application:stop(emqx_durable_storage). - -init_per_testcase(TC, Config) -> - ok = set_keyspace_config(keyspace(TC), ?DEFAULT_CONFIG), - {ok, _} = emqx_ds_storage_layer_sup:start_shard(shard(TC), #{}), - Config. - -end_per_testcase(TC, _Config) -> - ok = emqx_ds_storage_layer_sup:stop_shard(shard(TC)). - -keyspace(TC) -> - list_to_atom(lists:concat([?MODULE, "_", TC])). - -shard_id(_TC) -> - <<"shard">>. - -shard(TC) -> - {keyspace(TC), shard_id(TC)}. - -set_keyspace_config(Keyspace, Config) -> - ok = application:set_env(emqx_ds, keyspace_config, #{Keyspace => Config}). diff --git a/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl b/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl index e9daf2581..9b5af9428 100644 --- a/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl +++ b/apps/emqx_durable_storage/test/props/emqx_ds_message_storage_bitmask_shim.erl @@ -4,9 +4,11 @@ -module(emqx_ds_message_storage_bitmask_shim). +-include_lib("emqx/include/emqx.hrl"). + -export([open/0]). -export([close/1]). --export([store/5]). +-export([store/2]). -export([iterate/2]). -type topic() :: list(binary()). @@ -25,20 +27,21 @@ close(Tab) -> true = ets:delete(Tab), ok. --spec store(t(), emqx_guid:guid(), time(), topic(), binary()) -> +-spec store(t(), emqx_types:message()) -> ok | {error, _TODO}. -store(Tab, MessageID, PublishedAt, Topic, Payload) -> - true = ets:insert(Tab, {{PublishedAt, MessageID}, Topic, Payload}), +store(Tab, Msg = #message{id = MessageID, timestamp = PublishedAt}) -> + true = ets:insert(Tab, {{PublishedAt, MessageID}, Msg}), ok. -spec iterate(t(), emqx_ds:replay()) -> [binary()]. -iterate(Tab, {TopicFilter, StartTime}) -> +iterate(Tab, {TopicFilter0, StartTime}) -> + TopicFilter = iolist_to_binary(lists:join("/", TopicFilter0)), ets:foldr( - fun({{PublishedAt, _}, Topic, Payload}, Acc) -> + fun({{PublishedAt, _}, Msg = #message{topic = Topic}}, Acc) -> case emqx_topic:match(Topic, TopicFilter) of true when PublishedAt >= StartTime -> - [Payload | Acc]; + [Msg | Acc]; _ -> Acc end diff --git a/apps/emqx_durable_storage/test/props/prop_replay_message_storage.erl b/apps/emqx_durable_storage/test/props/prop_replay_message_storage.erl deleted file mode 100644 index f9964bebe..000000000 --- a/apps/emqx_durable_storage/test/props/prop_replay_message_storage.erl +++ /dev/null @@ -1,466 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(prop_replay_message_storage). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(WORK_DIR, ["_build", "test"]). --define(RUN_ID, {?MODULE, testrun_id}). - --define(KEYSPACE, ?MODULE). --define(SHARD_ID, <<"shard">>). --define(SHARD, {?KEYSPACE, ?SHARD_ID}). --define(GEN_ID, 42). - -%%-------------------------------------------------------------------- -%% Properties -%%-------------------------------------------------------------------- - -prop_bitstring_computes() -> - ?FORALL( - Keymapper, - keymapper(), - ?FORALL({Topic, Timestamp}, {topic(), integer()}, begin - BS = emqx_ds_message_storage_bitmask:compute_bitstring(Topic, Timestamp, Keymapper), - is_integer(BS) andalso (BS < (1 bsl get_keymapper_bitsize(Keymapper))) - end) - ). - -prop_topic_bitmask_computes() -> - Keymapper = make_keymapper(16, [8, 12, 16], 100), - ?FORALL(TopicFilter, topic_filter(), begin - Mask = emqx_ds_message_storage_bitmask:compute_topic_bitmask(TopicFilter, Keymapper), - % topic bits + timestamp LSBs - is_integer(Mask) andalso (Mask < (1 bsl (36 + 6))) - end). - -prop_next_seek_monotonic() -> - ?FORALL( - {TopicFilter, StartTime, Keymapper}, - {topic_filter(), pos_integer(), keymapper()}, - begin - Filter = emqx_ds_message_storage_bitmask:make_keyspace_filter( - {TopicFilter, StartTime}, - Keymapper - ), - ?FORALL( - Bitstring, - bitstr(get_keymapper_bitsize(Keymapper)), - emqx_ds_message_storage_bitmask:compute_next_seek(Bitstring, Filter) >= Bitstring - ) - end - ). - -prop_next_seek_eq_initial_seek() -> - ?FORALL( - Filter, - keyspace_filter(), - emqx_ds_message_storage_bitmask:compute_initial_seek(Filter) =:= - emqx_ds_message_storage_bitmask:compute_next_seek(0, Filter) - ). - -prop_iterate_messages() -> - TBPL = [4, 8, 12], - Options = #{ - timestamp_bits => 32, - topic_bits_per_level => TBPL, - epoch => 200 - }, - % TODO - % Shrinking is too unpredictable and leaves a LOT of garbage in the scratch dit. - ?FORALL(Stream, noshrink(non_empty(messages(topic(TBPL)))), begin - Filepath = make_filepath(?FUNCTION_NAME, erlang:system_time(microsecond)), - {DB, Handle} = open_db(Filepath, Options), - Shim = emqx_ds_message_storage_bitmask_shim:open(), - ok = store_db(DB, Stream), - ok = store_shim(Shim, Stream), - ?FORALL( - { - {Topic, _}, - Pattern, - StartTime - }, - { - nth(Stream), - topic_filter_pattern(), - start_time() - }, - begin - TopicFilter = make_topic_filter(Pattern, Topic), - Iteration = {TopicFilter, StartTime}, - Messages = iterate_db(DB, Iteration), - Reference = iterate_shim(Shim, Iteration), - ok = close_db(Handle), - ok = emqx_ds_message_storage_bitmask_shim:close(Shim), - ?WHENFAIL( - begin - io:format(user, " *** Filepath = ~s~n", [Filepath]), - io:format(user, " *** TopicFilter = ~p~n", [TopicFilter]), - io:format(user, " *** StartTime = ~p~n", [StartTime]) - end, - is_list(Messages) andalso equals(Messages -- Reference, Reference -- Messages) - ) - end - ) - end). - -prop_iterate_eq_iterate_with_preserve_restore() -> - TBPL = [4, 8, 16, 12], - Options = #{ - timestamp_bits => 32, - topic_bits_per_level => TBPL, - epoch => 500 - }, - {DB, _Handle} = open_db(make_filepath(?FUNCTION_NAME), Options), - ?FORALL(Stream, non_empty(messages(topic(TBPL))), begin - % TODO - % This proptest is impure because messages from testruns assumed to be - % independent of each other are accumulated in the same storage. This - % would probably confuse shrinker in the event a testrun fails. - ok = store_db(DB, Stream), - ?FORALL( - { - {Topic, _}, - Pat, - StartTime, - Commands - }, - { - nth(Stream), - topic_filter_pattern(), - start_time(), - shuffled(flat([non_empty(list({preserve, restore})), list(iterate)])) - }, - begin - Replay = {make_topic_filter(Pat, Topic), StartTime}, - Iterator = make_iterator(DB, Replay), - Ctx = #{db => DB, replay => Replay}, - Messages = run_iterator_commands(Commands, Iterator, Ctx), - equals(Messages, iterate_db(DB, Replay)) - end - ) - end). - -prop_iterate_eq_iterate_with_refresh() -> - TBPL = [4, 8, 16, 12], - Options = #{ - timestamp_bits => 32, - topic_bits_per_level => TBPL, - epoch => 500 - }, - {DB, _Handle} = open_db(make_filepath(?FUNCTION_NAME), Options), - ?FORALL(Stream, non_empty(messages(topic(TBPL))), begin - % TODO - % This proptest is also impure, see above. - ok = store_db(DB, Stream), - ?FORALL( - { - {Topic, _}, - Pat, - StartTime, - RefreshEvery - }, - { - nth(Stream), - topic_filter_pattern(), - start_time(), - pos_integer() - }, - ?TIMEOUT(5000, begin - Replay = {make_topic_filter(Pat, Topic), StartTime}, - IterationOptions = #{iterator_refresh => {every, RefreshEvery}}, - Iterator = make_iterator(DB, Replay, IterationOptions), - Messages = iterate_db(Iterator), - equals(Messages, iterate_db(DB, Replay)) - end) - ) - end). - -% store_message_stream(DB, [{Topic, {Payload, ChunkNum, _ChunkCount}} | Rest]) -> -% MessageID = emqx_guid:gen(), -% PublishedAt = ChunkNum, -% MessageID, PublishedAt, Topic -% ]), -% ok = emqx_ds_message_storage_bitmask:store(DB, MessageID, PublishedAt, Topic, Payload), -% store_message_stream(DB, payload_gen:next(Rest)); -% store_message_stream(_Zone, []) -> -% ok. - -store_db(DB, Messages) -> - lists:foreach( - fun({Topic, Payload = {MessageID, Timestamp, _}}) -> - Bin = term_to_binary(Payload), - emqx_ds_message_storage_bitmask:store(DB, MessageID, Timestamp, Topic, Bin) - end, - Messages - ). - -iterate_db(DB, Iteration) -> - iterate_db(make_iterator(DB, Iteration)). - -iterate_db(It) -> - case emqx_ds_message_storage_bitmask:next(It) of - {value, Payload, ItNext} -> - [binary_to_term(Payload) | iterate_db(ItNext)]; - none -> - [] - end. - -make_iterator(DB, Replay) -> - {ok, It} = emqx_ds_message_storage_bitmask:make_iterator(DB, Replay), - It. - -make_iterator(DB, Replay, Options) -> - {ok, It} = emqx_ds_message_storage_bitmask:make_iterator(DB, Replay, Options), - It. - -run_iterator_commands([iterate | Rest], It, Ctx) -> - case emqx_ds_message_storage_bitmask:next(It) of - {value, Payload, ItNext} -> - [binary_to_term(Payload) | run_iterator_commands(Rest, ItNext, Ctx)]; - none -> - [] - end; -run_iterator_commands([{preserve, restore} | Rest], It, Ctx) -> - #{ - db := DB, - replay := Replay - } = Ctx, - Serial = emqx_ds_message_storage_bitmask:preserve_iterator(It), - {ok, ItNext} = emqx_ds_message_storage_bitmask:restore_iterator(DB, Replay, Serial), - run_iterator_commands(Rest, ItNext, Ctx); -run_iterator_commands([], It, _Ctx) -> - iterate_db(It). - -store_shim(Shim, Messages) -> - lists:foreach( - fun({Topic, Payload = {MessageID, Timestamp, _}}) -> - Bin = term_to_binary(Payload), - emqx_ds_message_storage_bitmask_shim:store(Shim, MessageID, Timestamp, Topic, Bin) - end, - Messages - ). - -iterate_shim(Shim, Iteration) -> - lists:map( - fun binary_to_term/1, - emqx_ds_message_storage_bitmask_shim:iterate(Shim, Iteration) - ). - -%%-------------------------------------------------------------------- -%% Setup / teardown -%%-------------------------------------------------------------------- - -open_db(Filepath, Options) -> - {ok, Handle} = rocksdb:open(Filepath, [{create_if_missing, true}]), - {Schema, CFRefs} = emqx_ds_message_storage_bitmask:create_new(Handle, ?GEN_ID, Options), - DB = emqx_ds_message_storage_bitmask:open(?SHARD, Handle, ?GEN_ID, CFRefs, Schema), - {DB, Handle}. - -close_db(Handle) -> - rocksdb:close(Handle). - -make_filepath(TC) -> - make_filepath(TC, 0). - -make_filepath(TC, InstID) -> - Name = io_lib:format("~0p.~0p", [TC, InstID]), - Path = filename:join(?WORK_DIR ++ ["proper", "runs", get_run_id(), ?MODULE_STRING, Name]), - ok = filelib:ensure_dir(Path), - Path. - -get_run_id() -> - case persistent_term:get(?RUN_ID, undefined) of - RunID when RunID /= undefined -> - RunID; - undefined -> - RunID = make_run_id(), - ok = persistent_term:put(?RUN_ID, RunID), - RunID - end. - -make_run_id() -> - calendar:system_time_to_rfc3339(erlang:system_time(second), [{offset, "Z"}]). - -%%-------------------------------------------------------------------- -%% Type generators -%%-------------------------------------------------------------------- - -topic() -> - non_empty(list(topic_level())). - -topic(EntropyWeights) -> - ?LET(L, scaled(1 / 4, list(1)), begin - EWs = lists:sublist(EntropyWeights ++ L, length(L)), - ?SIZED(S, [oneof([topic_level(S * EW), topic_level_fixed()]) || EW <- EWs]) - end). - -topic_filter() -> - ?SUCHTHAT( - L, - non_empty( - list( - frequency([ - {5, topic_level()}, - {2, '+'}, - {1, '#'} - ]) - ) - ), - not lists:member('#', L) orelse lists:last(L) == '#' - ). - -topic_level_pattern() -> - frequency([ - {5, level}, - {2, '+'}, - {1, '#'} - ]). - -topic_filter_pattern() -> - list(topic_level_pattern()). - -topic_filter(Topic) -> - ?LET({T, Pat}, {Topic, topic_filter_pattern()}, make_topic_filter(Pat, T)). - -make_topic_filter([], _) -> - []; -make_topic_filter(_, []) -> - []; -make_topic_filter(['#' | _], _) -> - ['#']; -make_topic_filter(['+' | Rest], [_ | Levels]) -> - ['+' | make_topic_filter(Rest, Levels)]; -make_topic_filter([level | Rest], [L | Levels]) -> - [L | make_topic_filter(Rest, Levels)]. - -% topic() -> -% ?LAZY(?SIZED(S, frequency([ -% {S, [topic_level() | topic()]}, -% {1, []} -% ]))). - -% topic_filter() -> -% ?LAZY(?SIZED(S, frequency([ -% {round(S / 3 * 2), [topic_level() | topic_filter()]}, -% {round(S / 3 * 1), ['+' | topic_filter()]}, -% {1, []}, -% {1, ['#']} -% ]))). - -topic_level() -> - ?LET(L, list(oneof([range($a, $z), range($0, $9)])), iolist_to_binary(L)). - -topic_level(Entropy) -> - S = floor(1 + math:log2(Entropy) / 4), - ?LET(I, range(1, Entropy), iolist_to_binary(io_lib:format("~*.16.0B", [S, I]))). - -topic_level_fixed() -> - oneof([ - <<"foo">>, - <<"bar">>, - <<"baz">>, - <<"xyzzy">> - ]). - -keymapper() -> - ?LET( - {TimestampBits, TopicBits, Epoch}, - { - range(0, 128), - non_empty(list(range(1, 32))), - pos_integer() - }, - make_keymapper(TimestampBits, TopicBits, Epoch * 100) - ). - -keyspace_filter() -> - ?LET( - {TopicFilter, StartTime, Keymapper}, - {topic_filter(), pos_integer(), keymapper()}, - emqx_ds_message_storage_bitmask:make_keyspace_filter({TopicFilter, StartTime}, Keymapper) - ). - -messages(Topic) -> - ?LET( - Ts, - list(Topic), - interleaved( - ?LET(Messages, vector(length(Ts), scaled(4, list(message()))), lists:zip(Ts, Messages)) - ) - ). - -message() -> - ?LET({Timestamp, Payload}, {timestamp(), binary()}, {emqx_guid:gen(), Timestamp, Payload}). - -message_streams(Topic) -> - ?LET(Topics, list(Topic), [{T, payload_gen:binary_stream_gen(64)} || T <- Topics]). - -timestamp() -> - scaled(20, pos_integer()). - -start_time() -> - scaled(10, pos_integer()). - -bitstr(Size) -> - ?LET(B, binary(1 + (Size div 8)), binary:decode_unsigned(B) band (1 bsl Size - 1)). - -nth(L) -> - ?LET(I, range(1, length(L)), lists:nth(I, L)). - -scaled(Factor, T) -> - ?SIZED(S, resize(ceil(S * Factor), T)). - -interleaved(T) -> - ?LET({L, Seed}, {T, integer()}, interleave(L, rand:seed_s(exsss, Seed))). - -shuffled(T) -> - ?LET({L, Seed}, {T, integer()}, shuffle(L, rand:seed_s(exsss, Seed))). - -flat(T) -> - ?LET(L, T, lists:flatten(L)). - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -make_keymapper(TimestampBits, TopicBits, MaxEpoch) -> - emqx_ds_message_storage_bitmask:make_keymapper(#{ - timestamp_bits => TimestampBits, - topic_bits_per_level => TopicBits, - epoch => MaxEpoch - }). - -get_keymapper_bitsize(Keymapper) -> - maps:get(bitsize, emqx_ds_message_storage_bitmask:keymapper_info(Keymapper)). - --spec interleave(list({Tag, list(E)}), rand:state()) -> list({Tag, E}). -interleave(Seqs, Rng) -> - interleave(Seqs, length(Seqs), Rng). - -interleave(Seqs, L, Rng) when L > 0 -> - {N, RngNext} = rand:uniform_s(L, Rng), - {SeqHead, SeqTail} = lists:split(N - 1, Seqs), - case SeqTail of - [{Tag, [M | Rest]} | SeqRest] -> - [{Tag, M} | interleave(SeqHead ++ [{Tag, Rest} | SeqRest], L, RngNext)]; - [{_, []} | SeqRest] -> - interleave(SeqHead ++ SeqRest, L - 1, RngNext) - end; -interleave([], 0, _) -> - []. - --spec shuffle(list(E), rand:state()) -> list(E). -shuffle(L, Rng) -> - {Rands, _} = randoms(length(L), Rng), - [E || {_, E} <- lists:sort(lists:zip(Rands, L))]. - -randoms(N, Rng) when N > 0 -> - {Rand, RngNext} = rand:uniform_s(Rng), - {Tail, RngFinal} = randoms(N - 1, RngNext), - {[Rand | Tail], RngFinal}; -randoms(_, Rng) -> - {[], Rng}. diff --git a/apps/emqx_enterprise/src/emqx_enterprise.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src index 1a5359db6..06bc500f4 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ {application, emqx_enterprise, [ {description, "EMQX Enterprise Edition"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 16474b424..658666fc7 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -53,7 +53,7 @@ fields("log_audit_handler") -> {"path", hoconsc:mk( - emqx_conf_schema:file(), + string(), #{ desc => ?DESC(emqx_conf_schema, "audit_file_handler_path"), default => <<"${EMQX_LOG_DIR}/audit.log">>, diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 2dd33479c..cb86c1450 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl index 41020e76f..c886b86bd 100644 --- a/apps/emqx_ft/src/emqx_ft.erl +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -90,6 +90,10 @@ -define(FT_EVENT(EVENT), {?MODULE, EVENT}). +-define(ACK_AND_PUBLISH(Result), {true, Result}). +-define(ACK(Result), {false, Result}). +-define(DELAY_ACK, delay). + %%-------------------------------------------------------------------- %% API for app %%-------------------------------------------------------------------- @@ -116,46 +120,34 @@ unhook() -> %% API %%-------------------------------------------------------------------- -decode_filemeta(Payload) when is_binary(Payload) -> - case emqx_utils_json:safe_decode(Payload, [return_maps]) of - {ok, Map} -> - decode_filemeta(Map); - {error, Error} -> - {error, {invalid_filemeta_json, Error}} - end; -decode_filemeta(Map) when is_map(Map) -> - Schema = emqx_ft_schema:schema(filemeta), - try - Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), - {ok, Meta} - catch - throw:{_Schema, Errors} -> - {error, {invalid_filemeta, Errors}} - end. +decode_filemeta(Payload) -> + emqx_ft_schema:decode(filemeta, Payload). encode_filemeta(Meta = #{}) -> - Schema = emqx_ft_schema:schema(filemeta), - hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Meta), #{}). + emqx_ft_schema:encode(filemeta, Meta). + +encode_response(Response) -> + emqx_ft_schema:encode(command_response, Response). %%-------------------------------------------------------------------- %% Hooks %%-------------------------------------------------------------------- -on_message_publish( - Msg = #message{ - id = _Id, - topic = <<"$file/", _/binary>> - } -) -> +on_message_publish(Msg = #message{topic = <<"$file-async/", _/binary>>}) -> + Headers = Msg#message.headers, + {stop, Msg#message{headers = Headers#{allow_publish => false}}}; +on_message_publish(Msg = #message{topic = <<"$file/", _/binary>>}) -> Headers = Msg#message.headers, {stop, Msg#message{headers = Headers#{allow_publish => false}}}; on_message_publish(Msg) -> {ok, Msg}. -on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> +on_message_puback(PacketId, #message{from = From, topic = Topic} = Msg, _PubRes, _RC) -> case Topic of - <<"$file/", FileCommand/binary>> -> - {stop, on_file_command(PacketId, Msg, FileCommand)}; + <<"$file/", _/binary>> -> + {stop, on_file_command(sync, From, PacketId, Msg, Topic)}; + <<"$file-async/", _/binary>> -> + {stop, on_file_command(async, From, PacketId, Msg, Topic)}; _ -> ignore end. @@ -163,18 +155,33 @@ on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> on_channel_unregistered(ChannelPid) -> ok = emqx_ft_async_reply:deregister_all(ChannelPid). -on_client_timeout(_TRef, ?FT_EVENT({MRef, PacketId}), Acc) -> +on_client_timeout(_TRef0, ?FT_EVENT({MRef, TopicReplyData}), Acc) -> _ = erlang:demonitor(MRef, [flush]), - _ = emqx_ft_async_reply:take_by_mref(MRef), - {stop, [?REPLY_OUTGOING(?PUBACK_PACKET(PacketId, ?RC_UNSPECIFIED_ERROR)) | Acc]}; + Result = {error, timeout}, + _ = publish_response(Result, TopicReplyData), + case emqx_ft_async_reply:take_by_mref(MRef) of + {ok, undefined, _TRef1, _TopicReplyData} -> + {stop, Acc}; + {ok, PacketId, _TRef1, _TopicReplyData} -> + {stop, [?REPLY_OUTGOING(?PUBACK_PACKET(PacketId, result_to_rc(Result))) | Acc]}; + not_found -> + {ok, Acc} + end; on_client_timeout(_TRef, _Event, Acc) -> {ok, Acc}. -on_process_down(MRef, _Pid, Reason, Acc) -> +on_process_down(MRef, _Pid, DownReason, Acc) -> case emqx_ft_async_reply:take_by_mref(MRef) of - {ok, PacketId, TRef} -> + {ok, PacketId, TRef, TopicReplyData} -> _ = emqx_utils:cancel_timer(TRef), - {stop, [?REPLY_OUTGOING(?PUBACK_PACKET(PacketId, reason_to_rc(Reason))) | Acc]}; + Result = down_reason_to_result(DownReason), + _ = publish_response(Result, TopicReplyData), + case PacketId of + undefined -> + {stop, Acc}; + _ -> + {stop, [?REPLY_OUTGOING(?PUBACK_PACKET(PacketId, result_to_rc(Result))) | Acc]} + end; not_found -> {ok, Acc} end. @@ -185,24 +192,27 @@ on_process_down(MRef, _Pid, Reason, Acc) -> %% TODO Move to emqx_ft_mqtt? -on_file_command(PacketId, Msg, FileCommand) -> - case emqx_topic:tokens(FileCommand) of - [FileIdIn | Rest] -> - validate([{fileid, FileIdIn}], fun([FileId]) -> - on_file_command(PacketId, FileId, Msg, Rest) - end); - [] -> - ?RC_UNSPECIFIED_ERROR - end. +on_file_command(Mode, From, PacketId, Msg, Topic) -> + TopicReplyData = topic_reply_data(Mode, From, PacketId, Msg), + Result = + case emqx_topic:tokens(Topic) of + [_FTPrefix, FileIdIn | Rest] -> + validate([{fileid, FileIdIn}], fun([FileId]) -> + do_on_file_command(TopicReplyData, FileId, Msg, Rest) + end); + [] -> + ?ACK_AND_PUBLISH({error, {invalid_topic, Topic}}) + end, + maybe_publish_response(Result, TopicReplyData). -on_file_command(PacketId, FileId, Msg, FileCommand) -> +do_on_file_command(TopicReplyData, FileId, Msg, FileCommand) -> Transfer = transfer(Msg, FileId), case FileCommand of [<<"init">>] -> validate( [{filemeta, Msg#message.payload}], fun([Meta]) -> - on_init(PacketId, Msg, Transfer, Meta) + on_init(TopicReplyData, Msg, Transfer, Meta) end ); [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> @@ -210,14 +220,14 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> validate( [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], fun([FinalSize, FinalChecksum]) -> - on_fin(PacketId, Msg, Transfer, FinalSize, FinalChecksum) + on_fin(TopicReplyData, Msg, Transfer, FinalSize, FinalChecksum) end ); [<<"abort">>] -> - on_abort(Msg, Transfer); + on_abort(TopicReplyData, Msg, Transfer); [OffsetBin] -> validate([{offset, OffsetBin}], fun([Offset]) -> - on_segment(PacketId, Msg, Transfer, Offset, undefined) + on_segment(TopicReplyData, Msg, Transfer, Offset, undefined) end); [OffsetBin, ChecksumBin] -> validate( @@ -226,16 +236,16 @@ on_file_command(PacketId, FileId, Msg, FileCommand) -> validate( [{integrity, Msg#message.payload, Checksum}], fun(_) -> - on_segment(PacketId, Msg, Transfer, Offset, Checksum) + on_segment(TopicReplyData, Msg, Transfer, Offset, Checksum) end ) end ); _ -> - ?RC_UNSPECIFIED_ERROR + ?ACK_AND_PUBLISH({error, {invalid_file_command, FileCommand}}) end. -on_init(PacketId, Msg, Transfer, Meta) -> +on_init(#{packet_id := PacketId}, Msg, Transfer, Meta) -> ?tp(info, "file_transfer_init", #{ mqtt_msg => Msg, packet_id => PacketId, @@ -245,16 +255,13 @@ on_init(PacketId, Msg, Transfer, Meta) -> %% Currently synchronous. %% If we want to make it async, we need to use `emqx_ft_async_reply`, %% like in `on_fin`. - case store_filemeta(Transfer, Meta) of - ok -> ?RC_SUCCESS; - {error, _} -> ?RC_UNSPECIFIED_ERROR - end. + ?ACK_AND_PUBLISH(store_filemeta(Transfer, Meta)). -on_abort(_Msg, _FileId) -> +on_abort(_TopicReplyData, _Msg, _FileId) -> %% TODO - ?RC_SUCCESS. + ?ACK_AND_PUBLISH(ok). -on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> +on_segment(#{packet_id := PacketId}, Msg, Transfer, Offset, Checksum) -> ?tp(info, "file_transfer_segment", #{ mqtt_msg => Msg, packet_id => PacketId, @@ -266,12 +273,9 @@ on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> %% Currently synchronous. %% If we want to make it async, we need to use `emqx_ft_async_reply`, %% like in `on_fin`. - case store_segment(Transfer, Segment) of - ok -> ?RC_SUCCESS; - {error, _} -> ?RC_UNSPECIFIED_ERROR - end. + ?ACK_AND_PUBLISH(store_segment(Transfer, Segment)). -on_fin(PacketId, Msg, Transfer, FinalSize, FinalChecksum) -> +on_fin(#{packet_id := PacketId} = TopicReplyData, Msg, Transfer, FinalSize, FinalChecksum) -> ?tp(info, "file_transfer_fin", #{ mqtt_msg => Msg, packet_id => PacketId, @@ -280,30 +284,94 @@ on_fin(PacketId, Msg, Transfer, FinalSize, FinalChecksum) -> checksum => FinalChecksum }), %% TODO: handle checksum? Do we need it? - emqx_ft_async_reply:with_new_packet( + with_new_packet( + TopicReplyData, PacketId, fun() -> case assemble(Transfer, FinalSize, FinalChecksum) of ok -> - ?RC_SUCCESS; - %% Assembling started, packet will be acked by monitor or timeout + ?ACK_AND_PUBLISH(ok); + %% Assembling started, packet will be acked/replied by monitor or timeout {async, Pid} -> - ok = register_async_reply(Pid, PacketId), - ok = emqx_ft_storage:kickoff(Pid), - undefined; - {error, _} -> - ?RC_UNSPECIFIED_ERROR + register_async_worker(Pid, TopicReplyData); + {error, _} = Error -> + ?ACK_AND_PUBLISH(Error) end - end, - undefined + end ). -register_async_reply(Pid, PacketId) -> +register_async_worker(Pid, #{mode := Mode, packet_id := PacketId} = TopicReplyData) -> MRef = erlang:monitor(process, Pid), TRef = erlang:start_timer( - emqx_ft_conf:assemble_timeout(), self(), ?FT_EVENT({MRef, PacketId}) + emqx_ft_conf:assemble_timeout(), self(), ?FT_EVENT({MRef, TopicReplyData}) ), - ok = emqx_ft_async_reply:register(PacketId, MRef, TRef). + case Mode of + async -> + ok = emqx_ft_async_reply:register(MRef, TRef, TopicReplyData), + ok = emqx_ft_storage:kickoff(Pid), + ?ACK(ok); + sync -> + ok = emqx_ft_async_reply:register(PacketId, MRef, TRef, TopicReplyData), + ok = emqx_ft_storage:kickoff(Pid), + ?DELAY_ACK + end. + +topic_reply_data(Mode, From, PacketId, #message{topic = Topic, headers = Headers}) -> + Props = maps:get(properties, Headers, #{}), + #{ + mode => Mode, + clientid => From, + command_topic => Topic, + correlation_data => maps:get('Correlation-Data', Props, undefined), + response_topic => maps:get('Response-Topic', Props, undefined), + packet_id => PacketId + }. + +maybe_publish_response(?DELAY_ACK, _TopicReplyData) -> + undefined; +maybe_publish_response(?ACK(Result), _TopicReplyData) -> + result_to_rc(Result); +maybe_publish_response(?ACK_AND_PUBLISH(Result), TopicReplyData) -> + publish_response(Result, TopicReplyData). + +publish_response(Result, #{ + clientid := ClientId, + command_topic := CommandTopic, + correlation_data := CorrelationData, + response_topic := ResponseTopic, + packet_id := PacketId +}) -> + ResultCode = result_to_rc(Result), + Response = encode_response(#{ + topic => CommandTopic, + packet_id => PacketId, + reason_code => ResultCode, + reason_description => emqx_ft_error:format(Result) + }), + Payload = emqx_utils_json:encode(Response), + Topic = emqx_maybe:define(ResponseTopic, response_topic(ClientId)), + Msg = emqx_message:make( + emqx_guid:gen(), + undefined, + ?QOS_1, + Topic, + Payload, + #{}, + #{properties => response_properties(CorrelationData)} + ), + _ = emqx_broker:publish(Msg), + ResultCode. + +response_properties(undefined) -> #{}; +response_properties(CorrelationData) -> #{'Correlation-Data' => CorrelationData}. + +response_topic(ClientId) -> + <<"$file-response/", (clientid_to_binary(ClientId))/binary>>. + +result_to_rc(ok) -> + ?RC_SUCCESS; +result_to_rc({error, _}) -> + ?RC_UNSPECIFIED_ERROR. store_filemeta(Transfer, Segment) -> try @@ -347,9 +415,9 @@ validate(Validations, Fun) -> case do_validate(Validations, []) of {ok, Parsed} -> Fun(Parsed); - {error, Reason} -> + {error, Reason} = Error -> ?tp(info, "client_violated_protocol", #{reason => Reason}), - ?RC_UNSPECIFIED_ERROR + ?ACK_AND_PUBLISH(Error) end. do_validate([], Parsed) -> @@ -416,19 +484,18 @@ clientid_to_binary(A) when is_atom(A) -> clientid_to_binary(B) when is_binary(B) -> B. -reason_to_rc(Reason) -> - case map_down_reason(Reason) of - ok -> ?RC_SUCCESS; - {error, _} -> ?RC_UNSPECIFIED_ERROR - end. - -map_down_reason(normal) -> +down_reason_to_result(normal) -> ok; -map_down_reason(shutdown) -> +down_reason_to_result(shutdown) -> ok; -map_down_reason({shutdown, Result}) -> +down_reason_to_result({shutdown, Result}) -> Result; -map_down_reason(noproc) -> +down_reason_to_result(noproc) -> {error, noproc}; -map_down_reason(Error) -> +down_reason_to_result(Error) -> {error, {internal_error, Error}}. + +with_new_packet(#{mode := async}, _PacketId, Fun) -> + Fun(); +with_new_packet(#{mode := sync}, PacketId, Fun) -> + emqx_ft_async_reply:with_new_packet(PacketId, Fun, undefined). diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl index 0d9e86a49..b625d0ffe 100644 --- a/apps/emqx_ft/src/emqx_ft_assembler.erl +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -156,12 +156,16 @@ handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := % Currently, race is possible between getting segment info from the remote node and % this node garbage collecting the segment itself. % TODO: pipelining - % TODO: better error handling - {ok, Content} = pread(Node, Segment, St), - case emqx_ft_storage_exporter:write(Export, Content) of - {ok, NExport} -> - {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; - {error, _} = Error -> + case pread(Node, Segment, St) of + {ok, Content} -> + case emqx_ft_storage_exporter:write(Export, Content) of + {ok, NExport} -> + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}, maps:remove(export, St)} + end; + {error, ReadError} -> + Error = {error, {read_segment, ReadError}}, {stop, {shutdown, Error}, maps:remove(export, St)} end; handle_event(internal, _, {assemble, []}, St = #{}) -> diff --git a/apps/emqx_ft/src/emqx_ft_async_reply.erl b/apps/emqx_ft/src/emqx_ft_async_reply.erl index 501f91629..e099196f9 100644 --- a/apps/emqx_ft/src/emqx_ft_async_reply.erl +++ b/apps/emqx_ft/src/emqx_ft_async_reply.erl @@ -27,6 +27,7 @@ -export([ register/3, + register/4, take_by_mref/1, with_new_packet/3, deregister_all/1 @@ -42,12 +43,14 @@ -define(MON_TAB, emqx_ft_async_mons). -define(MON_KEY(MRef), ?MON_KEY(self(), MRef)). -define(MON_KEY(ChannelPid, MRef), {ChannelPid, MRef}). +-define(MON_RECORD(KEY, PACKET_ID, TREF, DATA), {KEY, PACKET_ID, TREF, DATA}). %% async worker monitors by packet ids -define(PACKET_TAB, emqx_ft_async_packets). -define(PACKET_KEY(PacketId), ?PACKET_KEY(self(), PacketId)). -define(PACKET_KEY(ChannelPid, PacketId), {ChannelPid, PacketId}). +-define(PACKET_RECORD(KEY, MREF, DATA), {KEY, MREF, DATA}). %%-------------------------------------------------------------------- %% API @@ -66,10 +69,15 @@ create_tables() -> ok = emqx_utils_ets:new(?PACKET_TAB, EtsOptions), ok. --spec register(packet_id(), mon_ref(), timer_ref()) -> ok. -register(PacketId, MRef, TRef) -> - _ = ets:insert(?PACKET_TAB, {?PACKET_KEY(PacketId), MRef}), - _ = ets:insert(?MON_TAB, {?MON_KEY(MRef), PacketId, TRef}), +-spec register(packet_id(), mon_ref(), timer_ref(), term()) -> ok. +register(PacketId, MRef, TRef, Data) -> + _ = ets:insert(?PACKET_TAB, ?PACKET_RECORD(?PACKET_KEY(PacketId), MRef, Data)), + _ = ets:insert(?MON_TAB, ?MON_RECORD(?MON_KEY(MRef), PacketId, TRef, Data)), + ok. + +-spec register(mon_ref(), timer_ref(), term()) -> ok. +register(MRef, TRef, Data) -> + _ = ets:insert(?MON_TAB, ?MON_RECORD(?MON_KEY(MRef), undefined, TRef, Data)), ok. -spec with_new_packet(packet_id(), fun(() -> any()), any()) -> any(). @@ -79,12 +87,12 @@ with_new_packet(PacketId, Fun, Default) -> false -> Fun() end. --spec take_by_mref(mon_ref()) -> {ok, packet_id(), timer_ref()} | not_found. +-spec take_by_mref(mon_ref()) -> {ok, packet_id() | undefined, timer_ref(), term()} | not_found. take_by_mref(MRef) -> case ets:take(?MON_TAB, ?MON_KEY(MRef)) of - [{_, PacketId, TRef}] -> - _ = ets:delete(?PACKET_TAB, ?PACKET_KEY(PacketId)), - {ok, PacketId, TRef}; + [?MON_RECORD(_, PacketId, TRef, Data)] -> + PacketId =/= undefined andalso ets:delete(?PACKET_TAB, ?PACKET_KEY(PacketId)), + {ok, PacketId, TRef, Data}; [] -> not_found end. @@ -104,11 +112,11 @@ info() -> %%------------------------------------------------------------------- deregister_packets(ChannelPid) when is_pid(ChannelPid) -> - MS = [{{?PACKET_KEY(ChannelPid, '_'), '_'}, [], [true]}], + MS = [{?PACKET_RECORD(?PACKET_KEY(ChannelPid, '_'), '_', '_'), [], [true]}], _ = ets:select_delete(?PACKET_TAB, MS), ok. deregister_mons(ChannelPid) -> - MS = [{{?MON_KEY(ChannelPid, '_'), '_', '_'}, [], [true]}], + MS = [{?MON_RECORD(?MON_KEY(ChannelPid, '_'), '_', '_', '_'), [], [true]}], _ = ets:select_delete(?MON_TAB, MS), ok. diff --git a/apps/emqx_ft/src/emqx_ft_error.erl b/apps/emqx_ft/src/emqx_ft_error.erl new file mode 100644 index 000000000..06d575ede --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_error.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc File Transfer error description module + +-module(emqx_ft_error). + +-export([format/1]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +format(ok) -> <<"success">>; +format({error, Reason}) -> format_error_reson(Reason). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +format_error_reson(Reason) when is_atom(Reason) -> + atom_to_binary(Reason, utf8); +format_error_reson({ErrorKind, _}) when is_atom(ErrorKind) -> + atom_to_binary(ErrorKind, utf8); +format_error_reson(_Reason) -> + <<"internal_error">>. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index d9187f14e..e6202b4f6 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -26,7 +26,7 @@ -export([schema/1]). %% Utilities --export([backend/1]). +-export([backend/1, encode/2, decode/2]). %% Test-only helpers -export([translate/1]). @@ -282,6 +282,16 @@ schema(filemeta) -> {segments_ttl, hoconsc:mk(pos_integer())}, {user_data, hoconsc:mk(json_value())} ] + }; +schema(command_response) -> + #{ + roots => [ + {vsn, hoconsc:mk(string(), #{default => <<"0.1">>})}, + {topic, hoconsc:mk(string())}, + {packet_id, hoconsc:mk(pos_integer())}, + {reason_code, hoconsc:mk(non_neg_integer())}, + {reason_description, hoconsc:mk(binary())} + ] }. validator(filename) -> @@ -345,6 +355,27 @@ backend(Config) -> emit_enabled(Type, BConf = #{enable := Enabled}) -> Enabled andalso throw({Type, BConf}). +decode(SchemaName, Payload) when is_binary(Payload) -> + case emqx_utils_json:safe_decode(Payload, [return_maps]) of + {ok, Map} -> + decode(SchemaName, Map); + {error, Error} -> + {error, {invalid_filemeta_json, Error}} + end; +decode(SchemaName, Map) when is_map(Map) -> + Schema = schema(SchemaName), + try + Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), + {ok, Meta} + catch + throw:{_Schema, Errors} -> + {error, {invalid_filemeta, Errors}} + end. + +encode(SchemaName, Map = #{}) -> + Schema = schema(SchemaName), + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Map), #{}). + %% Test-only helpers -spec translate(emqx_config:raw_config()) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index bc1b5fb4d..886dc27f6 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -195,7 +195,7 @@ verify_checksum(Ctx, {Algo, Digest} = Checksum) -> Digest -> {ok, Checksum}; Mismatch -> - {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + {error, {checksum_mismatch, Algo, binary:encode_hex(Mismatch)}} end; verify_checksum(Ctx, undefined) -> Digest = crypto:hash_final(Ctx), diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 5d0395989..0102756ca 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -145,7 +145,7 @@ store_filemeta(Storage, Transfer, Meta) -> % We won't see conflicts in case of concurrent `store_filemeta` % requests. It's rather odd scenario so it's fine not to worry % about it too much now. - {error, conflict}; + {error, filemeta_conflict}; {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)); {error, _} = Error -> diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl index 7da9ccf69..6b6437971 100644 --- a/apps/emqx_ft/test/emqx_ft_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -31,7 +31,8 @@ all() -> [ - {group, single_node}, + {group, async_mode}, + {group, sync_mode}, {group, cluster} ]. @@ -50,7 +51,14 @@ groups() -> t_nasty_filenames, t_no_meta, t_no_segment, - t_simple_transfer + t_simple_transfer, + t_assemble_timeout + ]}, + {async_mode, [], [ + {group, single_node} + ]}, + {sync_mode, [], [ + {group, single_node} ]}, {cluster, [], [ t_switch_node, @@ -72,9 +80,10 @@ init_per_suite(Config) -> emqx_ft_test_helpers:local_storage(Config), #{<<"local">> => #{<<"segments">> => #{<<"gc">> => #{<<"interval">> => 0}}}} ), + FTConfig = emqx_ft_test_helpers:config(Storage, #{<<"assemble_timeout">> => <<"2s">>}), Apps = emqx_cth_suite:start( [ - {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}} + {emqx_ft, #{config => FTConfig}} ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), @@ -85,7 +94,10 @@ end_per_suite(Config) -> ok. init_per_testcase(Case, Config) -> - ClientId = atom_to_binary(Case), + ClientId = iolist_to_binary([ + atom_to_binary(Case), <<"-">>, emqx_ft_test_helpers:unique_binary_string() + ]), + ok = set_client_specific_ft_dirs(ClientId, Config), case ?config(group, Config) of cluster -> [{clientid, ClientId} | Config]; @@ -103,6 +115,10 @@ init_per_group(Group = cluster, Config) -> Cluster = mk_cluster_specs(Config), Nodes = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}), [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(_Group = async_mode, Config) -> + [{mode, sync} | Config]; +init_per_group(_Group = sync_mode, Config) -> + [{mode, async} | Config]; init_per_group(Group, Config) -> [{group, Group} | Config]. @@ -127,7 +143,7 @@ mk_cluster_specs(_Config) -> ]. %%-------------------------------------------------------------------- -%% Tests +%% Single node tests %%-------------------------------------------------------------------- t_invalid_topic_format(Config) -> @@ -171,32 +187,32 @@ t_invalid_fileid(Config) -> C = ?config(client, Config), ?assertRCName( unspecified_error, - emqtt:publish(C, <<"$file//init">>, <<>>, 1) + emqtt:publish(C, mk_init_topic(Config, <<>>), <<>>, 1) ). t_invalid_filename(Config) -> C = ?config(client, Config), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f1">>), encode_meta(meta(".", <<>>)), 1) + emqtt:publish(C, mk_init_topic(Config, <<"f1">>), encode_meta(meta(".", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("..", <<>>)), 1) + emqtt:publish(C, mk_init_topic(Config, <<"f2">>), encode_meta(meta("..", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("../nice", <<>>)), 1) + emqtt:publish(C, mk_init_topic(Config, <<"f2">>), encode_meta(meta("../nice", <<>>)), 1) ), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) + emqtt:publish(C, mk_init_topic(Config, <<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) ), ?assertRCName( unspecified_error, emqtt:publish( C, - mk_init_topic(<<"f4">>), + mk_init_topic(Config, <<"f4">>), encode_meta(meta(lists:duplicate(1000, $A), <<>>)), 1 ) @@ -204,6 +220,7 @@ t_invalid_filename(Config) -> t_simple_transfer(Config) -> C = ?config(client, Config), + ClientId = ?config(clientid, Config), Filename = "topsecret.pdf", FileId = <<"f1">>, @@ -214,22 +231,24 @@ t_simple_transfer(Config) -> ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + emqtt:publish(C, mk_init_topic(Config, FileId), encode_meta(Meta), 1) ), lists:foreach( fun({Chunk, Offset}) -> ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), - ?assertRCName( - success, - emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ?assertEqual( + ok, + emqx_ft_test_helpers:fin_result( + mode(Config), ClientId, C, mk_fin_topic(Config, FileId, Filesize) + ) ), [Export] = list_files(?config(clientid, Config)), @@ -238,7 +257,7 @@ t_simple_transfer(Config) -> read_export(Export) ). -t_nasty_clientids_fileids(_Config) -> +t_nasty_clientids_fileids(Config) -> Transfers = [ {<<".">>, <<".">>}, {<<"🌚"/utf8>>, <<"🌝"/utf8>>}, @@ -249,15 +268,16 @@ t_nasty_clientids_fileids(_Config) -> ok = lists:foreach( fun({ClientId, FileId}) -> - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), + Data = ClientId, + ok = emqx_ft_test_helpers:upload_file(mode(Config), ClientId, FileId, "justfile", Data), [Export] = list_files(ClientId), ?assertMatch(#{meta := #{name := "justfile"}}, Export), - ?assertEqual({ok, ClientId}, read_export(Export)) + ?assertEqual({ok, Data}, read_export(Export)) end, Transfers ). -t_nasty_filenames(_Config) -> +t_nasty_filenames(Config) -> Filenames = [ {<<"nasty1">>, "146%"}, {<<"nasty2">>, "🌚"}, @@ -267,7 +287,7 @@ t_nasty_filenames(_Config) -> ok = lists:foreach( fun({ClientId, Filename}) -> FileId = unicode:characters_to_binary(Filename), - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Filename, FileId), + ok = emqx_ft_test_helpers:upload_file(mode(Config), ClientId, FileId, Filename, FileId), [Export] = list_files(ClientId), ?assertMatch(#{meta := #{name := Filename}}, Export), ?assertEqual({ok, FileId}, read_export(Export)) @@ -285,34 +305,36 @@ t_meta_conflict(Config) -> ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + emqtt:publish(C, mk_init_topic(Config, FileId), encode_meta(Meta), 1) ), ConflictMeta = Meta#{name => "conflict.pdf"}, ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(FileId), encode_meta(ConflictMeta), 1) + emqtt:publish(C, mk_init_topic(Config, FileId), encode_meta(ConflictMeta), 1) ). t_no_meta(Config) -> C = ?config(client, Config), + ClientId = ?config(clientid, Config), FileId = <<"f1">>, Data = <<"first">>, ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, 0), Data, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, 0), Data, 1) ), - ?assertRCName( - unspecified_error, - emqtt:publish(C, mk_fin_topic(FileId, 42), <<>>, 1) + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:fin_result(mode(Config), ClientId, C, mk_fin_topic(Config, FileId, 42)) ). t_no_segment(Config) -> C = ?config(client, Config), + ClientId = ?config(clientid, Config), Filename = "topsecret.pdf", FileId = <<"f1">>, @@ -323,23 +345,25 @@ t_no_segment(Config) -> ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + emqtt:publish(C, mk_init_topic(Config, FileId), encode_meta(Meta), 1) ), lists:foreach( fun({Chunk, Offset}) -> ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset), Chunk, 1) ) end, %% Skip the first segment tl(with_offsets(Data)) ), - ?assertRCName( - unspecified_error, - emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:fin_result( + mode(Config), ClientId, C, mk_fin_topic(Config, FileId, Filesize) + ) ). t_invalid_meta(Config) -> @@ -352,17 +376,18 @@ t_invalid_meta(Config) -> MetaPayload = emqx_utils_json:encode(Meta), ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(Config, FileId), MetaPayload, 1) ), %% Invalid JSON ?assertRCName( unspecified_error, - emqtt:publish(C, mk_init_topic(FileId), <<"{oops;">>, 1) + emqtt:publish(C, mk_init_topic(Config, FileId), <<"{oops;">>, 1) ). t_invalid_checksum(Config) -> C = ?config(client, Config), + ClientId = ?config(clientid, Config), Filename = "topsecret.pdf", FileId = <<"f1">>, @@ -374,35 +399,39 @@ t_invalid_checksum(Config) -> ?assertRCName( success, - emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + emqtt:publish(C, mk_init_topic(Config, FileId), MetaPayload, 1) ), lists:foreach( fun({Chunk, Offset}) -> ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset), Chunk, 1) ) end, with_offsets(Data) ), % Send `fin` w/o checksum, should fail since filemeta checksum is invalid - FinTopic = mk_fin_topic(FileId, Filesize), - ?assertRCName( - unspecified_error, - emqtt:publish(C, FinTopic, <<>>, 1) + FinTopic = mk_fin_topic(Config, FileId, Filesize), + + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:fin_result(mode(Config), ClientId, C, FinTopic) ), % Send `fin` with the correct checksum Checksum = binary:encode_hex(sha256(Data)), - ?assertRCName( - success, - emqtt:publish(C, <>, <<>>, 1) + ?assertEqual( + ok, + emqx_ft_test_helpers:fin_result( + mode(Config), ClientId, C, <> + ) ). t_corrupted_segment_retry(Config) -> C = ?config(client, Config), + ClientId = ?config(clientid, Config), Filename = "corruption.pdf", FileId = <<"4242-4242">>, @@ -421,35 +450,89 @@ t_corrupted_segment_retry(Config) -> Meta = #{size := Filesize} = meta(Filename, Data), - ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)), + ?assertRCName(success, emqtt:publish(C, mk_init_topic(Config, FileId), encode_meta(Meta), 1)), ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset1, Checksum1), Seg1, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset1, Checksum1), Seg1, 1) ), % segment is corrupted ?assertRCName( unspecified_error, - emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), <>, 1) + emqtt:publish( + C, mk_segment_topic(Config, FileId, Offset2, Checksum2), <>, 1 + ) ), % retry ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), Seg2, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset2, Checksum2), Seg2, 1) ), ?assertRCName( success, - emqtt:publish(C, mk_segment_topic(FileId, Offset3, Checksum3), Seg3, 1) + emqtt:publish(C, mk_segment_topic(Config, FileId, Offset3, Checksum3), Seg3, 1) ), - ?assertRCName( - success, - emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ?assertEqual( + ok, + emqx_ft_test_helpers:fin_result( + mode(Config), ClientId, C, mk_fin_topic(Config, FileId, Filesize) + ) ). +t_assemble_crash(Config) -> + C = ?config(client, Config), + + meck:new(emqx_ft_storage_fs), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _, _) -> meck:exception(error, oops) end), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) + ), + + meck:unload(emqx_ft_storage_fs). + +t_assemble_timeout(Config) -> + C = ?config(client, Config), + ClientId = ?config(clientid, Config), + + SleepForever = fun() -> + Ref = make_ref(), + receive + Ref -> ok + end + end, + + ok = meck:new(emqx_ft_storage, [passthrough]), + ok = meck:expect(emqx_ft_storage, assemble, fun(_, _, _) -> + {async, spawn_link(SleepForever)} + end), + + {Time, Res} = timer:tc( + fun() -> + emqx_ft_test_helpers:fin_result( + mode(Config), ClientId, C, <<"$file/someid/fin/9999999">> + ) + end + ), + + ok = meck:unload(emqx_ft_storage), + + ?assertEqual( + {error, unspecified_error}, + Res + ), + + ?assert(2_000_000 < Time). + +%%-------------------------------------------------------------------- +%% Cluster tests +%%-------------------------------------------------------------------- + t_switch_node(Config) -> [Node | _] = ?config(cluster_nodes, Config), AdditionalNodePort = emqx_ft_test_helpers:tcp_port(Node), @@ -471,11 +554,11 @@ t_switch_node(Config) -> ?assertRCName( success, - emqtt:publish(C1, mk_init_topic(FileId), encode_meta(Meta), 1) + emqtt:publish(C1, mk_init_topic(Config, FileId), encode_meta(Meta), 1) ), ?assertRCName( success, - emqtt:publish(C1, mk_segment_topic(FileId, Offset0), Data0, 1) + emqtt:publish(C1, mk_segment_topic(Config, FileId, Offset0), Data0, 1) ), %% Then, switch the client to the main node @@ -487,16 +570,16 @@ t_switch_node(Config) -> ?assertRCName( success, - emqtt:publish(C2, mk_segment_topic(FileId, Offset1), Data1, 1) + emqtt:publish(C2, mk_segment_topic(Config, FileId, Offset1), Data1, 1) ), ?assertRCName( success, - emqtt:publish(C2, mk_segment_topic(FileId, Offset2), Data2, 1) + emqtt:publish(C2, mk_segment_topic(Config, FileId, Offset2), Data2, 1) ), ?assertRCName( success, - emqtt:publish(C2, mk_fin_topic(FileId, Filesize), <<>>, 1) + emqtt:publish(C2, mk_fin_topic(Config, FileId, Filesize), <<>>, 1) ), ok = emqtt:stop(C2), @@ -509,17 +592,6 @@ t_switch_node(Config) -> read_export(Export) ). -t_assemble_crash(Config) -> - C = ?config(client, Config), - - meck:new(emqx_ft_storage_fs), - meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _, _) -> meck:exception(error, oops) end), - - ?assertRCName( - unspecified_error, - emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) - ). - t_unreliable_migrating_client(Config) -> NodeSelf = node(), [Node1, Node2] = ?config(cluster_nodes, Config), @@ -543,10 +615,10 @@ t_unreliable_migrating_client(Config) -> {fun connect_mqtt_client/2, [NodeSelf]}, % Send filemeta and 3 initial segments % (assuming client chose 100 bytes as a desired segment size) - {fun send_filemeta/2, [Meta]}, - {fun send_segment/3, [0, 100]}, - {fun send_segment/3, [100, 100]}, - {fun send_segment/3, [200, 100]}, + {fun send_filemeta/3, [Config, Meta]}, + {fun send_segment/4, [Config, 0, 100]}, + {fun send_segment/4, [Config, 100, 100]}, + {fun send_segment/4, [Config, 200, 100]}, % Disconnect the client cleanly {fun stop_mqtt_client/1, []}, % Connect to the broker on `Node1` @@ -555,27 +627,27 @@ t_unreliable_migrating_client(Config) -> % Client forgot the state for some reason and started the transfer again. % (assuming this is usual for a client on a device that was rebooted) {fun connect_mqtt_client/2, [Node2]}, - {fun send_filemeta/2, [Meta]}, + {fun send_filemeta/3, [Config, Meta]}, % This time it chose 200 bytes as a segment size - {fun send_segment/3, [0, 200]}, - {fun send_segment/3, [200, 200]}, + {fun send_segment/4, [Config, 0, 200]}, + {fun send_segment/4, [Config, 200, 200]}, % But now it downscaled back to 100 bytes segments - {fun send_segment/3, [400, 100]}, + {fun send_segment/4, [Config, 400, 100]}, % Client lost connectivity and reconnected % (also had last few segments unacked and decided to resend them) {fun connect_mqtt_client/2, [Node2]}, - {fun send_segment/3, [200, 200]}, - {fun send_segment/3, [400, 200]}, + {fun send_segment/4, [Config, 200, 200]}, + {fun send_segment/4, [Config, 400, 200]}, % Client lost connectivity and reconnected, this time to another node % (also had last segment unacked and decided to resend it) {fun connect_mqtt_client/2, [Node1]}, - {fun send_segment/3, [400, 200]}, - {fun send_segment/3, [600, eof]}, - {fun send_finish/1, []}, + {fun send_segment/4, [Config, 400, 200]}, + {fun send_segment/4, [Config, 600, eof]}, + {fun send_finish/2, [Config]}, % Client lost connectivity and reconnected, this time to the current node % (client had `fin` unacked and decided to resend it) {fun connect_mqtt_client/2, [NodeSelf]}, - {fun send_finish/1, []} + {fun send_finish/2, [Config]} ], _Context = run_commands(Commands, Context), @@ -621,8 +693,8 @@ t_concurrent_fins(Config) -> Context1 = run_commands( [ {fun connect_mqtt_client/2, [Node1]}, - {fun send_filemeta/2, [Meta]}, - {fun send_segment/3, [0, 100]}, + {fun send_filemeta/3, [Config, Meta]}, + {fun send_segment/4, [Config, 0, 100]}, {fun stop_mqtt_client/1, []} ], Context0 @@ -634,7 +706,7 @@ t_concurrent_fins(Config) -> run_commands( [ {fun connect_mqtt_client/2, [Node]}, - {fun send_finish/1, []} + {fun send_finish/2, [Config]} ], Context1 ) @@ -708,14 +780,16 @@ disown_mqtt_client(Context = #{client := Client}) -> disown_mqtt_client(Context = #{}) -> Context. -send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> +send_filemeta(Config, Meta, Context = #{client := Client, fileid := FileId}) -> ?assertRCName( success, - emqtt:publish(Client, mk_init_topic(FileId), encode_meta(Meta), 1) + emqtt:publish(Client, mk_init_topic(Config, FileId), encode_meta(Meta), 1) ), Context. -send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> +send_segment( + Config, Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload} +) -> Data = case Size of eof -> @@ -725,14 +799,14 @@ send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, paylo end, ?assertRCName( success, - emqtt:publish(Client, mk_segment_topic(FileId, Offset), Data, 1) + emqtt:publish(Client, mk_segment_topic(Config, FileId, Offset), Data, 1) ), Context. -send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> +send_finish(Config, Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> ?assertRCName( success, - emqtt:publish(Client, mk_fin_topic(FileId, Filesize), <<>>, 1) + emqtt:publish(Client, mk_fin_topic(Config, FileId, Filesize), <<>>, 1) ), Context. @@ -749,23 +823,30 @@ fs_exported_file_attributes(FSExports) -> lists:sort(FSExports) ). -mk_init_topic(FileId) -> - <<"$file/", FileId/binary, "/init">>. +mk_init_topic(Config, FileId) -> + RequestTopicPrefix = request_topic_prefix(Config, FileId), + <>. -mk_segment_topic(FileId, Offset) when is_integer(Offset) -> - mk_segment_topic(FileId, integer_to_binary(Offset)); -mk_segment_topic(FileId, Offset) when is_binary(Offset) -> - <<"$file/", FileId/binary, "/", Offset/binary>>. +mk_segment_topic(Config, FileId, Offset) when is_integer(Offset) -> + mk_segment_topic(Config, FileId, integer_to_binary(Offset)); +mk_segment_topic(Config, FileId, Offset) when is_binary(Offset) -> + RequestTopicPrefix = request_topic_prefix(Config, FileId), + <>. -mk_segment_topic(FileId, Offset, Checksum) when is_integer(Offset) -> - mk_segment_topic(FileId, integer_to_binary(Offset), Checksum); -mk_segment_topic(FileId, Offset, Checksum) when is_binary(Offset) -> - <<"$file/", FileId/binary, "/", Offset/binary, "/", Checksum/binary>>. +mk_segment_topic(Config, FileId, Offset, Checksum) when is_integer(Offset) -> + mk_segment_topic(Config, FileId, integer_to_binary(Offset), Checksum); +mk_segment_topic(Config, FileId, Offset, Checksum) when is_binary(Offset) -> + RequestTopicPrefix = request_topic_prefix(Config, FileId), + <>. -mk_fin_topic(FileId, Size) when is_integer(Size) -> - mk_fin_topic(FileId, integer_to_binary(Size)); -mk_fin_topic(FileId, Size) when is_binary(Size) -> - <<"$file/", FileId/binary, "/fin/", Size/binary>>. +mk_fin_topic(Config, FileId, Size) when is_integer(Size) -> + mk_fin_topic(Config, FileId, integer_to_binary(Size)); +mk_fin_topic(Config, FileId, Size) when is_binary(Size) -> + RequestTopicPrefix = request_topic_prefix(Config, FileId), + <>. + +request_topic_prefix(Config, FileId) -> + emqx_ft_test_helpers:request_topic_prefix(mode(Config), FileId). with_offsets(Items) -> {List, _} = lists:mapfoldl( @@ -799,3 +880,17 @@ list_files(ClientId) -> read_export(#{path := AbsFilepath}) -> % TODO: only works for the local filesystem exporter right now file:read_file(AbsFilepath). + +set_client_specific_ft_dirs(ClientId, Config) -> + FTRoot = emqx_ft_test_helpers:ft_root(Config), + ok = emqx_config:put( + [file_transfer, storage, local, segments, root], + filename:join([FTRoot, ClientId, segments]) + ), + ok = emqx_config:put( + [file_transfer, storage, local, exporter, local, root], + filename:join([FTRoot, ClientId, exports]) + ). + +mode(Config) -> + proplists:get_value(mode, Config, sync). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index ae8a5c01c..092927d70 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -85,7 +85,7 @@ t_list_files(Config) -> FileId = <<"f1">>, Node = lists:last(test_nodes(Config)), - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), + ok = emqx_ft_test_helpers:upload_file(sync, ClientId, FileId, "f1", <<"data">>, Node), {ok, 200, #{<<"files">> := Files}} = request_json(get, uri(["file_transfer", "files"]), Config), @@ -114,7 +114,7 @@ t_download_transfer(Config) -> Nodes = [Node | _] = test_nodes(Config), NodeUpload = lists:last(Nodes), - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, NodeUpload), + ok = emqx_ft_test_helpers:upload_file(sync, ClientId, FileId, "f1", <<"data">>, NodeUpload), ?assertMatch( {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, @@ -185,7 +185,7 @@ t_list_files_paging(Config) -> ], ok = lists:foreach( fun({FileId, Name, Node}) -> - ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>, Node) + ok = emqx_ft_test_helpers:upload_file(sync, ClientId, FileId, Name, <<"data">>, Node) end, Uploads ), diff --git a/apps/emqx_ft/test/emqx_ft_async_reply_SUITE.erl b/apps/emqx_ft/test/emqx_ft_async_reply_SUITE.erl index 78a9b371c..daa83de74 100644 --- a/apps/emqx_ft/test/emqx_ft_async_reply_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_async_reply_SUITE.erl @@ -55,7 +55,7 @@ t_register(_Config) -> PacketId = 1, MRef = make_ref(), TRef = make_ref(), - ok = emqx_ft_async_reply:register(PacketId, MRef, TRef), + ok = emqx_ft_async_reply:register(PacketId, MRef, TRef, somedata), ?assertEqual( undefined, @@ -68,7 +68,7 @@ t_register(_Config) -> ), ?assertEqual( - {ok, PacketId, TRef}, + {ok, PacketId, TRef, somedata}, emqx_ft_async_reply:take_by_mref(MRef) ). @@ -76,7 +76,7 @@ t_process_independence(_Config) -> PacketId = 1, MRef = make_ref(), TRef = make_ref(), - ok = emqx_ft_async_reply:register(PacketId, MRef, TRef), + ok = emqx_ft_async_reply:register(PacketId, MRef, TRef, somedata), Self = self(), @@ -112,10 +112,10 @@ t_take(_Config) -> PacketId = 1, MRef = make_ref(), TRef = make_ref(), - ok = emqx_ft_async_reply:register(PacketId, MRef, TRef), + ok = emqx_ft_async_reply:register(PacketId, MRef, TRef, somedata), ?assertEqual( - {ok, PacketId, TRef}, + {ok, PacketId, TRef, somedata}, emqx_ft_async_reply:take_by_mref(MRef) ), @@ -135,12 +135,12 @@ t_cleanup(_Config) -> TRef0 = make_ref(), MRef1 = make_ref(), TRef1 = make_ref(), - ok = emqx_ft_async_reply:register(PacketId, MRef0, TRef0), + ok = emqx_ft_async_reply:register(PacketId, MRef0, TRef0, somedata0), Self = self(), Pid = spawn_link(fun() -> - ok = emqx_ft_async_reply:register(PacketId, MRef1, TRef1), + ok = emqx_ft_async_reply:register(PacketId, MRef1, TRef1, somedata1), receive kickoff -> ?assertEqual( @@ -149,7 +149,7 @@ t_cleanup(_Config) -> ), ?assertEqual( - {ok, PacketId, TRef1}, + {ok, PacketId, TRef1, somedata1}, emqx_ft_async_reply:take_by_mref(MRef1) ), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 0acdea213..8ce282f6d 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -39,10 +39,10 @@ init_per_testcase(Case, Config) -> ], #{work_dir => emqx_cth_suite:work_dir(Case, Config)} ), - [{suite_apps, Apps} | Config]. + [{apps, Apps} | Config]. end_per_testcase(_Case, Config) -> - ok = emqx_cth_suite:stop(?config(suite_apps, Config)), + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_request_SUITE.erl b/apps/emqx_ft/test/emqx_ft_request_SUITE.erl new file mode 100644 index 000000000..b21917093 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_request_SUITE.erl @@ -0,0 +1,113 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_request_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + {emqx_ft, "file_transfer { enable = true, assemble_timeout = 1s}"} + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +%%------------------------------------------------------------------- +%% Tests +%%------------------------------------------------------------------- + +t_upload_via_requests(_Config) -> + C = emqx_ft_test_helpers:start_client(<<"client">>), + + FileId = <<"f1">>, + Data = <<"hello world">>, + Size = byte_size(Data), + Meta = #{ + name => "test.txt", + expire_at => erlang:system_time(_Unit = second) + 3600, + size => Size + }, + MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)), + MetaTopic = <<"$file/", FileId/binary, "/init">>, + + ?assertMatch( + {ok, #{<<"reason_code">> := 0, <<"topic">> := MetaTopic}}, + request(C, MetaTopic, MetaPayload) + ), + + SegmentTopic = <<"$file/", FileId/binary, "/0">>, + + ?assertMatch( + {ok, #{<<"reason_code">> := 0, <<"topic">> := SegmentTopic}}, + request(C, SegmentTopic, Data) + ), + + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, + + ?assertMatch( + {ok, #{<<"reason_code">> := 0, <<"topic">> := FinTopic}}, + request(C, FinTopic, <<>>) + ). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +request(C, Topic, Request) -> + CorrelaionData = emqx_ft_test_helpers:unique_binary_string(), + ResponseTopic = emqx_ft_test_helpers:unique_binary_string(), + + Properties = #{ + 'Correlation-Data' => CorrelaionData, + 'Response-Topic' => ResponseTopic + }, + Opts = [{qos, 1}], + + {ok, _, _} = emqtt:subscribe(C, ResponseTopic, 1), + {ok, _} = emqtt:publish(C, Topic, Properties, Request, Opts), + + try + receive + {publish, #{ + topic := ResponseTopic, + payload := Payload, + properties := #{'Correlation-Data' := CorrelaionData} + }} -> + {ok, emqx_utils_json:decode(Payload)} + after 1000 -> + {error, timeout} + end + after + emqtt:unsubscribe(C, ResponseTopic) + end. diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl index 9e6050f36..90da824ef 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -38,25 +38,23 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok. -set_special_configs(Config) -> - fun - (emqx_ft) -> - Storage = emqx_ft_test_helpers:local_storage(Config, #{ - exporter => s3, bucket_name => ?config(bucket_name, Config) - }), - emqx_ft_test_helpers:load_config(#{<<"enable">> => true, <<"storage">> => Storage}); - (_) -> - ok - end. - -init_per_testcase(Case, Config0) -> +init_per_testcase(Case, Config) -> ClientId = atom_to_binary(Case), BucketName = create_bucket(), - Config1 = [{bucket_name, BucketName}, {clientid, ClientId} | Config0], - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config1)), - Config1. -end_per_testcase(_Case, _Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + Storage = emqx_ft_test_helpers:local_storage(Config, #{ + exporter => s3, bucket_name => BucketName + }), + WorkDir = filename:join(?config(priv_dir, Config), atom_to_list(Case)), + Apps = emqx_cth_suite:start( + [ + emqx_conf, + {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}} + ], + #{work_dir => WorkDir} + ), + [{apps, Apps}, {bucket_name, BucketName}, {clientid, ClientId} | Config]. +end_per_testcase(_Case, Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 52d372e63..6ca158833 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -81,8 +81,8 @@ end_per_group(_Group, _Config) -> t_multinode_exports(Config) -> [Node1, Node2 | _] = ?config(cluster, Config), - ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), - ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), + ok = emqx_ft_test_helpers:upload_file(sync, <<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), + ok = emqx_ft_test_helpers:upload_file(sync, <<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), ?assertMatch( [ #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl index 217205f6f..f0b658e0d 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -25,11 +25,18 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), - Config. + WorkDir = ?config(priv_dir, Config), + Storage = emqx_ft_test_helpers:local_storage(Config), + Apps = emqx_cth_suite:start( + [ + {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}} + ], + #{work_dir => WorkDir} + ), + [{suite_apps, Apps} | Config]. -end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft]), +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), ok. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index 9e69118c8..efcaa3048 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -24,16 +24,15 @@ -define(S3_HOST, <<"minio">>). -define(S3_PORT, 9000). -env_handler(Config) -> - fun - (emqx_ft) -> - load_config(#{<<"enable">> => true, <<"storage">> => local_storage(Config)}); - (_) -> - ok - end. - config(Storage) -> - #{<<"file_transfer">> => #{<<"enable">> => true, <<"storage">> => Storage}}. + config(Storage, #{}). + +config(Storage, FTOptions0) -> + FTOptions1 = maps:merge( + #{<<"enable">> => true, <<"storage">> => Storage}, + FTOptions0 + ), + #{<<"file_transfer">> => FTOptions1}. local_storage(Config) -> local_storage(Config, #{exporter => local}). @@ -73,7 +72,13 @@ tcp_port(Node) -> Port. root(Config, Node, Tail) -> - iolist_to_binary(filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail])). + iolist_to_binary(filename:join([ft_root(Config), Node | Tail])). + +ft_root(Config) -> + filename:join([?config(priv_dir, Config), "file_transfer"]). + +cleanup_ft_root(Config) -> + file:del_dir_r(emqx_ft_test_helpers:ft_root(Config)). start_client(ClientId) -> start_client(ClientId, node()). @@ -85,11 +90,15 @@ start_client(ClientId, Node) -> Client. upload_file(ClientId, FileId, Name, Data) -> - upload_file(ClientId, FileId, Name, Data, node()). + upload_file(sync, ClientId, FileId, Name, Data). -upload_file(ClientId, FileId, Name, Data, Node) -> +upload_file(Mode, ClientId, FileId, Name, Data) -> + upload_file(Mode, ClientId, FileId, Name, Data, node()). + +upload_file(Mode, ClientId, FileId, Name, Data, Node) -> C1 = start_client(ClientId, Node), + ReqTopicPrefix = request_topic_prefix(Mode, FileId), Size = byte_size(Data), Meta = #{ name => Name, @@ -98,25 +107,53 @@ upload_file(ClientId, FileId, Name, Data, Node) -> }, MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)), - ct:pal("MetaPayload = ~ts", [MetaPayload]), - - MetaTopic = <<"$file/", FileId/binary, "/init">>, + MetaTopic = <>, {ok, #{reason_code_name := success}} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), {ok, #{reason_code_name := success}} = emqtt:publish( - C1, <<"$file/", FileId/binary, "/0">>, Data, 1 + C1, <>, Data, 1 ), - FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, - FinResult = - case emqtt:publish(C1, FinTopic, <<>>, 1) of - {ok, #{reason_code_name := success}} -> - ok; - {ok, #{reason_code_name := Error}} -> - {error, Error} - end, + FinTopic = <>, + FinResult = fin_result(Mode, ClientId, C1, FinTopic), ok = emqtt:stop(C1), FinResult. +fin_result(Mode, ClientId, C, FinTopic) -> + {ok, _, _} = emqtt:subscribe(C, response_topic(ClientId), 1), + case emqtt:publish(C, FinTopic, <<>>, 1) of + {ok, #{reason_code_name := success}} -> + maybe_wait_for_assemble(Mode, ClientId, FinTopic); + {ok, #{reason_code_name := Error}} -> + {error, Error} + end. + +maybe_wait_for_assemble(sync, _ClientId, _FinTopic) -> + ok; +maybe_wait_for_assemble(async, ClientId, FinTopic) -> + ResponseTopic = response_topic(ClientId), + receive + {publish, #{payload := Payload, topic := ResponseTopic}} -> + case emqx_utils_json:decode(Payload) of + #{<<"topic">> := FinTopic, <<"reason_code">> := 0} -> + ok; + #{<<"topic">> := FinTopic, <<"reason_code">> := Code} -> + {error, emqx_reason_codes:name(Code)}; + _ -> + maybe_wait_for_assemble(async, ClientId, FinTopic) + end + end. + +response_topic(ClientId) -> + <<"$file-response/", (to_bin(ClientId))/binary>>. + +request_topic_prefix(sync, FileId) -> + <<"$file/", (to_bin(FileId))/binary>>; +request_topic_prefix(async, FileId) -> + <<"$file-async/", (to_bin(FileId))/binary>>. + +to_bin(Val) -> + iolist_to_binary(Val). + aws_config() -> emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). @@ -129,3 +166,6 @@ pem_privkey() -> "ju0VBj6tOX1y6C0U+85VOM0UU5xqvw==\n" "-----END EC PRIVATE KEY-----\n" >>. + +unique_binary_string() -> + emqx_guid:to_hexstr(emqx_guid:gen()). diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 8dcbe500c..df681b00f 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.26"}, + {vsn, "0.1.27"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index e58e552e2..ed149d1f5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -32,19 +32,16 @@ -type duration() :: non_neg_integer(). -type duration_s() :: non_neg_integer(). -type bytesize() :: pos_integer(). --type comma_separated_list() :: list(). -typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). --typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). -reflect_type([ duration/0, duration_s/0, bytesize/0, - comma_separated_list/0, ip_port/0 ]). -elvis([{elvis_style, dont_repeat_yourself, disable}]). diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 467ac20a2..5e3461c52 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -86,7 +86,6 @@ -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). -define(DEF_IDLE_TIME, timer:seconds(30)). --define(GET_IDLE_TIME(Cfg), maps:get(idle_timeout, Cfg, ?DEF_IDLE_TIME)). -import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). @@ -150,8 +149,7 @@ init( mountpoint => Mountpoint } ), - %% FIXME: it should coap.hearbeat instead of idle_timeout? - Heartbeat = ?GET_IDLE_TIME(Config), + Heartbeat = maps:get(heartbeat, Config, ?DEF_IDLE_TIME), #channel{ ctx = Ctx, conninfo = ConnInfo, @@ -179,8 +177,8 @@ send_request(Channel, Request) -> | {ok, replies(), channel()} | {shutdown, Reason :: term(), channel()} | {shutdown, Reason :: term(), replies(), channel()}. -handle_in(Msg, ChannleT) -> - Channel = ensure_keepalive_timer(ChannleT), +handle_in(Msg, Channel0) -> + Channel = ensure_keepalive_timer(Channel0), case emqx_coap_message:is_request(Msg) of true -> check_auth_state(Msg, Channel); @@ -321,6 +319,9 @@ handle_call(Req, _From, Channel) -> handle_cast(close, Channel) -> ?SLOG(info, #{msg => "close_connection"}), shutdown(normal, Channel); +handle_cast(inc_recv_pkt, Channel) -> + _ = emqx_pd:inc_counter(recv_pkt, 1), + {ok, Channel}; handle_cast(Req, Channel) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Req}), {ok, Channel}. @@ -455,6 +456,13 @@ check_token( Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), {shutdown, normal, Reply, Channel}; true -> + %% hack: since each message request can spawn a new connection + %% process, we can't rely on the `inc_incoming_stats' call in + %% `emqx_gateway_conn:handle_incoming' to properly keep track of + %% bumping incoming requests for an existing channel. Since this + %% number is used by keepalive, we have to bump it inside the + %% requested channel/connection pid so heartbeats actually work. + emqx_gateway_cm:cast(coap, ReqClientId, inc_recv_pkt), call_session(handle_request, Msg, Channel) end; _ -> diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index 755532723..30c176139 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 4459d84f1..c066b84ff 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -83,10 +83,26 @@ init_per_testcase(t_connection_with_authn_failed, Config) -> fun(_) -> {error, bad_username_or_password} end ), Config; +init_per_testcase(t_heartbeat, Config) -> + NewHeartbeat = 800, + OldConf = emqx:get_raw_config([gateway, coap]), + {ok, _} = emqx_gateway_conf:update_gateway( + coap, + OldConf#{<<"heartbeat">> => <<"800ms">>} + ), + [ + {old_conf, OldConf}, + {new_heartbeat, NewHeartbeat} + | Config + ]; init_per_testcase(_, Config) -> ok = meck:new(emqx_access_control, [passthrough]), Config. +end_per_testcase(t_heartbeat, Config) -> + OldConf = ?config(old_conf, Config), + {ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf), + ok; end_per_testcase(_, Config) -> ok = meck:unload(emqx_access_control), Config. @@ -123,13 +139,49 @@ t_connection(_) -> ), %% heartbeat - HeartURI = - ?MQTT_PREFIX ++ - "/connection?clientid=client1&token=" ++ - Token, + {ok, changed, _} = send_heartbeat(Token), - ?LOGT("send heartbeat request:~ts~n", [HeartURI]), - {ok, changed, _} = er_coap_client:request(put, HeartURI), + disconnection(Channel, Token), + + timer:sleep(100), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ) + end, + do(Action). + +t_heartbeat(Config) -> + Heartbeat = ?config(new_heartbeat, Config), + Action = fun(Channel) -> + Token = connection(Channel), + + timer:sleep(100), + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + %% must keep client connection alive + Delay = Heartbeat div 2, + lists:foreach( + fun(_) -> + ?assertMatch({ok, changed, _}, send_heartbeat(Token)), + timer:sleep(Delay) + end, + lists:seq(1, 5) + ), + + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + timer:sleep(Heartbeat * 2), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), disconnection(Channel, Token), @@ -491,6 +543,15 @@ t_connectionless_pubsub(_) -> %%-------------------------------------------------------------------- %% helpers +send_heartbeat(Token) -> + HeartURI = + ?MQTT_PREFIX ++ + "/connection?clientid=client1&token=" ++ + Token, + + ?LOGT("send heartbeat request:~ts~n", [HeartURI]), + er_coap_client:request(put, HeartURI). + connection(Channel) -> URI = ?MQTT_PREFIX ++ diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src index 22dd4efde..dfe49972d 100644 --- a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_stomp, [ {description, "Stomp Gateway"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 8e9be8359..453fa9fd2 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -1052,9 +1052,16 @@ handle_deliver( _ -> Headers0 end, + Headers2 = lists:foldl( + fun({Key, _Val} = KV, Acc1) -> + lists:keystore(Key, 1, Acc1, KV) + end, + Headers1, + maps:get(stomp_headers, Headers, []) + ), Frame = #stomp_frame{ command = <<"MESSAGE">>, - headers = Headers1 ++ maps:get(stomp_headers, Headers, []), + headers = Headers2, body = Payload }, [Frame | Acc]; @@ -1160,12 +1167,12 @@ do_negotiate_version(Accepts) -> lists:reverse(lists:sort(binary:split(Accepts, <<",">>, [global]))) ). -do_negotiate_version(Ver, []) -> - {error, <<"Supported protocol versions < ", Ver/binary>>}; do_negotiate_version(Ver, [AcceptVer | _]) when Ver >= AcceptVer -> {ok, AcceptVer}; do_negotiate_version(Ver, [_ | T]) -> - do_negotiate_version(Ver, T). + do_negotiate_version(Ver, T); +do_negotiate_version(Ver, _) -> + {error, <<"Supported protocol versions < ", Ver/binary>>}. header(Name, Headers) -> get_value(Name, Headers). @@ -1227,7 +1234,6 @@ frame2message( [ <<"destination">>, <<"content-length">>, - <<"content-type">>, <<"transaction">>, <<"receipt">> ] diff --git a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index 2ba753ca4..58913cf2f 100644 --- a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -181,11 +181,15 @@ t_subscribe(_) -> %% 'user-defined' header will be retain ok = send_message_frame(Sock, <<"/queue/foo">>, <<"hello">>, [ - {<<"user-defined">>, <<"emq">>} + {<<"user-defined">>, <<"emq">>}, + {<<"content-type">>, <<"text/html">>} ]), ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), {ok, Frame} = recv_a_frame(Sock), + ?assertEqual( + <<"text/html">>, proplists:get_value(<<"content-type">>, Frame#stomp_frame.headers) + ), ?assertMatch( #stomp_frame{ @@ -977,6 +981,10 @@ t_mountpoint(_) -> }} = recv_a_frame(Sock), ?assertEqual(<<"t/a">>, proplists:get_value(<<"destination">>, Headers)), + ?assertEqual( + <<"text/plain">>, proplists:get_value(<<"content-type">>, Headers) + ), + ok = send_disconnect_frame(Sock) end, diff --git a/apps/emqx_gcp_device/src/emqx_gcp_device.app.src b/apps/emqx_gcp_device/src/emqx_gcp_device.app.src index 01c722e98..7f1d81f14 100644 --- a/apps/emqx_gcp_device/src/emqx_gcp_device.app.src +++ b/apps/emqx_gcp_device/src/emqx_gcp_device.app.src @@ -1,6 +1,6 @@ {application, emqx_gcp_device, [ {description, "Application simplifying migration from GCP IoT Core"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_gcp_device_app, []}}, {applications, [ diff --git a/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl b/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl index a01c6d0e4..975e17ff0 100644 --- a/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl +++ b/apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl @@ -16,18 +16,21 @@ -module(emqx_gcp_device_authn_schema). --include("emqx_gcp_device.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -behaviour(emqx_authn_schema). -export([ + namespace/0, fields/1, desc/1, refs/0, select_union_member/1 ]). +-include("emqx_gcp_device.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + refs() -> [?R_REF(gcp_device)]. select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN}) -> diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index 1db88b924..774f11bd4 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 94b8992e0..1c0c7124f 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_ldap). @@ -25,7 +37,7 @@ %% ecpool connect & reconnect -export([connect/1]). --export([roots/0, fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1]). -export([do_get_status/1]). @@ -45,6 +57,9 @@ %%===================================================================== %% Hocon schema + +namespace() -> "ldap". + roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. diff --git a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl index 1b1bd3ce9..722e79006 100644 --- a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl +++ b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_ldap_bind_worker). diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index 3b4851fc4..9e5c772ab 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -29,6 +29,18 @@ Erlang code. %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- %% eldap does not support neither the '\28value\29' nor '\(value\)' %% so after the tokenization we should remove all escape character diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl index a400132f8..b12ba846e 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -1,5 +1,17 @@ Header "%%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the \"License\"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an \"AS IS\" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%--------------------------------------------------------------------". Nonterminals diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 79c549c22..e14e0feab 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_ldap_SUITE). diff --git a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl index e1aacef88..8c08b518c 100644 --- a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_ldap_filter_SUITE). diff --git a/lib-ee/emqx_license/.gitignore b/apps/emqx_license/.gitignore similarity index 100% rename from lib-ee/emqx_license/.gitignore rename to apps/emqx_license/.gitignore diff --git a/apps/emqx_ldap/BSL.txt b/apps/emqx_license/BSL.txt similarity index 100% rename from apps/emqx_ldap/BSL.txt rename to apps/emqx_license/BSL.txt diff --git a/lib-ee/emqx_license/README.md b/apps/emqx_license/README.md similarity index 100% rename from lib-ee/emqx_license/README.md rename to apps/emqx_license/README.md diff --git a/lib-ee/emqx_license/etc/emqx_license.conf b/apps/emqx_license/etc/emqx_license.conf similarity index 100% rename from lib-ee/emqx_license/etc/emqx_license.conf rename to apps/emqx_license/etc/emqx_license.conf diff --git a/lib-ee/emqx_license/include/emqx_license.hrl b/apps/emqx_license/include/emqx_license.hrl similarity index 100% rename from lib-ee/emqx_license/include/emqx_license.hrl rename to apps/emqx_license/include/emqx_license.hrl diff --git a/lib-ee/emqx_license/rebar.config b/apps/emqx_license/rebar.config similarity index 100% rename from lib-ee/emqx_license/rebar.config rename to apps/emqx_license/rebar.config diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/apps/emqx_license/src/emqx_license.app.src similarity index 100% rename from lib-ee/emqx_license/src/emqx_license.app.src rename to apps/emqx_license/src/emqx_license.app.src diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/apps/emqx_license/src/emqx_license.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license.erl rename to apps/emqx_license/src/emqx_license.erl diff --git a/lib-ee/emqx_license/src/emqx_license_app.erl b/apps/emqx_license/src/emqx_license_app.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_app.erl rename to apps/emqx_license/src/emqx_license_app.erl diff --git a/lib-ee/emqx_license/src/emqx_license_checker.erl b/apps/emqx_license/src/emqx_license_checker.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_checker.erl rename to apps/emqx_license/src/emqx_license_checker.erl diff --git a/lib-ee/emqx_license/src/emqx_license_cli.erl b/apps/emqx_license/src/emqx_license_cli.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_cli.erl rename to apps/emqx_license/src/emqx_license_cli.erl diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/apps/emqx_license/src/emqx_license_http_api.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_http_api.erl rename to apps/emqx_license/src/emqx_license_http_api.erl diff --git a/lib-ee/emqx_license/src/emqx_license_parser.erl b/apps/emqx_license/src/emqx_license_parser.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_parser.erl rename to apps/emqx_license/src/emqx_license_parser.erl diff --git a/lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl b/apps/emqx_license/src/emqx_license_parser_v20220101.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_parser_v20220101.erl rename to apps/emqx_license/src/emqx_license_parser_v20220101.erl diff --git a/lib-ee/emqx_license/src/emqx_license_resources.erl b/apps/emqx_license/src/emqx_license_resources.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_resources.erl rename to apps/emqx_license/src/emqx_license_resources.erl diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/apps/emqx_license/src/emqx_license_schema.erl similarity index 96% rename from lib-ee/emqx_license/src/emqx_license_schema.erl rename to apps/emqx_license/src/emqx_license_schema.erl index 8f2d7f20d..f2b91811e 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/apps/emqx_license/src/emqx_license_schema.erl @@ -13,12 +13,14 @@ -behaviour(hocon_schema). --export([roots/0, fields/1, validations/0, desc/1, tags/0]). +-export([namespace/0, roots/0, fields/1, validations/0, desc/1, tags/0]). -export([ default_license/0 ]). +namespace() -> "license". + roots() -> [ {license, diff --git a/lib-ee/emqx_license/src/emqx_license_sup.erl b/apps/emqx_license/src/emqx_license_sup.erl similarity index 100% rename from lib-ee/emqx_license/src/emqx_license_sup.erl rename to apps/emqx_license/src/emqx_license_sup.erl diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl b/apps/emqx_license/src/proto/emqx_license_proto_v1.erl similarity index 100% rename from lib-ee/emqx_license/src/proto/emqx_license_proto_v1.erl rename to apps/emqx_license/src/proto/emqx_license_proto_v1.erl diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl b/apps/emqx_license/src/proto/emqx_license_proto_v2.erl similarity index 100% rename from lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl rename to apps/emqx_license/src/proto/emqx_license_proto_v2.erl diff --git a/lib-ee/emqx_license/test/data/emqx.lic b/apps/emqx_license/test/data/emqx.lic similarity index 100% rename from lib-ee/emqx_license/test/data/emqx.lic rename to apps/emqx_license/test/data/emqx.lic diff --git a/lib-ee/emqx_license/test/data/pub.pem b/apps/emqx_license/test/data/pub.pem similarity index 100% rename from lib-ee/emqx_license/test/data/pub.pem rename to apps/emqx_license/test/data/pub.pem diff --git a/lib-ee/emqx_license/test/data/pvt.key b/apps/emqx_license/test/data/pvt.key similarity index 100% rename from lib-ee/emqx_license/test/data/pvt.key rename to apps/emqx_license/test/data/pvt.key diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/apps/emqx_license/test/emqx_license_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_SUITE.erl rename to apps/emqx_license/test/emqx_license_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl b/apps/emqx_license/test/emqx_license_checker_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_checker_SUITE.erl rename to apps/emqx_license/test/emqx_license_checker_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl b/apps/emqx_license/test/emqx_license_cli_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_cli_SUITE.erl rename to apps/emqx_license/test/emqx_license_cli_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/apps/emqx_license/test/emqx_license_http_api_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl rename to apps/emqx_license/test/emqx_license_http_api_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl b/apps/emqx_license/test/emqx_license_parser_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_parser_SUITE.erl rename to apps/emqx_license/test/emqx_license_parser_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl b/apps/emqx_license/test/emqx_license_resources_SUITE.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl rename to apps/emqx_license/test/emqx_license_resources_SUITE.erl diff --git a/lib-ee/emqx_license/test/emqx_license_test_lib.erl b/apps/emqx_license/test/emqx_license_test_lib.erl similarity index 100% rename from lib-ee/emqx_license/test/emqx_license_test_lib.erl rename to apps/emqx_license/test/emqx_license_test_lib.erl diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 9be3e2f0c..16f901d27 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -56,6 +56,8 @@ emqx_auth_mysql, emqx_auth_postgresql, emqx_auth_redis, + emqx_ldap, + emqx_auth_ldap, emqx_auto_subscribe, emqx_gateway, emqx_gateway_stomp, @@ -78,6 +80,7 @@ emqx_mongodb, emqx_redis, emqx_mysql, + emqx_postgresql, emqx_plugins, emqx_opentelemetry, quicer, @@ -119,8 +122,6 @@ emqx_eviction_agent, emqx_node_rebalance, emqx_ft, - emqx_ldap, - emqx_auth_ldap, emqx_gcp_device, emqx_dashboard_rbac, emqx_dashboard_sso diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 8cf85e936..496afcd64 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.15"}, + {vsn, "0.2.16"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 3c13a1935..efa05ad37 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.32"}, + {vsn, "5.0.33"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 29afa9d09..d5879be36 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -38,12 +38,21 @@ -define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}). -define(TAGS, ["Configs"]). +-if(?EMQX_RELEASE_EDITION == ee). +-define(ROOT_KEYS_EE, [ + <<"file_transfer">> +]). +-else. +-define(ROOT_KEYS_EE, []). +-endif. + -define(ROOT_KEYS, [ <<"dashboard">>, <<"alarm">>, <<"sys_topics">>, <<"sysmon">>, <<"log">> + | ?ROOT_KEYS_EE ]). %% erlfmt-ignore diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 8295047b9..1718a14cf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -313,7 +313,7 @@ create_listener_schema(Opts) -> ], Example = maps:remove(id, tcp_schema_example()), emqx_dashboard_swagger:schema_with_example( - ?UNION(Schemas), + hoconsc:union(Schemas), Example#{name => <<"demo">>} ). diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index bcc21a97b..4aff66efc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -552,6 +552,8 @@ group_trace_files(TraceLog, TraceFiles) -> empty; ({ok, _Node, _Bin}) -> nonempty; + ({error, _Node, enoent}) -> + empty; ({error, Node, Reason}) -> ?SLOG(error, #{ msg => "download_trace_log_error", diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 15f05da04..866efb267 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -157,7 +157,10 @@ sort_map_list_field(Field, Map) -> %% @doc Query clients clients(["list"]) -> - dump(?CHAN_TAB, client); + case ets:info(?CHAN_TAB, size) of + 0 -> emqx_ctl:print("No clients.~n"); + _ -> dump(?CHAN_TAB, client) + end; clients(["show", ClientId]) -> if_client(ClientId, fun print/1); clients(["kick", ClientId]) -> @@ -180,10 +183,15 @@ if_client(ClientId, Fun) -> %% @doc Topics Command topics(["list"]) -> - emqx_router:foldr_routes( - fun(Route, Acc) -> [print({emqx_topic, Route}) | Acc] end, - [] - ); + Res = + emqx_router:foldr_routes( + fun(Route, Acc) -> [print({emqx_topic, Route}) | Acc] end, + [] + ), + case Res of + [] -> emqx_ctl:print("No topics.~n"); + _ -> ok + end; topics(["show", Topic]) -> Routes = emqx_router:lookup_routes(Topic), [print({emqx_topic, Route}) || Route <- Routes]; @@ -194,12 +202,17 @@ topics(_) -> ]). subscriptions(["list"]) -> - lists:foreach( - fun(Suboption) -> - print({?SUBOPTION, Suboption}) - end, - ets:tab2list(?SUBOPTION) - ); + case ets:info(?SUBOPTION, size) of + 0 -> + emqx_ctl:print("No subscriptions.~n"); + _ -> + lists:foreach( + fun(SubOption) -> + print({?SUBOPTION, SubOption}) + end, + ets:tab2list(?SUBOPTION) + ) + end; subscriptions(["show", ClientId]) -> case ets:lookup(emqx_subid, bin(ClientId)) of [] -> @@ -207,7 +220,7 @@ subscriptions(["show", ClientId]) -> [{_, Pid}] -> case ets:match_object(?SUBOPTION, {{'_', Pid}, '_'}) of [] -> emqx_ctl:print("Not Found.~n"); - Suboption -> [print({?SUBOPTION, Sub}) || Sub <- Suboption] + SubOption -> [print({?SUBOPTION, Sub}) || Sub <- SubOption] end end; subscriptions(["add", ClientId, Topic, QoS]) -> @@ -446,13 +459,20 @@ log(_) -> %% @doc Trace Command trace(["list"]) -> - lists:foreach( - fun(Trace) -> - #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, - emqx_ctl:print("Trace(~s=~s, level=~s, destination=~0p)~n", [Type, Filter, Level, Dst]) - end, - emqx_trace_handler:running() - ); + case emqx_trace_handler:running() of + [] -> + emqx_ctl:print("Trace is empty~n", []); + Traces -> + lists:foreach( + fun(Trace) -> + #{type := Type, filter := Filter, level := Level, dst := Dst} = Trace, + emqx_ctl:print("Trace(~s=~s, level=~s, destination=~0p)~n", [ + Type, Filter, Level, Dst + ]) + end, + Traces + ) + end; trace(["stop", Operation, Filter0]) -> case trace_type(Operation, Filter0) of {ok, Type, Filter} -> trace_off(Type, Filter); diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index f4725b453..9b8222d20 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -389,7 +389,16 @@ t_download_empty_trace(_Config) -> ), {error, {{_, 404, _}, _Headers, Body}} = request_api(get, api_path(<<"trace/", Name/binary, "/download">>), [], #{return_all => true}), - ?assertMatch(#{<<"message">> := <<"Trace is empty">>}, emqx_utils_json:decode(Body)). + ?assertMatch(#{<<"message">> := <<"Trace is empty">>}, emqx_utils_json:decode(Body)), + File = emqx_trace:log_file(Name, Now), + ct:pal("FileName: ~p", [File]), + ?assertEqual({ok, <<>>}, file:read_file(File)), + ?assertEqual(ok, file:delete(File)), + %% return 404 if trace file is not found + {error, {{_, 404, _}, _Headers, Body}} = + request_api(get, api_path(<<"trace/", Name/binary, "/download">>), [], #{return_all => true}), + ?assertMatch(#{<<"message">> := <<"Trace is empty">>}, emqx_utils_json:decode(Body)), + ok. to_rfc3339(Second) -> list_to_binary(calendar:system_time_to_rfc3339(Second)). diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 09a404a44..e986a3fe1 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_observer_cli.erl b/apps/emqx_modules/src/emqx_observer_cli.erl index abed31edc..0be17decb 100644 --- a/apps/emqx_modules/src/emqx_observer_cli.erl +++ b/apps/emqx_modules/src/emqx_observer_cli.erl @@ -40,10 +40,18 @@ cmd(["bin_leak"]) -> recon:bin_leak(100) ); cmd(["load", Mod]) -> - Module = list_to_existing_atom(Mod), - Nodes = nodes(), - Res = remote_load(Nodes, Module), - emqx_ctl:print("Loaded ~p module on ~p: ~p~n", [Module, Nodes, Res]); + case nodes() of + [] -> + emqx_ctl:print("No other nodes in the cluster~n"); + Nodes -> + case emqx_utils:safe_to_existing_atom(Mod) of + {ok, Module} -> + Res = recon:remote_load(Nodes, Module), + emqx_ctl:print("Loaded ~p module on ~p: ~p~n", [Module, Nodes, Res]); + {error, Reason} -> + emqx_ctl:print("Module(~s) not found: ~p~n", [Mod, Reason]) + end + end; cmd(_) -> emqx_ctl:usage([ {"observer status", "Start observer in the current console"}, @@ -51,12 +59,5 @@ cmd(_) -> "Force all processes to perform garbage collection " "and prints the top-100 processes that freed the " "biggest amount of binaries, potentially highlighting leaks."}, - {"observer load Mod", "Ensure a module is loaded in all EMQX nodes in the cluster"} + {"observer load Mod", "Enhanced module synchronization across all cluster nodes"} ]). - -%% recon:remote_load/1 has a bug, when nodes() returns [], it is -%% taken by recon as a node name. -%% before OTP 23, the call returns a 'badrpc' tuple -%% after OTP 23, it crashes with 'badarg' error -remote_load([], _Module) -> ok; -remote_load(Nodes, Module) -> recon:remote_load(Nodes, Module). diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index eb846a7ab..2212ac7d4 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index 77161911a..a5795a554 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -22,6 +22,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(emqx_resource). +-behaviour(hocon_schema). %% callbacks of behaviour emqx_resource -export([ @@ -29,7 +30,8 @@ on_start/2, on_stop/2, on_query/3, - on_get_status/2 + on_get_status/2, + namespace/0 ]). %% ecpool callback @@ -50,6 +52,9 @@ }). %%===================================================================== + +namespace() -> "mongo". + roots() -> [ {config, #{ diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index da24c5071..135f6878e 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 4440bcfbb..d8b7994ab 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -46,16 +46,12 @@ default_port => ?MYSQL_DEFAULT_PORT }). --type prepares() :: #{atom() => binary()}. --type params_tokens() :: #{atom() => list()}. --type sqls() :: #{atom() => binary()}. +-type template() :: {unicode:chardata(), emqx_template:str()}. -type state() :: #{ pool_name := binary(), - prepare_statement := prepares(), - params_tokens := params_tokens(), - batch_inserts := sqls(), - batch_params_tokens := params_tokens() + prepares := ok | {error, _}, + templates := #{{atom(), batch | prepstmt} => template()} }. %%===================================================================== @@ -154,13 +150,13 @@ on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> on_query( InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, - #{pool_name := PoolName, prepare_statement := Prepares} = State + State ) -> MySqlFunction = mysql_function(TypeOrKey), {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State), case on_sql_query(InstId, MySqlFunction, SQLOrKey2, Data, Timeout, State) of {error, not_prepared} -> - case maybe_prepare_sql(SQLOrKey2, Prepares, PoolName) of + case maybe_prepare_sql(SQLOrKey2, State) of ok -> ?tp( mysql_connector_on_query_prepared_sql, @@ -187,23 +183,27 @@ on_query( on_batch_query( InstId, - BatchReq, - #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State + BatchReq = [{Key, _} | _], + #{query_templates := Templates} = State ) -> - case hd(BatchReq) of - {Key, _} -> - case maps:get(Key, Inserts, undefined) of - undefined -> - {error, {unrecoverable_error, batch_select_not_implemented}}; - InsertSQL -> - Tokens = maps:get(Key, ParamsTokens), - on_batch_insert(InstId, BatchReq, InsertSQL, Tokens, State) - end; - Request -> - LogMeta = #{connector => InstId, first_request => Request, state => State}, - ?SLOG(error, LogMeta#{msg => "invalid request"}), - {error, {unrecoverable_error, invalid_request}} - end. + case maps:get({Key, batch}, Templates, undefined) of + undefined -> + {error, {unrecoverable_error, batch_select_not_implemented}}; + Template -> + on_batch_insert(InstId, BatchReq, Template, State) + end; +on_batch_query( + InstId, + BatchReq, + State +) -> + ?SLOG(error, #{ + msg => "invalid request", + connector => InstId, + request => BatchReq, + state => State + }), + {error, {unrecoverable_error, invalid_request}}. mysql_function(sql) -> query; @@ -222,8 +222,8 @@ on_get_status(_InstId, #{pool_name := PoolName} = State) -> {ok, NState} -> %% return new state with prepared statements {connected, NState}; - {error, {undefined_table, NState}} -> - {disconnected, NState, unhealthy_target}; + {error, undefined_table} -> + {disconnected, State, unhealthy_target}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn connecting @@ -238,8 +238,8 @@ do_get_status(Conn) -> do_check_prepares( #{ pool_name := PoolName, - prepare_statement := #{send_message := SQL} - } = State + templates := #{{send_message, prepstmt} := SQL} + } ) -> % it's already connected. Verify if target table still exists Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], @@ -250,7 +250,7 @@ do_check_prepares( {ok, Conn} -> case mysql:prepare(Conn, get_status, SQL) of {error, {1146, _, _}} -> - {error, {undefined_table, State}}; + {error, undefined_table}; {ok, Statement} -> mysql:unprepare(Conn, Statement); _ -> @@ -265,17 +265,14 @@ do_check_prepares( ok, Workers ); -do_check_prepares(#{prepare_statement := Statement}) when is_map(Statement) -> +do_check_prepares(#{prepares := ok}) -> ok; -do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, Prepares}}) -> +do_check_prepares(#{prepares := {error, _}} = State) -> %% retry to prepare - case prepare_sql(Prepares, PoolName) of + case prepare_sql(State) of ok -> %% remove the error - {ok, State#{prepare_statement => Prepares}}; - {error, undefined_table} -> - %% indicate the error - {error, {undefined_table, State#{prepare_statement => {error, Prepares}}}}; + {ok, State#{prepares => ok}}; {error, Reason} -> {error, Reason} end. @@ -285,41 +282,44 @@ do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, connect(Options) -> mysql:start_link(Options). -init_prepare(State = #{prepare_statement := Prepares, pool_name := PoolName}) -> - case maps:size(Prepares) of +init_prepare(State = #{query_templates := Templates}) -> + case maps:size(Templates) of 0 -> - State; + State#{prepares => ok}; _ -> - case prepare_sql(Prepares, PoolName) of + case prepare_sql(State) of ok -> - State; + State#{prepares => ok}; {error, Reason} -> - LogMeta = #{msg => <<"mysql_init_prepare_statement_failed">>, reason => Reason}, - ?SLOG(error, LogMeta), + ?SLOG(error, #{ + msg => <<"MySQL init prepare statement failed">>, + reason => Reason + }), %% mark the prepare_statement as failed - State#{prepare_statement => {error, Prepares}} + State#{prepares => {error, Reason}} end end. -maybe_prepare_sql(SQLOrKey, Prepares, PoolName) -> - case maps:is_key(SQLOrKey, Prepares) of - true -> prepare_sql(Prepares, PoolName); +maybe_prepare_sql(SQLOrKey, State = #{query_templates := Templates}) -> + case maps:is_key({SQLOrKey, prepstmt}, Templates) of + true -> prepare_sql(State); false -> {error, {unrecoverable_error, prepared_statement_invalid}} end. -prepare_sql(Prepares, PoolName) when is_map(Prepares) -> - prepare_sql(maps:to_list(Prepares), PoolName); -prepare_sql(Prepares, PoolName) -> - case do_prepare_sql(Prepares, PoolName) of +prepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> + prepare_sql(maps:to_list(Templates), PoolName). + +prepare_sql(Templates, PoolName) -> + case do_prepare_sql(Templates, PoolName) of ok -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Templates]}), ok; {error, R} -> {error, R} end. -do_prepare_sql(Prepares, PoolName) -> +do_prepare_sql(Templates, PoolName) -> Conns = [ begin @@ -328,33 +328,30 @@ do_prepare_sql(Prepares, PoolName) -> end || {_Name, Worker} <- ecpool:workers(PoolName) ], - prepare_sql_to_conn_list(Conns, Prepares). + prepare_sql_to_conn_list(Conns, Templates). -prepare_sql_to_conn_list([], _PrepareList) -> +prepare_sql_to_conn_list([], _Templates) -> ok; -prepare_sql_to_conn_list([Conn | ConnList], PrepareList) -> - case prepare_sql_to_conn(Conn, PrepareList) of +prepare_sql_to_conn_list([Conn | ConnList], Templates) -> + case prepare_sql_to_conn(Conn, Templates) of ok -> - prepare_sql_to_conn_list(ConnList, PrepareList); + prepare_sql_to_conn_list(ConnList, Templates); {error, R} -> %% rollback - Fun = fun({Key, _}) -> - _ = unprepare_sql_to_conn(Conn, Key), - ok - end, - lists:foreach(Fun, PrepareList), + _ = [unprepare_sql_to_conn(Conn, Template) || Template <- Templates], {error, R} end. -prepare_sql_to_conn(Conn, []) when is_pid(Conn) -> ok; -prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> - LogMeta = #{msg => "mysql_prepare_statement", name => Key, prepare_sql => SQL}, +prepare_sql_to_conn(_Conn, []) -> + ok; +prepare_sql_to_conn(Conn, [{{Key, prepstmt}, {SQL, _RowTemplate}} | Rest]) -> + LogMeta = #{msg => "MySQL Prepare Statement", name => Key, prepare_sql => SQL}, ?SLOG(info, LogMeta), _ = unprepare_sql_to_conn(Conn, Key), case mysql:prepare(Conn, Key, SQL) of {ok, _Key} -> ?SLOG(info, LogMeta#{result => success}), - prepare_sql_to_conn(Conn, PrepareList); + prepare_sql_to_conn(Conn, Rest); {error, {1146, _, _} = Reason} -> %% Target table is not created ?tp(mysql_undefined_table, #{}), @@ -365,84 +362,92 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> % syntax failures. Retrying syntax failures is not very productive. ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, Reason} - end. + end; +prepare_sql_to_conn(Conn, [{_Key, _Template} | Rest]) -> + prepare_sql_to_conn(Conn, Rest). -unprepare_sql_to_conn(Conn, PrepareSqlKey) -> - mysql:unprepare(Conn, PrepareSqlKey). +unprepare_sql_to_conn(Conn, {{Key, prepstmt}, _}) -> + mysql:unprepare(Conn, Key); +unprepare_sql_to_conn(Conn, Key) when is_atom(Key) -> + mysql:unprepare(Conn, Key); +unprepare_sql_to_conn(_Conn, _) -> + ok. parse_prepare_sql(Config) -> - SQL = - case maps:get(prepare_statement, Config, undefined) of - undefined -> - case maps:get(sql, Config, undefined) of - undefined -> #{}; - Template -> #{send_message => Template} - end; - Any -> - Any + Queries = + case Config of + #{prepare_statement := Qs} -> + Qs; + #{sql := Query} -> + #{send_message => Query}; + _ -> + #{} end, - parse_prepare_sql(maps:to_list(SQL), #{}, #{}, #{}, #{}). + Templates = maps:fold(fun parse_prepare_sql/3, #{}, Queries), + #{query_templates => Templates}. -parse_prepare_sql([{Key, H} | _] = L, Prepares, Tokens, BatchInserts, BatchTks) -> - {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H), - parse_batch_prepare_sql( - L, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}, BatchInserts, BatchTks - ); -parse_prepare_sql([], Prepares, Tokens, BatchInserts, BatchTks) -> - #{ - prepare_statement => Prepares, - params_tokens => Tokens, - batch_inserts => BatchInserts, - batch_params_tokens => BatchTks - }. +parse_prepare_sql(Key, Query, Acc) -> + Template = emqx_template_sql:parse_prepstmt(Query, #{parameters => '?'}), + AccNext = Acc#{{Key, prepstmt} => Template}, + parse_batch_sql(Key, Query, AccNext). -parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks) -> - case emqx_utils_sql:get_statement_type(H) of - select -> - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks); +parse_batch_sql(Key, Query, Acc) -> + case emqx_utils_sql:get_statement_type(Query) of insert -> - case emqx_utils_sql:parse_insert(H) of - {ok, {InsertSQL, Params}} -> - ParamsTks = emqx_placeholder:preproc_tmpl(Params), - parse_prepare_sql( - T, - Prepares, - Tokens, - BatchInserts#{Key => InsertSQL}, - BatchTks#{Key => ParamsTks} - ); + case emqx_utils_sql:parse_insert(Query) of + {ok, {Insert, Params}} -> + RowTemplate = emqx_template_sql:parse(Params), + Acc#{{Key, batch} => {Insert, RowTemplate}}; {error, Reason} -> - ?SLOG(error, #{msg => "split_sql_failed", sql => H, reason => Reason}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks) + ?SLOG(error, #{ + msg => "parse insert sql statement failed", + sql => Query, + reason => Reason + }), + Acc end; - Type when is_atom(Type) -> - ?SLOG(error, #{msg => "detect_sql_type_unsupported", sql => H, type => Type}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks); - {error, Reason} -> - ?SLOG(error, #{msg => "detect_sql_type_failed", sql => H, reason => Reason}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks) + select -> + Acc; + Otherwise -> + ?SLOG(error, #{ + msg => "invalid sql statement type", + sql => Query, + type => Otherwise + }), + Acc end. proc_sql_params(query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; -proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> - case maps:get(TypeOrKey, ParamsTokens, undefined) of +proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) -> + case maps:get({TypeOrKey, prepstmt}, Templates, undefined) of undefined -> {SQLOrData, Params}; - Tokens -> - {TypeOrKey, emqx_placeholder:proc_sql(Tokens, SQLOrData)} + {_InsertPart, RowTemplate} -> + % NOTE + % Ignoring errors here, missing variables are set to `null`. + {Row, _Errors} = emqx_template_sql:render_prepstmt( + RowTemplate, + {emqx_jsonish, SQLOrData} + ), + {TypeOrKey, Row} end. -on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) -> - ValuesPart = lists:join($,, [ - emqx_placeholder:proc_param_str(Tokens, Msg, fun emqx_placeholder:quote_mysql/1) - || {_, Msg} <- BatchReqs - ]), - Query = [InsertPart, <<" values ">> | ValuesPart], +on_batch_insert(InstId, BatchReqs, {InsertPart, RowTemplate}, State) -> + Rows = [render_row(RowTemplate, Msg) || {_, Msg} <- BatchReqs], + Query = [InsertPart, <<" values ">> | lists:join($,, Rows)], on_sql_query(InstId, query, Query, no_params, default_timeout, State). +render_row(RowTemplate, Data) -> + % NOTE + % Ignoring errors here, missing variables are set to "'undefined'" due to backward + % compatibility requirements. + RenderOpts = #{escaping => mysql, undefined => <<"undefined">>}, + {Row, _Errors} = emqx_template_sql:render(RowTemplate, {emqx_jsonish, Data}, RenderOpts), + Row. + on_sql_query( InstId, SQLFunc, diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index d9c2d50df..963d1ec39 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 0e11062fc..41538daf6 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -433,9 +433,16 @@ do_ensure_started(NameVsn) -> tryit( "start_plugins", fun() -> - ok = ensure_exists_and_installed(NameVsn), - Plugin = do_read_plugin(NameVsn), - ok = load_code_start_apps(NameVsn, Plugin) + case ensure_exists_and_installed(NameVsn) of + ok -> + Plugin = do_read_plugin(NameVsn), + ok = load_code_start_apps(NameVsn, Plugin); + {error, plugin_not_found} -> + ?SLOG(error, #{ + msg => "plugin_not_found", + name_vsn => NameVsn + }) + end end ). @@ -665,6 +672,7 @@ do_load_plugin_app(AppName, Ebin) -> lists:foreach( fun(BeamFile) -> Module = list_to_atom(filename:basename(BeamFile, ".beam")), + _ = code:purge(Module), case code:load_file(Module) of {module, _} -> ok; diff --git a/apps/emqx_postgresql/README.md b/apps/emqx_postgresql/README.md new file mode 100644 index 000000000..bdc21db72 --- /dev/null +++ b/apps/emqx_postgresql/README.md @@ -0,0 +1,14 @@ +# PostgreSQL Connector + +This application houses the PostgreSQL Database connector. +It provides the APIs to connect to PostgreSQL Databases. + +It is used by the PostgreSQL bridge to insert messages and by the `emqx_auth_postgresql` application to check user permissions. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [APL](../../APL.txt). diff --git a/apps/emqx_connector/docker-ct b/apps/emqx_postgresql/docker-ct similarity index 100% rename from apps/emqx_connector/docker-ct rename to apps/emqx_postgresql/docker-ct diff --git a/apps/emqx_postgresql/include/emqx_postgresql.hrl b/apps/emqx_postgresql/include/emqx_postgresql.hrl new file mode 100644 index 000000000..3810bd6c2 --- /dev/null +++ b/apps/emqx_postgresql/include/emqx_postgresql.hrl @@ -0,0 +1,17 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(PGSQL_DEFAULT_PORT, 5432). diff --git a/apps/emqx_postgresql/rebar.config b/apps/emqx_postgresql/rebar.config new file mode 100644 index 000000000..1ae1309c9 --- /dev/null +++ b/apps/emqx_postgresql/rebar.config @@ -0,0 +1,8 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ + {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} +]}. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src new file mode 100644 index 000000000..efe422cd0 --- /dev/null +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -0,0 +1,16 @@ +{application, emqx_postgresql, [ + {description, "EMQX PostgreSQL Database Connector"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + epgsql, + emqx_connector, + emqx_resource + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl similarity index 69% rename from apps/emqx_connector/src/emqx_connector_pgsql.erl rename to apps/emqx_postgresql/src/emqx_postgresql.erl index 41d587a02..814d8a074 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -13,9 +13,10 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_pgsql). +-module(emqx_postgresql). --include("emqx_connector.hrl"). +-include("emqx_postgresql.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -51,15 +52,12 @@ default_port => ?PGSQL_DEFAULT_PORT }). --type prepares() :: #{atom() => binary()}. --type params_tokens() :: #{atom() => list()}. - +-type template() :: {unicode:chardata(), emqx_template_sql:row_template()}. -type state() :: #{ pool_name := binary(), - prepare_sql := prepares(), - params_tokens := params_tokens(), - prepare_statement := epgsql:statement() + query_templates := #{binary() => template()}, + prepares := #{binary() => epgsql:statement()} | {error, _} }. %% FIXME: add `{error, sync_required}' to `epgsql:execute_batch' @@ -141,7 +139,7 @@ on_start( State = parse_prepare_sql(Config), case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; + {ok, init_prepare(State#{pool_name => InstId, prepares => #{}})}; {error, Reason} -> ?tp( pgsql_connector_start_failed, @@ -188,55 +186,50 @@ pgsql_query_type(_) -> on_batch_query( InstId, - BatchReq, - #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State + [{Key, _} = Request | _] = BatchReq, + #{pool_name := PoolName, query_templates := Templates, prepares := PrepStatements} = State ) -> - case BatchReq of - [{Key, _} = Request | _] -> - BinKey = to_bin(Key), - case maps:get(BinKey, Tokens, undefined) of - undefined -> - Log = #{ - connector => InstId, - first_request => Request, - state => State, - msg => "batch_prepare_not_implemented" - }, - ?SLOG(error, Log), - {error, {unrecoverable_error, batch_prepare_not_implemented}}; - TokenList -> - {_, Datas} = lists:unzip(BatchReq), - Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas], - St = maps:get(BinKey, Sts), - case on_sql_query(InstId, PoolName, execute_batch, St, Datas2) of - {error, _Error} = Result -> - handle_result(Result); - {_Column, Results} -> - handle_batch_result(Results, 0) - end - end; - _ -> + BinKey = to_bin(Key), + case maps:get(BinKey, Templates, undefined) of + undefined -> Log = #{ connector => InstId, - request => BatchReq, + first_request => Request, state => State, - msg => "invalid_request" + msg => "batch prepare not implemented" }, ?SLOG(error, Log), - {error, {unrecoverable_error, invalid_request}} - end. + {error, {unrecoverable_error, batch_prepare_not_implemented}}; + {_Statement, RowTemplate} -> + PrepStatement = maps:get(BinKey, PrepStatements), + Rows = [render_prepare_sql_row(RowTemplate, Data) || {_Key, Data} <- BatchReq], + case on_sql_query(InstId, PoolName, execute_batch, PrepStatement, Rows) of + {error, _Error} = Result -> + handle_result(Result); + {_Column, Results} -> + handle_batch_result(Results, 0) + end + end; +on_batch_query(InstId, BatchReq, State) -> + ?SLOG(error, #{ + connector => InstId, + request => BatchReq, + state => State, + msg => "invalid request" + }), + {error, {unrecoverable_error, invalid_request}}. proc_sql_params(query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; -proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> +proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) -> Key = to_bin(TypeOrKey), - case maps:get(Key, ParamsTokens, undefined) of + case maps:get(Key, Templates, undefined) of undefined -> {SQLOrData, Params}; - Tokens -> - {Key, emqx_placeholder:proc_sql(Tokens, SQLOrData)} + {_Statement, RowTemplate} -> + {Key, render_prepare_sql_row(RowTemplate, SQLOrData)} end. on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> @@ -246,13 +239,18 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> pgsql_connector_query_return, #{error => Reason} ), - ?SLOG(error, #{ - msg => "postgresql_connector_do_sql_query_failed", - connector => InstId, - type => Type, - sql => NameOrSQL, - reason => Reason - }), + ?SLOG( + error, + maps:merge( + #{ + msg => "postgresql_connector_do_sql_query_failed", + connector => InstId, + type => Type, + sql => NameOrSQL + }, + translate_to_log_context(Reason) + ) + ), case Reason of sync_required -> {error, {recoverable_error, Reason}}; @@ -291,9 +289,9 @@ on_get_status(_InstId, #{pool_name := PoolName} = State) -> {ok, NState} -> %% return new state with prepared statements {connected, NState}; - {error, {undefined_table, NState}} -> + {error, undefined_table} -> %% return new state indicating that we are connected but the target table is not created - {disconnected, NState, unhealthy_target}; + {disconnected, State, unhealthy_target}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn connecting @@ -308,29 +306,26 @@ do_get_status(Conn) -> do_check_prepares( #{ pool_name := PoolName, - prepare_sql := #{<<"send_message">> := SQL} - } = State + query_templates := #{<<"send_message">> := {SQL, _RowTemplate}} + } ) -> WorkerPids = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], case validate_table_existence(WorkerPids, SQL) of ok -> ok; - {error, undefined_table} -> - {error, {undefined_table, State}} + {error, Reason} -> + {error, Reason} end; -do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> +do_check_prepares(#{prepares := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{pool_name := PoolName, prepare_sql := {error, Prepares}}) -> +do_check_prepares(#{prepares := {error, _}} = State) -> %% retry to prepare - case prepare_sql(Prepares, PoolName) of - {ok, Sts} -> + case prepare_sql(State) of + {ok, PrepStatements} -> %% remove the error - {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}; - {error, undefined_table} -> - %% indicate the error - {error, {undefined_table, State#{prepare_sql => {error, Prepares}}}}; - Error -> - {error, Error} + {ok, State#{prepares := PrepStatements}}; + {error, Reason} -> + {error, Reason} end. -spec validate_table_existence([pid()], binary()) -> ok | {error, undefined_table}. @@ -420,67 +415,66 @@ conn_opts([_Opt | Opts], Acc) -> conn_opts(Opts, Acc). parse_prepare_sql(Config) -> - SQL = - case maps:get(prepare_statement, Config, undefined) of - undefined -> - case maps:get(sql, Config, undefined) of - undefined -> #{}; - Template -> #{<<"send_message">> => Template} - end; - Any -> - Any + Queries = + case Config of + #{prepare_statement := Qs} -> + Qs; + #{sql := Query} -> + #{<<"send_message">> => Query}; + #{} -> + #{} end, - parse_prepare_sql(maps:to_list(SQL), #{}, #{}). + Templates = maps:fold(fun parse_prepare_sql/3, #{}, Queries), + #{query_templates => Templates}. -parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> - {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, '$n'), - parse_prepare_sql( - T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} - ); -parse_prepare_sql([], Prepares, Tokens) -> - #{ - prepare_sql => Prepares, - params_tokens => Tokens - }. +parse_prepare_sql(Key, Query, Acc) -> + Template = emqx_template_sql:parse_prepstmt(Query, #{parameters => '$n'}), + Acc#{Key => Template}. -init_prepare(State = #{prepare_sql := Prepares, pool_name := PoolName}) -> - case maps:size(Prepares) of - 0 -> - State; - _ -> - case prepare_sql(Prepares, PoolName) of - {ok, Sts} -> - State#{prepare_statement := Sts}; - Error -> - LogMeta = #{ - msg => <<"postgresql_init_prepare_statement_failed">>, error => Error - }, - ?SLOG(error, LogMeta), - %% mark the prepare_sql as failed - State#{prepare_sql => {error, Prepares}} - end +render_prepare_sql_row(RowTemplate, Data) -> + % NOTE: ignoring errors here, missing variables will be replaced with `null`. + {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, {emqx_jsonish, Data}), + Row. + +init_prepare(State = #{query_templates := Templates}) when map_size(Templates) == 0 -> + State; +init_prepare(State = #{}) -> + case prepare_sql(State) of + {ok, PrepStatements} -> + State#{prepares => PrepStatements}; + Error -> + ?SLOG( + error, + maps:merge( + #{msg => <<"postgresql_init_prepare_statement_failed">>}, + translate_to_log_context(Error) + ) + ), + %% mark the prepares failed + State#{prepares => Error} end. -prepare_sql(Prepares, PoolName) when is_map(Prepares) -> - prepare_sql(maps:to_list(Prepares), PoolName); -prepare_sql(Prepares, PoolName) -> - case do_prepare_sql(Prepares, PoolName) of +prepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> + prepare_sql(maps:to_list(Templates), PoolName). + +prepare_sql(Templates, PoolName) -> + case do_prepare_sql(Templates, PoolName) of {ok, _Sts} = Ok -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Templates]}), Ok; Error -> Error end. -do_prepare_sql(Prepares, PoolName) -> - do_prepare_sql(ecpool:workers(PoolName), Prepares, #{}). +do_prepare_sql(Templates, PoolName) -> + do_prepare_sql(ecpool:workers(PoolName), Templates, #{}). -do_prepare_sql([{_Name, Worker} | T], Prepares, _LastSts) -> +do_prepare_sql([{_Name, Worker} | Rest], Templates, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), - case prepare_sql_to_conn(Conn, Prepares) of + case prepare_sql_to_conn(Conn, Templates) of {ok, Sts} -> - do_prepare_sql(T, Prepares, Sts); + do_prepare_sql(Rest, Templates, Sts); Error -> Error end; @@ -490,20 +484,31 @@ do_prepare_sql([], _Prepares, LastSts) -> prepare_sql_to_conn(Conn, Prepares) -> prepare_sql_to_conn(Conn, Prepares, #{}). -prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; -prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> - LogMeta = #{msg => "postgresql_prepare_statement", name => Key, prepare_sql => SQL}, +prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> + {ok, Statements}; +prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when is_pid(Conn) -> + LogMeta = #{msg => "postgresql_prepare_statement", name => Key, sql => SQL}, ?SLOG(info, LogMeta), case epgsql:parse2(Conn, Key, SQL, []) of {ok, Statement} -> - prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); + prepare_sql_to_conn(Conn, Rest, Statements#{Key => Statement}); {error, {error, error, _, undefined_table, _, _} = Error} -> %% Target table is not created ?tp(pgsql_undefined_table, #{}), - ?SLOG(error, LogMeta#{msg => "postgresql_parse_failed", error => Error}), + LogMsg = + maps:merge( + LogMeta#{msg => "postgresql_parse_failed"}, + translate_to_log_context(Error) + ), + ?SLOG(error, LogMsg), {error, undefined_table}; {error, Error} = Other -> - ?SLOG(error, LogMeta#{msg => "postgresql_parse_failed", error => Error}), + LogMsg = + maps:merge( + LogMeta#{msg => "postgresql_parse_failed"}, + translate_to_log_context(Error) + ), + ?SLOG(error, LogMsg), Other end. @@ -529,3 +534,21 @@ handle_batch_result([{error, Error} | _Rest], _Acc) -> {error, {unrecoverable_error, Error}}; handle_batch_result([], Acc) -> {ok, Acc}. + +translate_to_log_context(#error{} = Reason) -> + #error{ + severity = Severity, + code = Code, + codename = Codename, + message = Message, + extra = Extra + } = Reason, + #{ + driver_severity => Severity, + driver_error_codename => Codename, + driver_error_code => Code, + driver_error_message => emqx_logger_textfmt:try_format_unicode(Message), + driver_error_extra => Extra + }; +translate_to_log_context(Reason) -> + #{reason => Reason}. diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl similarity index 96% rename from apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl rename to apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl index a4ac4f932..5a93a0578 100644 --- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl +++ b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl @@ -13,18 +13,19 @@ % %% limitations under the License. % %%-------------------------------------------------------------------- --module(emqx_connector_pgsql_SUITE). +-module(emqx_postgresql_SUITE). -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). +-include("emqx_connector/include/emqx_connector.hrl"). +-include_lib("emqx_postgresql/include/emqx_postgresql.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). -define(PGSQL_HOST, "pgsql"). --define(PGSQL_RESOURCE_MOD, emqx_connector_pgsql). +-define(PGSQL_RESOURCE_MOD, emqx_postgresql). all() -> emqx_common_test_helpers:all(?MODULE). @@ -60,7 +61,7 @@ end_per_testcase(_, _Config) -> t_lifecycle(_Config) -> perform_lifecycle_check( - <<"emqx_connector_pgsql_SUITE">>, + <<"emqx_postgresql_SUITE">>, pgsql_config() ). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index c4abbec27..4631fec8b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index e9030d3ed..a242931c4 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -24,7 +24,6 @@ -include("emqx_prometheus.hrl"). --include_lib("prometheus/include/prometheus.hrl"). -include_lib("prometheus/include/prometheus_model.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -114,16 +113,20 @@ handle_info(_Msg, State) -> push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), - JobName1 = emqx_placeholder:preproc_tmpl(JobName), - JobName2 = binary_to_list( - emqx_placeholder:proc_tmpl( - JobName1, - #{<<"name">> => Name, <<"host">> => Ip} - ) + % NOTE: allowing errors here to keep rough backward compatibility + {JobName1, Errors} = emqx_template:render( + emqx_template:parse(JobName), + #{<<"name">> => Name, <<"host">> => Ip} ), - - Url = lists:concat([Uri, "/metrics/job/", JobName2]), + _ = + Errors == [] orelse + ?SLOG(warning, #{ + msg => "prometheus_job_name_template_invalid", + errors => Errors, + template => JobName + }), Data = prometheus_text_format:format(), + Url = lists:concat([Uri, "/metrics/job/", unicode:characters_to_list(JobName1)]), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} -> ok; diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index f34675c0b..3aaf4292f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -57,7 +57,7 @@ fields("prometheus") -> )}, {headers, ?HOCON( - list({string(), string()}), + typerefl:alias("map", list({string(), string()}), #{}, [string(), string()]), #{ default => #{}, required => false, diff --git a/apps/emqx_redis/src/emqx_redis.app.src b/apps/emqx_redis/src/emqx_redis.app.src index 36f4b0cab..c9513bcf9 100644 --- a/apps/emqx_redis/src/emqx_redis.app.src +++ b/apps/emqx_redis/src/emqx_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_redis, [ {description, "EMQX Redis Database Connector"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_redis/src/emqx_redis_command.erl b/apps/emqx_redis/src/emqx_redis_command.erl new file mode 100644 index 000000000..7de80e1fa --- /dev/null +++ b/apps/emqx_redis/src/emqx_redis_command.erl @@ -0,0 +1,129 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc `split/1` function reimplements the one used by Redis itself for `redis-cli`. +%% See `sdssplitargs` function, https://github.com/redis/redis/blob/unstable/src/sds.c. + +-module(emqx_redis_command). + +-export([split/1]). + +-define(CH_SPACE, 32). +-define(CH_N, 10). +-define(CH_R, 13). +-define(CH_T, 9). +-define(CH_B, 8). +-define(CH_A, 11). + +-define(IS_CH_HEX_DIGIT(C), + (((C >= $a) andalso (C =< $f)) orelse + ((C >= $A) andalso (C =< $F)) orelse + ((C >= $0) andalso (C =< $9))) +). +-define(IS_CH_SPACE(C), + (C =:= ?CH_SPACE orelse + C =:= ?CH_N orelse + C =:= ?CH_R orelse + C =:= ?CH_T orelse + C =:= ?CH_B orelse + C =:= ?CH_A) +). + +split(Line) when is_binary(Line) -> + case split(binary_to_list(Line)) of + {ok, Args} -> + {ok, [list_to_binary(Arg) || Arg <- Args]}; + {error, _} = Error -> + Error + end; +split(Line) -> + split(Line, []). + +split([], Acc) -> + {ok, lists:reverse(Acc)}; +split([C | Rest] = Line, Acc) -> + case ?IS_CH_SPACE(C) of + true -> split(Rest, Acc); + false -> split_noq([], Line, Acc) + end. + +hex_digit_to_int(C) when (C >= $a) andalso (C =< $f) -> 10 + C - $a; +hex_digit_to_int(C) when (C >= $A) andalso (C =< $F) -> 10 + C - $A; +hex_digit_to_int(C) when (C >= $0) andalso (C =< $9) -> C - $0. + +maybe_special_char($n) -> ?CH_N; +maybe_special_char($r) -> ?CH_R; +maybe_special_char($t) -> ?CH_T; +maybe_special_char($b) -> ?CH_B; +maybe_special_char($a) -> ?CH_A; +maybe_special_char(C) -> C. + +%% Inside double quotes +split_inq(CurAcc, Line, Acc) -> + case Line of + [$\\, $x, HD1, HD2 | LineRest] when ?IS_CH_HEX_DIGIT(HD1) andalso ?IS_CH_HEX_DIGIT(HD2) -> + C = hex_digit_to_int(HD1) * 16 + hex_digit_to_int(HD2), + NewCurAcc = [C | CurAcc], + split_inq(NewCurAcc, LineRest, Acc); + [$\\, SC | LineRest] -> + C = maybe_special_char(SC), + NewCurAcc = [C | CurAcc], + split_inq(NewCurAcc, LineRest, Acc); + [$", C | _] when not ?IS_CH_SPACE(C) -> + {error, trailing_after_quote}; + [$" | LineRest] -> + split(LineRest, [lists:reverse(CurAcc) | Acc]); + [] -> + {error, unterminated_quote}; + [C | LineRest] -> + NewCurAcc = [C | CurAcc], + split_inq(NewCurAcc, LineRest, Acc) + end. + +%% Inside single quotes +split_insq(CurAcc, Line, Acc) -> + case Line of + [$\\, $' | LineRest] -> + NewCurAcc = [$' | CurAcc], + split_insq(NewCurAcc, LineRest, Acc); + [$', C | _] when not ?IS_CH_SPACE(C) -> + {error, trailing_after_single_quote}; + [$' | LineRest] -> + split(LineRest, [lists:reverse(CurAcc) | Acc]); + [] -> + {error, unterminated_single_quote}; + [C | LineRest] -> + NewCurAcc = [C | CurAcc], + split_insq(NewCurAcc, LineRest, Acc) + end. + +%% Outside quotes +split_noq(CurAcc, Line, Acc) -> + case Line of + [C | LineRest] when + ?IS_CH_SPACE(C); C =:= ?CH_N; C =:= ?CH_R; C =:= ?CH_T + -> + split(LineRest, [lists:reverse(CurAcc) | Acc]); + [] -> + split([], [lists:reverse(CurAcc) | Acc]); + [$' | LineRest] -> + split_insq(CurAcc, LineRest, Acc); + [$" | LineRest] -> + split_inq(CurAcc, LineRest, Acc); + [C | LineRest] -> + NewCurAcc = [C | CurAcc], + split_noq(NewCurAcc, LineRest, Acc) + end. diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index e03b05921..8fcbf2b63 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -1,17 +1,17 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_redis_SUITE). @@ -190,9 +190,9 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> % Should not even be able to get the resource data out of ets now unlike just stopping. ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). -% %%------------------------------------------------------------------------------ -% %% Helpers -% %%------------------------------------------------------------------------------ +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ redis_config_single() -> redis_config_base("single", "server"). diff --git a/apps/emqx_redis/test/emqx_redis_command_SUITE.erl b/apps/emqx_redis/test/emqx_redis_command_SUITE.erl new file mode 100644 index 000000000..1c6f87eff --- /dev/null +++ b/apps/emqx_redis/test/emqx_redis_command_SUITE.erl @@ -0,0 +1,76 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_redis_command_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +t_split_ok(_Config) -> + ?assertEqual( + {ok, [<<"ab">>, <<"cd">>, <<"ef">>]}, + emqx_redis_command:split(<<" \"ab\" 'cd' ef ">>) + ), + ?assertEqual( + {ok, [<<"ab">>, <<"cd">>, <<"ef">>]}, + emqx_redis_command:split(<<" ab\tcd ef">>) + ), + ?assertEqual( + {ok, [<<"abc'd">>, <<"ef">>]}, + emqx_redis_command:split(<<"ab\"c'd\" ef">>) + ), + ?assertEqual( + {ok, [<<"abc\"d">>, <<"ef">>]}, + emqx_redis_command:split(<<"ab'c\"d' ef">>) + ), + ?assertEqual( + {ok, [<<"IJK">>, <<"\\x49\\x4a\\x4B">>]}, + emqx_redis_command:split(<<"\"\\x49\\x4a\\x4B\" \\x49\\x4a\\x4B">>) + ), + ?assertEqual( + {ok, [<<"x\t\n\r\b\v">>]}, + emqx_redis_command:split(<<"\"\\x\\t\\n\\r\\b\\a\"">>) + ), + ?assertEqual( + {ok, [<<"abc\'d">>, <<"ef">>]}, + emqx_redis_command:split(<<"'abc\\'d' ef">>) + ), + ?assertEqual( + {ok, [<<>>, <<>>]}, + emqx_redis_command:split(<<" '' \"\" ">>) + ). + +t_split_error(_Config) -> + ?assertEqual( + {error, trailing_after_quote}, + emqx_redis_command:split(<<"\"a\"b">>) + ), + ?assertEqual( + {error, unterminated_quote}, + emqx_redis_command:split(<<"\"ab">>) + ), + ?assertEqual( + {error, trailing_after_single_quote}, + emqx_redis_command:split(<<"'a'b'c">>) + ), + ?assertEqual( + {error, unterminated_single_quote}, + emqx_redis_command:split(<<"'ab">>) + ). diff --git a/apps/emqx_redis/test/props/prop_emqx_redis_command.erl b/apps/emqx_redis/test/props/prop_emqx_redis_command.erl new file mode 100644 index 000000000..dc7ce2ada --- /dev/null +++ b/apps/emqx_redis/test/props/prop_emqx_redis_command.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(prop_emqx_redis_command). + +-include_lib("proper/include/proper.hrl"). + +%%-------------------------------------------------------------------- +%% Properties +%%-------------------------------------------------------------------- + +prop_split() -> + ?FORALL( + Cmd, + binary(), + %% Should terminate and not crash + is_tuple(emqx_redis_command:split(Cmd)) + ). diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 8f7c9aa17..cab070826 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl index 5710e4df3..3fef4c8b0 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia_cli.erl @@ -32,10 +32,11 @@ load() -> ok = emqx_ctl:register_command(retainer, {?MODULE, retainer}, []). retainer(["info"]) -> - ?PRINT("Number of retained messages: ~p~n", [emqx_retainer:retained_count()]); + count(); retainer(["topics"]) -> - [?PRINT("~ts~n", [I]) || I <- emqx_retainer_mnesia:topics()], - ok; + topic(1, 1000); +retainer(["topics", Start, Len]) -> + topic(list_to_integer(Start), list_to_integer(Len)); retainer(["clean", Topic]) -> emqx_retainer:delete(list_to_binary(Topic)); retainer(["clean"]) -> @@ -65,7 +66,9 @@ retainer(_) -> emqx_ctl:usage( [ {"retainer info", "Show the count of retained messages"}, - {"retainer topics", "Show all topics of retained messages"}, + {"retainer topics", "Same as retainer topic 1 1000"}, + {"retainer topics ", + "Show topics of retained messages by the specified range"}, {"retainer clean", "Clean all retained messages"}, {"retainer clean ", "Clean retained messages by the specified topic filter"}, {"retainer reindex status", "Show reindex status"}, @@ -98,3 +101,12 @@ do_reindex(Force) -> end ), ?PRINT_MSG("Reindexing finished~n"). + +count() -> + ?PRINT("Number of retained messages: ~p~n", [emqx_retainer:retained_count()]). + +topic(Start, Len) -> + count(), + Topics = lists:sublist(emqx_retainer_mnesia:topics(), Start, Len), + [?PRINT("~ts~n", [I]) || I <- Topics], + ok. diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 7b1a9675e..983b27601 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -77,7 +77,7 @@ fields("retainer") -> )}, {delivery_rate, ?HOCON( - emqx_limiter_schema:rate(), + emqx_limiter_schema:rate_type(), #{ required => false, desc => ?DESC(delivery_rate), diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl index bddad5fb3..c04f7a6de 100644 --- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl @@ -44,6 +44,9 @@ t_info(_Config) -> t_topics(_Config) -> ok = emqx_retainer_mnesia_cli:retainer(["topics"]). +t_topics_with_len(_Config) -> + ok = emqx_retainer_mnesia_cli:retainer(["topics", "100", "200"]). + t_clean(_Config) -> ok = emqx_retainer_mnesia_cli:retainer(["clean"]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index f136cd5df..cd8d597de 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -82,23 +82,18 @@ pre_process_action_args( qos := QoS, retain := Retain, payload := Payload, - mqtt_properties := MQTTPropertiesTemplate0, - user_properties := UserPropertiesTemplate + mqtt_properties := MQTTProperties, + user_properties := UserProperties } = Args ) -> - MQTTPropertiesTemplate = - maps:map( - fun(_Key, V) -> emqx_placeholder:preproc_tmpl(V) end, - MQTTPropertiesTemplate0 - ), Args#{ preprocessed_tmpl => #{ - topic => emqx_placeholder:preproc_tmpl(Topic), - qos => preproc_vars(QoS), - retain => preproc_vars(Retain), - payload => emqx_placeholder:preproc_tmpl(Payload), - mqtt_properties => MQTTPropertiesTemplate, - user_properties => preproc_user_properties(UserPropertiesTemplate) + topic => emqx_template:parse(Topic), + qos => parse_simple_var(QoS), + retain => parse_simple_var(Retain), + payload => parse_payload(Payload), + mqtt_properties => parse_mqtt_properties(MQTTProperties), + user_properties => parse_user_properties(UserProperties) } }; pre_process_action_args(_, Args) -> @@ -131,25 +126,28 @@ republish( #{metadata := #{rule_id := RuleId}} = Env, #{ preprocessed_tmpl := #{ - qos := QoSTks, - retain := RetainTks, - topic := TopicTks, - payload := PayloadTks, + qos := QoSTemplate, + retain := RetainTemplate, + topic := TopicTemplate, + payload := PayloadTemplate, mqtt_properties := MQTTPropertiesTemplate, - user_properties := UserPropertiesTks + user_properties := UserPropertiesTemplate } } ) -> - Topic = emqx_placeholder:proc_tmpl(TopicTks, Selected), - Payload = format_msg(PayloadTks, Selected), - QoS = replace_simple_var(QoSTks, Selected, 0), - Retain = replace_simple_var(RetainTks, Selected, false), + % NOTE: rendering missing bindings as string "undefined" + {TopicString, _Errors1} = render_template(TopicTemplate, Selected), + {PayloadString, _Errors2} = render_template(PayloadTemplate, Selected), + Topic = iolist_to_binary(TopicString), + Payload = iolist_to_binary(PayloadString), + QoS = render_simple_var(QoSTemplate, Selected, 0), + Retain = render_simple_var(RetainTemplate, Selected, false), %% 'flags' is set for message re-publishes or message related %% events such as message.acked and message.dropped Flags0 = maps:get(flags, Env, #{}), Flags = Flags0#{retain => Retain}, - PubProps0 = format_pub_props(UserPropertiesTks, Selected, Env), - MQTTProps = format_mqtt_properties(MQTTPropertiesTemplate, Selected, Env), + PubProps0 = render_pub_props(UserPropertiesTemplate, Selected, Env), + MQTTProps = render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env), PubProps = maps:merge(PubProps0, MQTTProps), ?TRACE( "RULE", @@ -220,79 +218,90 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> _ = emqx_broker:safe_publish(Msg), emqx_metrics:inc_msg(Msg). -preproc_vars(Data) when is_binary(Data) -> - emqx_placeholder:preproc_tmpl(Data); -preproc_vars(Data) -> - Data. +parse_simple_var(Data) when is_binary(Data) -> + emqx_template:parse(Data); +parse_simple_var(Data) -> + {const, Data}. -preproc_user_properties(<<"${pub_props.'User-Property'}">>) -> +parse_payload(Payload) -> + case string:is_empty(Payload) of + false -> emqx_template:parse(Payload); + true -> emqx_template:parse("${.}") + end. + +parse_mqtt_properties(MQTTPropertiesTemplate) -> + maps:map( + fun(_Key, V) -> emqx_template:parse(V) end, + MQTTPropertiesTemplate + ). + +parse_user_properties(<<"${pub_props.'User-Property'}">>) -> %% keep the original %% avoid processing this special variable because %% we do not want to force users to select the value %% the value will be taken from Env.pub_props directly ?ORIGINAL_USER_PROPERTIES; -preproc_user_properties(<<"${", _/binary>> = V) -> +parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable - emqx_placeholder:preproc_tmpl(V); -preproc_user_properties(_) -> + emqx_template:parse(V); +parse_user_properties(_) -> %% invalid, discard undefined. -replace_simple_var(Tokens, Data, Default) when is_list(Tokens) -> - [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), - case Var of +render_template(Template, Bindings) -> + emqx_template:render(Template, {emqx_jsonish, Bindings}). + +render_simple_var([{var, _Name, Accessor}], Data, Default) -> + case emqx_jsonish:lookup(Accessor, Data) of + {ok, Var} -> Var; %% cannot find the variable from Data - undefined -> Default; - _ -> Var + {error, _} -> Default end; -replace_simple_var(Val, _Data, _Default) -> +render_simple_var({const, Val}, _Data, _Default) -> Val. -format_msg([], Selected) -> - emqx_utils_json:encode(Selected); -format_msg(Tokens, Selected) -> - emqx_placeholder:proc_tmpl(Tokens, Selected). - -format_pub_props(UserPropertiesTks, Selected, Env) -> +render_pub_props(UserPropertiesTemplate, Selected, Env) -> UserProperties = - case UserPropertiesTks of + case UserPropertiesTemplate of ?ORIGINAL_USER_PROPERTIES -> maps:get('User-Property', maps:get(pub_props, Env, #{}), #{}); undefined -> #{}; _ -> - replace_simple_var(UserPropertiesTks, Selected, #{}) + render_simple_var(UserPropertiesTemplate, Selected, #{}) end, #{'User-Property' => UserProperties}. -format_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> - #{metadata := #{rule_id := RuleId}} = Env, - MQTTProperties0 = - maps:fold( - fun(K, Template, Acc) -> - try - V = emqx_placeholder:proc_tmpl(Template, Selected), - Acc#{K => V} - catch - Kind:Error -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - exception => Kind, - reason => Error, - property => K, - selected => Selected - } - ), - Acc - end +%% + +-define(BADPROP(K, REASON, ENV, DATA), + ?SLOG( + debug, + DATA#{ + msg => "bad_mqtt_property_value_ignored", + rule_id => emqx_utils_maps:deep_get([metadata, rule_id], ENV, undefined), + reason => REASON, + property => K + } + ) +). + +render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> + MQTTProperties = + maps:map( + fun(K, Template) -> + {V, Errors} = render_template(Template, Selected), + case Errors of + [] -> + ok; + Errors -> + ?BADPROP(K, Errors, Env, #{selected => Selected}) + end, + iolist_to_binary(V) end, - #{}, MQTTPropertiesTemplate ), - coerce_properties_values(MQTTProperties0, Env). + coerce_properties_values(MQTTProperties, Env). ensure_int(B) when is_binary(B) -> try @@ -304,42 +313,24 @@ ensure_int(B) when is_binary(B) -> ensure_int(I) when is_integer(I) -> I. -coerce_properties_values(MQTTProperties, #{metadata := #{rule_id := RuleId}}) -> - maps:fold( - fun(K, V0, Acc) -> +coerce_properties_values(MQTTProperties, Env) -> + maps:filtermap( + fun(K, V) -> try - V = encode_mqtt_property(K, V0), - Acc#{K => V} + {true, encode_mqtt_property(K, V)} catch - throw:bad_integer -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - reason => bad_integer, - property => K, - value => V0 - } - ), - Acc; + throw:Reason -> + ?BADPROP(K, Reason, Env, #{value => V}), + false; Kind:Reason:Stacktrace -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - exception => Kind, - reason => Reason, - property => K, - value => V0, - stacktrace => Stacktrace - } - ), - Acc + ?BADPROP(K, Reason, Env, #{ + value => V, + exception => Kind, + stacktrace => Stacktrace + }), + false end end, - #{}, MQTTProperties ). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 0424bfb60..e9adbbdf6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -24,7 +24,7 @@ -export([check_params/2]). --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). -type tag() :: rule_creation | rule_test | rule_engine. @@ -46,6 +46,8 @@ check_params(Params, Tag) -> %%====================================================================================== %% Hocon Schema Definitions +namespace() -> "rule_engine". + roots() -> [ {"rule_engine", sc(ref("rule_engine"), #{desc => ?DESC("root_rule_engine")})}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index c353742ae..7feacee77 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,10 +2,19 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.28"}, + {vsn, "5.0.29"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, - {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, + {applications, [ + kernel, + stdlib, + rulesql, + getopt, + uuid, + emqx, + emqx_utils, + emqx_ctl + ]}, {mod, {emqx_rule_engine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index d0019a1c5..c6d3c7ff8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -262,7 +262,7 @@ actions() -> end. qos() -> - ?UNION([emqx_schema:qos(), binary()]). + hoconsc:union([emqx_schema:qos(), binary()]). rule_engine_settings() -> [ diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 2c2af9c9a..14682eff1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -81,6 +81,7 @@ groups() -> t_sqlselect_3, t_sqlselect_message_publish_event_keep_original_props_1, t_sqlselect_message_publish_event_keep_original_props_2, + t_sqlselect_missing_template_vars_render_as_undefined, t_sqlparse_event_1, t_sqlparse_event_2, t_sqlparse_event_3, @@ -92,6 +93,7 @@ groups() -> t_sqlparse_foreach_6, t_sqlparse_foreach_7, t_sqlparse_foreach_8, + t_sqlparse_foreach_9, t_sqlparse_case_when_1, t_sqlparse_case_when_2, t_sqlparse_case_when_3, @@ -1371,14 +1373,13 @@ t_sqlselect_inject_props(_Config) -> actions => [Repub] } ), - Props = user_properties(#{<<"inject_key">> => <<"inject_val">>}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), emqtt:publish(Client, <<"t1">>, #{}, <<"{\"x\":1}">>, [{qos, 0}]), receive - {publish, #{topic := T, payload := Payload, properties := Props2}} -> - ?assertEqual(Props, Props2), + {publish, #{topic := T, payload := Payload, properties := Props}} -> + ?assertEqual(user_properties(#{<<"inject_key">> => <<"inject_val">>}), Props), ?assertEqual(<<"t2">>, T), ?assertEqual(<<"{\"x\":1}">>, Payload) after 2000 -> @@ -1954,6 +1955,32 @@ t_sqlselect_as_put(_Config) -> PayloadMap2 ). +t_sqlselect_missing_template_vars_render_as_undefined(_Config) -> + SQL = <<"SELECT * FROM \"$events/client_connected\"">>, + Repub = republish_action(<<"t2">>, <<"${clientid}:${missing.var}">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [Repub] + } + ), + {ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}]), + {ok, _} = emqtt:connect(Client1), + {ok, _, _} = emqtt:subscribe(Client1, <<"t2">>), + {ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}]), + {ok, _} = emqtt:connect(Client2), + emqtt:publish(Client2, <<"foo/bar/1">>, <<>>), + receive + {publish, Msg} -> + ?assertMatch(#{topic := <<"t2">>, payload := <<"pub-02:undefined">>}, Msg) + after 2000 -> + ct:fail(wait_for_t2) + end, + emqtt:stop(Client2), + emqtt:stop(Client1), + delete_rule(TopicRule). + t_sqlparse_event_1(_Config) -> Sql = "select topic as tp " @@ -2459,6 +2486,53 @@ t_sqlparse_foreach_8(_Config) -> || SqlN <- [Sql3] ]. +t_sqlparse_foreach_9(_Config) -> + Sql1 = + "foreach json_decode(payload) as p " + "do p.ts as ts " + "from \"t/#\" ", + Context = #{ + payload => + emqx_utils_json:encode( + [ + #{ + <<"ts">> => 1451649600512, + <<"values">> => + #{ + <<"respiratoryrate">> => 20, + <<"heartrate">> => 130, + <<"systolic">> => 50 + } + } + ] + ), + topic => <<"t/a">> + }, + ?assertMatch( + {ok, [#{<<"ts">> := 1451649600512}]}, + emqx_rule_sqltester:test( + #{ + sql => Sql1, + context => Context + } + ) + ), + %% doesn't work if we don't decode it first + Sql2 = + "foreach payload as p " + "do p.ts as ts " + "from \"t/#\" ", + ?assertMatch( + {ok, []}, + emqx_rule_sqltester:test( + #{ + sql => Sql2, + context => Context + } + ) + ), + ok. + t_sqlparse_case_when_1(_Config) -> %% case-when-else clause Sql = diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index ba94f66e1..bd17dc6c4 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index db37c6e2d..5478f6416 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -37,7 +37,7 @@ fields(s3) -> )}, {secret_access_key, mk( - secret_access_key(), + typerefl:alias("string", secret_access_key()), #{ desc => ?DESC("secret_access_key"), required => false, diff --git a/apps/emqx_telemetry/src/emqx_telemetry.app.src b/apps/emqx_telemetry/src/emqx_telemetry.app.src index d9483298f..32c2baa91 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry.app.src +++ b/apps/emqx_telemetry/src/emqx_telemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_telemetry, [ {description, "Report telemetry data for EMQX Opensource edition"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, [emqx_telemetry_sup, emqx_telemetry]}, {mod, {emqx_telemetry_app, []}}, {applications, [ diff --git a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl index 1e1f547c5..586b70f72 100644 --- a/apps/emqx_telemetry/src/emqx_telemetry_schema.erl +++ b/apps/emqx_telemetry/src/emqx_telemetry_schema.erl @@ -22,11 +22,15 @@ -behaviour(hocon_schema). -export([ + namespace/0, roots/0, fields/1, desc/1 ]). +%% 'emqxtel' to distinguish open-telemetry +namespace() -> "emqxtel". + roots() -> ["telemetry"]. fields("telemetry") -> diff --git a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl index fa0dd07df..07cb18e60 100644 --- a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl @@ -738,7 +738,7 @@ create_authn(ChainName, redis) -> backend => redis, enable => true, user_id_type => username, - cmd => "HMGET mqtt_user:${username} password_hash salt is_superuser", + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, password_hash_algorithm => #{ name => plain, salt_position => suffix diff --git a/apps/emqx_utils/include/bpapi.hrl b/apps/emqx_utils/include/bpapi.hrl new file mode 100644 index 000000000..1373e0381 --- /dev/null +++ b/apps/emqx_utils/include/bpapi.hrl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_BPAPI_HRL). +-define(EMQX_BPAPI_HRL, true). + +-compile({parse_transform, emqx_bpapi_trans}). + +-endif. diff --git a/apps/emqx_utils/include/emqx_message.hrl b/apps/emqx_utils/include/emqx_message.hrl new file mode 100644 index 000000000..a0d196fa9 --- /dev/null +++ b/apps/emqx_utils/include/emqx_message.hrl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_MESSAGE_HRL). +-define(EMQX_MESSAGE_HRL, true). + +%% See 'Application Message' in MQTT Version 5.0 +-record(message, { + %% Global unique message ID + id :: binary(), + %% Message QoS + qos = 0, + %% Message from + from :: atom() | binary(), + %% Message flags + flags = #{} :: emqx_types:flags(), + %% Message headers. May contain any metadata. e.g. the + %% protocol version number, username, peerhost or + %% the PUBLISH properties (MQTT 5.0). + headers = #{} :: emqx_types:headers(), + %% Topic that the message is published to + topic :: emqx_types:topic(), + %% Message Payload + payload :: emqx_types:payload(), + %% Timestamp (Unit: millisecond) + timestamp :: integer(), + %% not used so far, for future extension + extra = [] :: term() +}). + +-endif. diff --git a/apps/emqx/src/bpapi/emqx_bpapi_trans.erl b/apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl similarity index 100% rename from apps/emqx/src/bpapi/emqx_bpapi_trans.erl rename to apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl diff --git a/apps/emqx_utils/src/emqx_jsonish.erl b/apps/emqx_utils/src/emqx_jsonish.erl new file mode 100644 index 000000000..ef26da1d8 --- /dev/null +++ b/apps/emqx_utils/src/emqx_jsonish.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_jsonish). + +-behaviour(emqx_template). +-export([lookup/2]). + +-export_type([t/0]). + +%% @doc Either a map or a JSON serial. +%% Think of it as a kind of lazily parsed and/or constructed JSON. +-type t() :: propmap() | serial(). + +%% @doc JSON in serialized form. +-type serial() :: binary(). + +-type propmap() :: #{prop() => value()}. +-type prop() :: atom() | binary(). +-type value() :: scalar() | [scalar() | propmap()] | t(). +-type scalar() :: atom() | unicode:chardata() | number(). + +%% + +%% @doc Lookup a value in the JSON-ish map accessible through the given accessor. +%% If accessor implies drilling down into a binary, it will be treated as JSON serial. +%% Failure to parse the binary as JSON will result in an _invalid type_ error. +%% Nested JSON is NOT parsed recursively. +-spec lookup(emqx_template:accessor(), t()) -> + {ok, value()} + | {error, undefined | {_Location :: non_neg_integer(), _InvalidType :: atom()}}. +lookup(Var, Jsonish) -> + lookup(0, _Decoded = false, Var, Jsonish). + +lookup(_, _, [], Value) -> + {ok, Value}; +lookup(Loc, Decoded, [Prop | Rest], Jsonish) when is_map(Jsonish) -> + case emqx_template:lookup(Prop, Jsonish) of + {ok, Value} -> + lookup(Loc + 1, Decoded, Rest, Value); + {error, Reason} -> + {error, Reason} + end; +lookup(Loc, _Decoded = false, Props, Json) when is_binary(Json) -> + try emqx_utils_json:decode(Json) of + Value -> + % NOTE: This is intentional, we don't want to parse nested JSON. + lookup(Loc, true, Props, Value) + catch + error:_ -> + {error, {Loc, binary}} + end; +lookup(Loc, _, _, Invalid) -> + {error, {Loc, type_name(Invalid)}}. + +type_name(Term) when is_atom(Term) -> atom; +type_name(Term) when is_number(Term) -> number; +type_name(Term) when is_binary(Term) -> binary; +type_name(Term) when is_list(Term) -> list. diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl index 4d386840f..90df6003b 100644 --- a/apps/emqx_utils/src/emqx_placeholder.erl +++ b/apps/emqx_utils/src/emqx_placeholder.erl @@ -249,15 +249,15 @@ bin(Val) -> emqx_utils_conv:bin(Val). -spec quote_sql(_Value) -> iolist(). quote_sql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => sql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => sql, undefined => <<"undefined">>}). -spec quote_cql(_Value) -> iolist(). quote_cql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => cql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => cql, undefined => <<"undefined">>}). -spec quote_mysql(_Value) -> iolist(). quote_mysql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => mysql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => mysql, undefined => <<"undefined">>}). lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> Value; diff --git a/apps/emqx_utils/src/emqx_template.erl b/apps/emqx_utils/src/emqx_template.erl new file mode 100644 index 000000000..ac330becf --- /dev/null +++ b/apps/emqx_utils/src/emqx_template.erl @@ -0,0 +1,386 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_template). + +-export([parse/1]). +-export([parse/2]). +-export([parse_deep/1]). +-export([parse_deep/2]). +-export([validate/2]). +-export([is_const/1]). +-export([unparse/1]). +-export([render/2]). +-export([render/3]). +-export([render_strict/2]). +-export([render_strict/3]). + +-export([lookup_var/2]). +-export([lookup/2]). + +-export([to_string/1]). + +-export_type([t/0]). +-export_type([str/0]). +-export_type([deep/0]). +-export_type([placeholder/0]). +-export_type([varname/0]). +-export_type([bindings/0]). +-export_type([accessor/0]). + +-export_type([context/0]). +-export_type([render_opts/0]). + +-type t() :: str() | {'$tpl', deeptpl()}. + +-type str() :: [iodata() | byte() | placeholder()]. +-type deep() :: {'$tpl', deeptpl()}. + +-type deeptpl() :: + t() + | #{deeptpl() => deeptpl()} + | {list, [deeptpl()]} + | {tuple, [deeptpl()]} + | scalar() + | function() + | pid() + | port() + | reference(). + +-type placeholder() :: {var, varname(), accessor()}. +-type accessor() :: [binary()]. +-type varname() :: string(). + +-type scalar() :: atom() | unicode:chardata() | number(). +-type binding() :: scalar() | list(scalar()) | bindings(). +-type bindings() :: #{atom() | binary() => binding()}. + +-type reason() :: undefined | {location(), _InvalidType :: atom()}. +-type location() :: non_neg_integer(). + +-type var_trans() :: + fun((Value :: term()) -> unicode:chardata()) + | fun((varname(), Value :: term()) -> unicode:chardata()). + +-type parse_opts() :: #{ + strip_double_quote => boolean() +}. + +-type render_opts() :: #{ + var_trans => var_trans() +}. + +-type context() :: + %% Map with (potentially nested) bindings. + bindings() + %% Arbitrary term accessible via an access module with `lookup/2` function. + | {_AccessModule :: module(), _Bindings}. + +%% Access module API +-callback lookup(accessor(), _Bindings) -> {ok, _Value} | {error, reason()}. + +-define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}"). +-define(RE_ESCAPE, "\\$\\{(\\$)\\}"). + +%% @doc Parse a unicode string into a template. +%% String might contain zero or more of placeholders in the form of `${var}`, +%% where `var` is a _location_ (possibly deeply nested) of some value in the +%% bindings map. +%% String might contain special escaped form `$${...}` which interpreted as a +%% literal `${...}`. +-spec parse(String :: unicode:chardata()) -> + t(). +parse(String) -> + parse(String, #{}). + +-spec parse(String :: unicode:chardata(), parse_opts()) -> + t(). +parse(String, Opts) -> + RE = + case Opts of + #{strip_double_quote := true} -> + <<"((?|" ?RE_PLACEHOLDER "|\"" ?RE_PLACEHOLDER "\")|" ?RE_ESCAPE ")">>; + #{} -> + <<"(" ?RE_PLACEHOLDER "|" ?RE_ESCAPE ")">> + end, + Splits = re:split(String, RE, [{return, binary}, group, trim, unicode]), + lists:flatmap(fun parse_split/1, Splits). + +parse_split([Part, _PH, Var, <<>>]) -> + % Regular placeholder + prepend(Part, [{var, unicode:characters_to_list(Var), parse_accessor(Var)}]); +parse_split([Part, _Escape, <<>>, <<"$">>]) -> + % Escaped literal `$`. + % Use single char as token so the `unparse/1` function can distinguish escaped `$`. + prepend(Part, [$$]); +parse_split([Tail]) -> + [Tail]. + +prepend(<<>>, To) -> + To; +prepend(Head, To) -> + [Head | To]. + +parse_accessor(Var) -> + case string:split(Var, <<".">>, all) of + [<<>>] -> + []; + Name -> + Name + end. + +%% @doc Validate a template against a set of allowed variables. +%% If the given template contains any variable not in the allowed set, an error +%% is returned. +-spec validate([varname()], t()) -> + ok | {error, [_Error :: {varname(), disallowed}]}. +validate(Allowed, Template) -> + {_, Errors} = render(Template, #{}), + {Used, _} = lists:unzip(Errors), + case lists:usort(Used) -- Allowed of + [] -> + ok; + Disallowed -> + {error, [{Var, disallowed} || Var <- Disallowed]} + end. + +%% @doc Check if a template is constant with respect to rendering, i.e. does not +%% contain any placeholders. +-spec is_const(t()) -> + boolean(). +is_const(Template) -> + validate([], Template) == ok. + +%% @doc Restore original term from a parsed template. +-spec unparse(t()) -> + term(). +unparse({'$tpl', Template}) -> + unparse_deep(Template); +unparse(Template) -> + unicode:characters_to_list(lists:map(fun unparse_part/1, Template)). + +unparse_part({var, Name, _Accessor}) -> + render_placeholder(Name); +unparse_part($$) -> + <<"${$}">>; +unparse_part(Part) -> + Part. + +render_placeholder(Name) -> + "${" ++ Name ++ "}". + +%% @doc Render a template with given bindings. +%% Returns a term with all placeholders replaced with values from bindings. +%% If one or more placeholders are not found in bindings, an error is returned. +%% By default, all binding values are converted to strings using `to_string/1` +%% function. Option `var_trans` can be used to override this behaviour. +-spec render(t(), context()) -> + {term(), [_Error :: {varname(), reason()}]}. +render(Template, Context) -> + render(Template, Context, #{}). + +-spec render(t(), context(), render_opts()) -> + {term(), [_Error :: {varname(), undefined}]}. +render(Template, Context, Opts) when is_list(Template) -> + lists:mapfoldl( + fun + ({var, Name, Accessor}, EAcc) -> + {String, Errors} = render_binding(Name, Accessor, Context, Opts), + {String, Errors ++ EAcc}; + (String, EAcc) -> + {String, EAcc} + end, + [], + Template + ); +render({'$tpl', Template}, Context, Opts) -> + render_deep(Template, Context, Opts). + +render_binding(Name, Accessor, Context, Opts) -> + case lookup_value(Accessor, Context) of + {ok, Value} -> + {render_value(Name, Value, Opts), []}; + {error, Reason} -> + % TODO + % Currently, it's not possible to distinguish between a missing value + % and an atom `undefined` in `TransFun`. + {render_value(Name, undefined, Opts), [{Name, Reason}]} + end. + +lookup_value(Accessor, {AccessMod, Bindings}) -> + AccessMod:lookup(Accessor, Bindings); +lookup_value(Accessor, Bindings) -> + lookup_var(Accessor, Bindings). + +render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) -> + TransFun(Value); +render_value(Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 2) -> + TransFun(Name, Value); +render_value(_Name, Value, #{}) -> + to_string(Value). + +%% @doc Render a template with given bindings. +%% Behaves like `render/2`, but raises an error exception if one or more placeholders +%% are not found in the bindings. +-spec render_strict(t(), context()) -> + term(). +render_strict(Template, Context) -> + render_strict(Template, Context, #{}). + +-spec render_strict(t(), context(), render_opts()) -> + term(). +render_strict(Template, Context, Opts) -> + case render(Template, Context, Opts) of + {Render, []} -> + Render; + {_, Errors = [_ | _]} -> + error(Errors, [unparse(Template), Context]) + end. + +%% @doc Parse an arbitrary Erlang term into a "deep" template. +%% Any binaries nested in the term are treated as string templates, while +%% lists are not analyzed for "printability" and are treated as nested terms. +%% The result is a usual template, and can be fed to other functions in this +%% module. +-spec parse_deep(term()) -> + t(). +parse_deep(Term) -> + parse_deep(Term, #{}). + +-spec parse_deep(term(), parse_opts()) -> + t(). +parse_deep(Term, Opts) -> + {'$tpl', parse_deep_term(Term, Opts)}. + +parse_deep_term(Term, Opts) when is_map(Term) -> + maps:fold( + fun(K, V, Acc) -> + Acc#{parse_deep_term(K, Opts) => parse_deep_term(V, Opts)} + end, + #{}, + Term + ); +parse_deep_term(Term, Opts) when is_list(Term) -> + {list, [parse_deep_term(E, Opts) || E <- Term]}; +parse_deep_term(Term, Opts) when is_tuple(Term) -> + {tuple, [parse_deep_term(E, Opts) || E <- tuple_to_list(Term)]}; +parse_deep_term(Term, Opts) when is_binary(Term) -> + parse(Term, Opts); +parse_deep_term(Term, _Opts) -> + Term. + +render_deep(Template, Context, Opts) when is_map(Template) -> + maps:fold( + fun(KT, VT, {Acc, Errors}) -> + {K, KErrors} = render_deep(KT, Context, Opts), + {V, VErrors} = render_deep(VT, Context, Opts), + {Acc#{K => V}, KErrors ++ VErrors ++ Errors} + end, + {#{}, []}, + Template + ); +render_deep({list, Template}, Context, Opts) when is_list(Template) -> + lists:mapfoldr( + fun(T, Errors) -> + {E, VErrors} = render_deep(T, Context, Opts), + {E, VErrors ++ Errors} + end, + [], + Template + ); +render_deep({tuple, Template}, Context, Opts) when is_list(Template) -> + {Term, Errors} = render_deep({list, Template}, Context, Opts), + {list_to_tuple(Term), Errors}; +render_deep(Template, Context, Opts) when is_list(Template) -> + {String, Errors} = render(Template, Context, Opts), + {unicode:characters_to_binary(String), Errors}; +render_deep(Term, _Bindings, _Opts) -> + {Term, []}. + +unparse_deep(Template) when is_map(Template) -> + maps:fold( + fun(K, V, Acc) -> + Acc#{unparse_deep(K) => unparse_deep(V)} + end, + #{}, + Template + ); +unparse_deep({list, Template}) when is_list(Template) -> + [unparse_deep(E) || E <- Template]; +unparse_deep({tuple, Template}) when is_list(Template) -> + list_to_tuple(unparse_deep({list, Template})); +unparse_deep(Template) when is_list(Template) -> + unicode:characters_to_binary(unparse(Template)); +unparse_deep(Term) -> + Term. + +%% + +%% @doc Lookup a variable in the bindings accessible through the accessor. +%% Lookup is "loose" in the sense that atom and binary keys in the bindings are +%% treated equally. This is useful for both hand-crafted and JSON-like bindings. +%% This is the default lookup function used by rendering functions. +-spec lookup_var(accessor(), bindings()) -> + {ok, binding()} | {error, reason()}. +lookup_var(Var, Bindings) -> + lookup_var(0, Var, Bindings). + +lookup_var(_, [], Value) -> + {ok, Value}; +lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> + case lookup(Prop, Bindings) of + {ok, Value} -> + lookup_var(Loc + 1, Rest, Value); + {error, Reason} -> + {error, Reason} + end; +lookup_var(Loc, _, Invalid) -> + {error, {Loc, type_name(Invalid)}}. + +type_name(Term) when is_atom(Term) -> atom; +type_name(Term) when is_number(Term) -> number; +type_name(Term) when is_binary(Term) -> binary; +type_name(Term) when is_list(Term) -> list. + +-spec lookup(Prop :: binary(), bindings()) -> + {ok, binding()} | {error, undefined}. +lookup(Prop, Bindings) when is_binary(Prop) -> + case maps:get(Prop, Bindings, undefined) of + undefined -> + try + {ok, maps:get(binary_to_existing_atom(Prop, utf8), Bindings)} + catch + error:{badkey, _} -> + {error, undefined}; + error:badarg -> + {error, undefined} + end; + Value -> + {ok, Value} + end. + +-spec to_string(binding()) -> + unicode:chardata(). +to_string(Bin) when is_binary(Bin) -> Bin; +to_string(Num) when is_integer(Num) -> integer_to_binary(Num); +to_string(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]); +to_string(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_string(Map) when is_map(Map) -> emqx_utils_json:encode(Map); +to_string(List) when is_list(List) -> + case io_lib:printable_unicode_list(List) of + true -> List; + false -> emqx_utils_json:encode(List) + end. diff --git a/apps/emqx_utils/src/emqx_template_sql.erl b/apps/emqx_utils/src/emqx_template_sql.erl new file mode 100644 index 000000000..9b2c1d55c --- /dev/null +++ b/apps/emqx_utils/src/emqx_template_sql.erl @@ -0,0 +1,142 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_template_sql). + +-export([parse/1]). +-export([parse/2]). +-export([render/3]). +-export([render_strict/3]). + +-export([parse_prepstmt/2]). +-export([render_prepstmt/2]). +-export([render_prepstmt_strict/2]). + +-export_type([row_template/0]). + +-type template() :: emqx_template:str(). +-type row_template() :: [emqx_template:placeholder()]. +-type context() :: emqx_template:context(). + +-type values() :: [emqx_utils_sql:value()]. + +-type parse_opts() :: #{ + parameters => '$n' | ':n' | '?', + % Inherited from `emqx_template:parse_opts()` + strip_double_quote => boolean() +}. + +-type render_opts() :: #{ + %% String escaping rules to use. + %% Default: `sql` (generic) + escaping => sql | mysql | cql, + %% Value to map `undefined` to, either to NULLs or to arbitrary strings. + %% Default: `null` + undefined => null | unicode:chardata() +}. + +-define(TEMPLATE_PARSE_OPTS, [strip_double_quote]). + +%% + +%% @doc Parse an SQL statement string with zero or more placeholders into a template. +-spec parse(unicode:chardata()) -> + template(). +parse(String) -> + parse(String, #{}). + +%% @doc Parse an SQL statement string with zero or more placeholders into a template. +-spec parse(unicode:chardata(), parse_opts()) -> + template(). +parse(String, Opts) -> + emqx_template:parse(String, Opts). + +%% @doc Render an SQL statement template given a set of bindings. +%% Interpolation generally follows the SQL syntax, strings are escaped according to the +%% `escaping` option. +-spec render(template(), context(), render_opts()) -> + {unicode:chardata(), [_Error]}. +render(Template, Context, Opts) -> + emqx_template:render(Template, Context, #{ + var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end + }). + +%% @doc Render an SQL statement template given a set of bindings. +%% Errors are raised if any placeholders are not bound. +-spec render_strict(template(), context(), render_opts()) -> + unicode:chardata(). +render_strict(Template, Context, Opts) -> + emqx_template:render_strict(Template, Context, #{ + var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end + }). + +%% @doc Parse an SQL statement string into a prepared statement and a row template. +%% The row template is a template for a row of SQL values to be inserted to a database +%% during the execution of the prepared statement. +%% Example: +%% ``` +%% {Statement, RowTemplate} = emqx_template_sql:parse_prepstmt( +%% "INSERT INTO table (id, name, age) VALUES (${id}, ${name}, 42)", +%% #{parameters => '$n'} +%% ), +%% Statement = <<"INSERT INTO table (id, name, age) VALUES ($1, $2, 42)">>, +%% RowTemplate = [{var, "...", [...]}, ...] +%% ``` +-spec parse_prepstmt(unicode:chardata(), parse_opts()) -> + {unicode:chardata(), row_template()}. +parse_prepstmt(String, Opts) -> + Template = emqx_template:parse(String, maps:with(?TEMPLATE_PARSE_OPTS, Opts)), + Statement = mk_prepared_statement(Template, Opts), + Placeholders = [Placeholder || Placeholder <- Template, element(1, Placeholder) == var], + {Statement, Placeholders}. + +mk_prepared_statement(Template, Opts) -> + ParameterFormat = maps:get(parameters, Opts, '?'), + {Statement, _} = + lists:mapfoldl( + fun + (Var, Acc) when element(1, Var) == var -> + mk_replace(ParameterFormat, Acc); + (String, Acc) -> + {String, Acc} + end, + 1, + Template + ), + Statement. + +mk_replace('?', Acc) -> + {"?", Acc}; +mk_replace('$n', N) -> + {"$" ++ integer_to_list(N), N + 1}; +mk_replace(':n', N) -> + {":" ++ integer_to_list(N), N + 1}. + +%% @doc Render a row template into a list of SQL values. +%% An _SQL value_ is a vaguely defined concept here, it is something that's considered +%% compatible with the protocol of the database being used. See the definition of +%% `emqx_utils_sql:value()` for more details. +-spec render_prepstmt(template(), context()) -> + {values(), [_Error]}. +render_prepstmt(Template, Context) -> + Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, + emqx_template:render(Template, Context, Opts). + +-spec render_prepstmt_strict(template(), context()) -> + values(). +render_prepstmt_strict(Template, Context) -> + Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, + emqx_template:render_strict(Template, Context, Opts). diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 05e2d0162..a86a8d841 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index bf4e07ff9..f827f65de 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -62,7 +62,8 @@ merge_lists/3, tcp_keepalive_opts/4, format/1, - format_mfal/1 + format_mfal/1, + call_first_defined/1 ]). -export([ @@ -554,6 +555,22 @@ format_mfal(Data) -> undefined end. +-spec call_first_defined(list({module(), atom(), list()})) -> term() | no_return(). +call_first_defined([{Module, Function, Args} | Rest]) -> + try + apply(Module, Function, Args) + catch + error:undef:Stacktrace -> + case Stacktrace of + [{Module, Function, _, _} | _] -> + call_first_defined(Rest); + _ -> + erlang:raise(error, undef, Stacktrace) + end + end; +call_first_defined([]) -> + error(none_fun_is_defined). + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_utils/src/emqx_utils_sql.erl b/apps/emqx_utils/src/emqx_utils_sql.erl index 3caed6b62..9ce9e576d 100644 --- a/apps/emqx_utils/src/emqx_utils_sql.erl +++ b/apps/emqx_utils/src/emqx_utils_sql.erl @@ -80,10 +80,15 @@ to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map). %% @doc Convert an Erlang term to a string that can be interpolated in literal %% SQL statements. The value is escaped if necessary. --spec to_sql_string(term(), Options) -> iodata() when +-spec to_sql_string(term(), Options) -> unicode:chardata() when Options :: #{ - escaping => cql | mysql | sql + escaping => mysql | sql | cql, + undefined => null | unicode:chardata() }. +to_sql_string(undefined, #{undefined := Str} = Opts) when Str =/= null -> + to_sql_string(Str, Opts); +to_sql_string(undefined, #{}) -> + <<"NULL">>; to_sql_string(String, #{escaping := mysql}) when is_binary(String) -> try escape_mysql(String) @@ -98,7 +103,7 @@ to_sql_string(Term, #{escaping := cql}) -> to_sql_string(Term, #{}) -> maybe_escape(Term, fun escape_sql/1). --spec maybe_escape(_Value, fun((binary()) -> iodata())) -> iodata(). +-spec maybe_escape(_Value, fun((binary()) -> iodata())) -> unicode:chardata(). maybe_escape(Str, EscapeFun) when is_binary(Str) -> EscapeFun(Str); maybe_escape(Str, EscapeFun) when is_list(Str) -> @@ -109,9 +114,9 @@ maybe_escape(Str, EscapeFun) when is_list(Str) -> error(Otherwise) end; maybe_escape(Val, EscapeFun) when is_atom(Val) orelse is_map(Val) -> - EscapeFun(emqx_utils_conv:bin(Val)); + EscapeFun(emqx_template:to_string(Val)); maybe_escape(Val, _EscapeFun) -> - emqx_utils_conv:bin(Val). + emqx_template:to_string(Val). -spec escape_sql(binary()) -> iodata(). escape_sql(S) -> diff --git a/apps/emqx_utils/test/emqx_jsonish_tests.erl b/apps/emqx_utils/test/emqx_jsonish_tests.erl new file mode 100644 index 000000000..c776615a1 --- /dev/null +++ b/apps/emqx_utils/test/emqx_jsonish_tests.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_jsonish_tests). + +-include_lib("eunit/include/eunit.hrl"). + +prop_prio_test_() -> + [ + ?_assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>], #{<<"foo">> => 42, foo => 1337}) + ), + ?_assertEqual( + {ok, 1337}, + emqx_jsonish:lookup([<<"foo">>], #{foo => 1337}) + ) + ]. + +undefined_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup([<<"foo">>], #{}) + ). + +undefined_deep_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{}) + ). + +undefined_deep_json_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup( + [<<"foo">>, <<"bar">>, <<"baz">>], + <<"{\"foo\":{\"bar\":{\"no\":{}}}}">> + ) + ). + +invalid_type_test() -> + ?assertEqual( + {error, {0, number}}, + emqx_jsonish:lookup([<<"foo">>], <<"42">>) + ). + +invalid_type_deep_test() -> + ?assertEqual( + {error, {2, atom}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>, <<"tuple">>], #{foo => #{bar => baz}}) + ). + +decode_json_test() -> + ?assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], <<"{\"foo\":{\"bar\":42}}">>) + ). + +decode_json_deep_test() -> + ?assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"{\"bar\": 42}">>}) + ). + +decode_json_invalid_type_test() -> + ?assertEqual( + {error, {1, list}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"[1,2,3]">>}) + ). + +decode_no_json_test() -> + ?assertEqual( + {error, {1, binary}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<0, 1, 2, 3>>}) + ). + +decode_json_no_nested_test() -> + ?assertEqual( + {error, {2, binary}}, + emqx_jsonish:lookup( + [<<"foo">>, <<"bar">>, <<"baz">>], + #{<<"foo">> => <<"{\"bar\":\"{\\\"baz\\\":42}\"}">>} + ) + ). diff --git a/apps/emqx_utils/test/emqx_template_SUITE.erl b/apps/emqx_utils/test/emqx_template_SUITE.erl new file mode 100644 index 000000000..4dfe5de2e --- /dev/null +++ b/apps/emqx_utils/test/emqx_template_SUITE.erl @@ -0,0 +1,360 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_template_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +t_render(_) -> + Context = #{ + a => <<"1">>, + b => 1, + c => 1.0, + d => #{<<"d1">> => <<"hi">>}, + l => [0, 1, 1000], + u => "utf-8 is ǝɹǝɥ" + }, + Template = emqx_template:parse( + <<"a:${a},b:${b},c:${c},d:${d},d1:${d.d1},l:${l},u:${u}">> + ), + ?assertEqual( + {<<"a:1,b:1,c:1.0,d:{\"d1\":\"hi\"},d1:hi,l:[0,1,1000],u:utf-8 is ǝɹǝɥ"/utf8>>, []}, + render_string(Template, Context) + ). + +t_render_var_trans(_) -> + Context = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), + {String, Errors} = emqx_template:render( + Template, + Context, + #{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end} + ), + ?assertEqual( + {<<"a:,b:,c:">>, []}, + {bin(String), Errors} + ). + +t_render_path(_) -> + Context = #{d => #{d1 => <<"hi">>}}, + Template = emqx_template:parse(<<"d.d1:${d.d1}">>), + ?assertEqual( + ok, + emqx_template:validate(["d.d1"], Template) + ), + ?assertEqual( + {<<"d.d1:hi">>, []}, + render_string(Template, Context) + ). + +t_render_custom_ph(_) -> + Context = #{a => <<"a">>, b => <<"b">>}, + Template = emqx_template:parse(<<"a:${a},b:${b}">>), + ?assertEqual( + {error, [{"b", disallowed}]}, + emqx_template:validate(["a"], Template) + ), + ?assertEqual( + <<"a:a,b:b">>, + render_strict_string(Template, Context) + ). + +t_render_this(_) -> + Context = #{a => <<"a">>, b => [1, 2, 3]}, + Template = emqx_template:parse(<<"this:${} / also:${.}">>), + ?assertEqual(ok, emqx_template:validate(["."], Template)), + ?assertEqual( + % NOTE: order of the keys in the JSON object depends on the JSON encoder + <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, + render_strict_string(Template, Context) + ). + +t_render_missing_bindings(_) -> + Context = #{no => #{}, c => #{<<"c1">> => 42}}, + Template = emqx_template:parse( + <<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">> + ), + ?assertEqual( + {<<"a:undefined,b:undefined,c:undefined,d:undefined,e:undefined">>, [ + {"no.such_atom_i_swear", undefined}, + {"d.d1", undefined}, + {"c.c1.c2", {2, number}}, + {"b", undefined}, + {"a", undefined} + ]}, + render_string(Template, Context) + ), + ?assertError( + [ + {"no.such_atom_i_swear", undefined}, + {"d.d1", undefined}, + {"c.c1.c2", {2, number}}, + {"b", undefined}, + {"a", undefined} + ], + render_strict_string(Template, Context) + ). + +t_render_custom_bindings(_) -> + _ = erlang:put(a, <<"foo">>), + _ = erlang:put(b, #{<<"bar">> => #{atom => 42}}), + Template = emqx_template:parse( + <<"a:${a},b:${b.bar.atom},c:${c},oops:${b.bar.atom.oops}">> + ), + ?assertEqual( + {<<"a:foo,b:42,c:undefined,oops:undefined">>, [ + {"b.bar.atom.oops", {2, number}}, + {"c", undefined} + ]}, + render_string(Template, {?MODULE, []}) + ). + +t_unparse(_) -> + TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}},e:${$}{e},lit:${$}{$}">>, + Template = emqx_template:parse(TString), + ?assertEqual( + TString, + unicode:characters_to_binary(emqx_template:unparse(Template)) + ). + +t_const(_) -> + ?assertEqual( + true, + emqx_template:is_const(emqx_template:parse(<<"">>)) + ), + ?assertEqual( + false, + emqx_template:is_const( + emqx_template:parse(<<"a:${a},b:${b},c:${$}{c}">>) + ) + ), + ?assertEqual( + true, + emqx_template:is_const( + emqx_template:parse(<<"a:${$}{a},b:${$}{b}">>) + ) + ). + +t_render_partial_ph(_) -> + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Template = emqx_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), + ?assertEqual( + <<"a:$a,b:b},c:{c},d:${d">>, + render_strict_string(Template, Context) + ). + +t_parse_escaped(_) -> + Context = #{a => <<"1">>, b => 1, c => "VAR"}, + Template = emqx_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), + ?assertEqual( + <<"a:1,b:${b},c:${VAR},lit:${$}">>, + render_strict_string(Template, Context) + ). + +t_parse_escaped_dquote(_) -> + Context = #{a => <<"1">>, b => 1}, + Template = emqx_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ + strip_double_quote => true + }), + ?assertEqual( + <<"a:1,b:\"${b}\"">>, + render_strict_string(Template, Context) + ). + +t_parse_sql_prepstmt(_) -> + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + {PrepareStatement, RowTemplate} = + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + parameters => '?' + }), + ?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)), + ?assertEqual( + {[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []}, + emqx_template_sql:render_prepstmt(RowTemplate, Context) + ). + +t_parse_sql_prepstmt_n(_) -> + Context = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, + {PrepareStatement, RowTemplate} = + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + parameters => '$n' + }), + ?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)), + ?assertEqual( + [null, true, <<"atom">>, <<"{\"d1\":42.1337}">>], + emqx_template_sql:render_prepstmt_strict(RowTemplate, Context) + ). + +t_parse_sql_prepstmt_colon(_) -> + {PrepareStatement, _RowTemplate} = + emqx_template_sql:parse_prepstmt(<<"a=${a},b=${b},c=${c},d=${d}">>, #{ + parameters => ':n' + }), + ?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)). + +t_parse_sql_prepstmt_partial_ph(_) -> + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + {PrepareStatement, RowTemplate} = + emqx_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), + ?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)), + ?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Context)). + +t_render_sql(_) -> + Context = #{ + a => <<"1">>, + b => 1, + c => 1.0, + d => #{d1 => <<"hi">>}, + n => undefined, + u => "utf8's cool 🐸" + }, + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), + ?assertMatch( + {_String, _Errors = []}, + emqx_template_sql:render(Template, Context, #{}) + ), + ?assertEqual( + <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, + bin(emqx_template_sql:render_strict(Template, Context, #{})) + ), + ?assertEqual( + <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:'undefined',u:'utf8\\'s cool 🐸'"/utf8>>, + bin(emqx_template_sql:render_strict(Template, Context, #{undefined => "undefined"})) + ). + +t_render_mysql(_) -> + %% with apostrophes + %% https://github.com/emqx/emqx/issues/4135 + Context = #{ + a => <<"1''2">>, + b => 1, + c => 1.0, + d => #{d1 => <<"someone's phone">>}, + e => <<$\\, 0, "💩"/utf8>>, + f => <<"non-utf8", 16#DCC900:24>>, + g => "utf8's cool 🐸", + h => imgood + }, + Template = emqx_template_sql:parse( + <<"a:${a},b:${b},c:${c},d:${d},e:${e},f:${f},g:${g},h:${h}">> + ), + ?assertEqual( + << + "a:'1\\'\\'2',b:1,c:1.0,d:'{\"d1\":\"someone\\'s phone\"}'," + "e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8, + "h:'imgood'" + >>, + bin(emqx_template_sql:render_strict(Template, Context, #{escaping => mysql})) + ). + +t_render_cql(_) -> + %% with apostrophes for cassandra + %% https://github.com/emqx/emqx/issues/4148 + Context = #{ + a => <<"1''2">>, + b => 1, + c => 1.0, + d => #{d1 => <<"someone's phone">>} + }, + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), + ?assertEqual( + <<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>, + bin(emqx_template_sql:render_strict(Template, Context, #{escaping => cql})) + ). + +t_render_sql_custom_ph(_) -> + {PrepareStatement, RowTemplate} = + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b.c}">>, #{parameters => '$n'}), + ?assertEqual( + {error, [{"b.c", disallowed}]}, + emqx_template:validate(["a"], RowTemplate) + ), + ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). + +t_render_sql_strip_double_quote(_) -> + Context = #{a => <<"a">>, b => <<"b">>}, + + %% no strip_double_quote option: "${key}" -> "value" + {PrepareStatement1, RowTemplate1} = emqx_template_sql:parse_prepstmt( + <<"a:\"${a}\",b:\"${b}\"">>, + #{parameters => '$n'} + ), + ?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)), + ?assertEqual( + [<<"a">>, <<"b">>], + emqx_template_sql:render_prepstmt_strict(RowTemplate1, Context) + ), + + %% strip_double_quote = true: "${key}" -> value + {PrepareStatement2, RowTemplate2} = emqx_template_sql:parse_prepstmt( + <<"a:\"${a}\",b:\"${b}\"">>, + #{parameters => '$n', strip_double_quote => true} + ), + ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)), + ?assertEqual( + [<<"a">>, <<"b">>], + emqx_template_sql:render_prepstmt_strict(RowTemplate2, Context) + ). + +t_render_tmpl_deep(_) -> + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + + Template = emqx_template:parse_deep( + #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]} + ), + + ?assertEqual( + {error, [{V, disallowed} || V <- ["b", "c"]]}, + emqx_template:validate(["a"], Template) + ), + + ?assertEqual( + #{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, + emqx_template:render_strict(Template, Context) + ). + +t_unparse_tmpl_deep(_) -> + Term = #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>], <<"${$}{d}">>, 0}]}, + Template = emqx_template:parse_deep(Term), + ?assertEqual(Term, emqx_template:unparse(Template)). + +%% + +render_string(Template, Context) -> + {String, Errors} = emqx_template:render(Template, Context), + {bin(String), Errors}. + +render_strict_string(Template, Context) -> + bin(emqx_template:render_strict(Template, Context)). + +bin(String) -> + unicode:characters_to_binary(String). + +%% Access module API + +lookup([], _) -> + {error, undefined}; +lookup([Prop | Rest], _) -> + case erlang:get(binary_to_atom(Prop)) of + undefined -> {error, undefined}; + Value -> emqx_template:lookup_var(Rest, Value) + end. diff --git a/bin/emqx b/bin/emqx index f24210cdf..5b61042d7 100755 --- a/bin/emqx +++ b/bin/emqx @@ -48,7 +48,7 @@ RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" -WHOAMI=$(whoami) +WHOAMI=$(whoami 2>/dev/null || id -u) # hocon try to read environment variables starting with "EMQX_" export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 3e39c787b..421e63b21 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -4,9 +4,11 @@ %% ex: ft=erlang ts=4 sw=4 et -define(TIMEOUT, 300000). --define(INFO(Fmt,Args), io:format(standard_io, Fmt++"~n",Args)). --define(ERROR(Fmt,Args), io:format(standard_error, "ERROR: "++Fmt++"~n",Args)). --define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>). +-define(INFO(Fmt, Args), io:format(standard_io, Fmt ++ "~n", Args)). +-define(ERROR(Fmt, Args), io:format(standard_error, "ERROR: " ++ Fmt ++ "~n", Args)). +-define(SEMVER_RE, + <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">> +). -mode(compile). @@ -17,14 +19,15 @@ main([Command0, DistInfoStr | CommandArgs]) -> %% convert arguments into a proplist Opts = parse_arguments(CommandArgs), %% invoke the command passed as argument - F = case Command0 of - "install" -> fun(A, B) -> install(A, B) end; - "unpack" -> fun(A, B) -> unpack(A, B) end; - "upgrade" -> fun(A, B) -> upgrade(A, B) end; - "downgrade" -> fun(A, B) -> downgrade(A, B) end; - "uninstall" -> fun(A, B) -> uninstall(A, B) end; - "versions" -> fun(A, B) -> versions(A, B) end - end, + F = + case Command0 of + "install" -> fun(A, B) -> install(A, B) end; + "unpack" -> fun(A, B) -> unpack(A, B) end; + "upgrade" -> fun(A, B) -> upgrade(A, B) end; + "downgrade" -> fun(A, B) -> downgrade(A, B) end; + "uninstall" -> fun(A, B) -> uninstall(A, B) end; + "versions" -> fun(A, B) -> versions(A, B) end + end, F(DistInfo, Opts); main(Args) -> ?INFO("unknown args: ~p", [Args]), @@ -38,15 +41,15 @@ unpack({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> ?INFO("Unpacked successfully: ~p", [Vsn]); old -> %% no need to unpack, has been installed previously - ?INFO("Release ~s is marked old.",[Version]); + ?INFO("Release ~s is marked old.", [Version]); unpacked -> - ?INFO("Release ~s is already unpacked.",[Version]); + ?INFO("Release ~s is already unpacked.", [Version]); current -> - ?INFO("Release ~s is already installed and current.",[Version]); + ?INFO("Release ~s is already installed and current.", [Version]); permanent -> - ?INFO("Release ~s is already installed and set permanent.",[Version]); + ?INFO("Release ~s is already installed and set permanent.", [Version]); {error, Reason} -> - ?INFO("Unpack failed: ~p.",[Reason]), + ?INFO("Unpack failed: ~p.", [Reason]), print_existing_versions(TargetNode), erlang:halt(2) end; @@ -64,38 +67,46 @@ install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> maybe_permafy(TargetNode, RelName, Vsn, Opts); old -> %% no need to unpack, has been installed previously - ?INFO("Release ~s is marked old, switching to it.",[Version]), + ?INFO("Release ~s is marked old, switching to it.", [Version]), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts); unpacked -> - ?INFO("Release ~s is already unpacked, now installing.",[Version]), + ?INFO("Release ~s is already unpacked, now installing.", [Version]), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts); current -> case proplists:get_value(permanent, Opts, true) of true -> - ?INFO("Release ~s is already installed and current, making permanent.", - [Version]), + ?INFO( + "Release ~s is already installed and current, making permanent.", + [Version] + ), permafy(TargetNode, RelName, Version); false -> - ?INFO("Release ~s is already installed and current.", - [Version]) + ?INFO( + "Release ~s is already installed and current.", + [Version] + ) end; permanent -> %% this release is marked permanent, however it might not the %% one currently running case current_release_version(TargetNode) of Version -> - ?INFO("Release ~s is already installed, running and set permanent.", - [Version]); + ?INFO( + "Release ~s is already installed, running and set permanent.", + [Version] + ); CurrentVersion -> - ?INFO("Release ~s is the currently running version.", - [CurrentVersion]), + ?INFO( + "Release ~s is the currently running version.", + [CurrentVersion] + ), check_and_install(TargetNode, Version), maybe_permafy(TargetNode, RelName, Version, Opts) end; {error, Reason} -> - ?INFO("Unpack failed: ~p",[Reason]), + ?INFO("Unpack failed: ~p", [Reason]), print_existing_versions(TargetNode), erlang:halt(2) end; @@ -119,8 +130,10 @@ uninstall({_RelName, NameTypeArg, NodeName, Cookie}, Opts) -> ?INFO("Release ~s is marked old, uninstalling it.", [Version]), remove_release(TargetNode, Version); unpacked -> - ?INFO("Release ~s is marked unpacked, uninstalling it", - [Version]), + ?INFO( + "Release ~s is marked unpacked, uninstalling it", + [Version] + ), remove_release(TargetNode, Version); current -> ?INFO("Uninstall failed: Release ~s is marked current.", [Version]), @@ -140,10 +153,11 @@ parse_arguments(Args) -> IsEnterprise = os:getenv("IS_ENTERPRISE") == "yes", parse_arguments(Args, [{is_enterprise, IsEnterprise}]). -parse_arguments([], Acc) -> Acc; -parse_arguments(["--no-permanent"|Rest], Acc) -> +parse_arguments([], Acc) -> + Acc; +parse_arguments(["--no-permanent" | Rest], Acc) -> parse_arguments(Rest, [{permanent, false}] ++ Acc); -parse_arguments([VersionStr|Rest], Acc) -> +parse_arguments([VersionStr | Rest], Acc) -> Version = parse_version(VersionStr), parse_arguments(Rest, [{version, Version}] ++ Acc). @@ -162,18 +176,29 @@ unpack_release(RelName, TargetNode, Version, Opts) -> {_, undefined} -> {error, release_package_not_found}; {ReleasePackage, ReleasePackageLink} -> - ?INFO("Release ~s not found, attempting to unpack ~s", - [Version, ReleasePackage]), - case rpc:call(TargetNode, release_handler, unpack_release, - [ReleasePackageLink], ?TIMEOUT) of - {ok, Vsn} -> {ok, Vsn}; + ?INFO( + "Release ~s not found, attempting to unpack ~s", + [Version, ReleasePackage] + ), + case + rpc:call( + TargetNode, + release_handler, + unpack_release, + [ReleasePackageLink], + ?TIMEOUT + ) + of + {ok, Vsn} -> + {ok, Vsn}; {error, {existing_release, Vsn}} -> %% sometimes the user may have removed the release/ dir %% for an `unpacked` release, then we need to re-unpack it from %% the .tar ball untar_for_unpacked_release(str(RelName), Vsn), {ok, Vsn}; - {error, _} = Error -> Error + {error, _} = Error -> + Error end end; Other -> @@ -198,8 +223,8 @@ untar_for_unpacked_release(RelName, Vsn) -> extract_tar(Cwd, Tar) -> case erl_tar:extract(Tar, [keep_old_files, {cwd, Cwd}, compressed]) of ok -> ok; - {error, {Name, Reason}} -> % New erl_tar (R3A). - throw({error, {cannot_extract_file, Name, Reason}}) + % New erl_tar (R3A). + {error, {Name, Reason}} -> throw({error, {cannot_extract_file, Name, Reason}}) end. %% 1. look for a release package tarball with the provided version: @@ -217,8 +242,11 @@ find_and_link_release_package(Version, RelName, IsEnterprise) -> ReleaseHandlerPackageLink = filename:join(Version, RelNameStr), %% this is the symlink name we'll create once %% we've found where the actual release package is located - ReleaseLink = filename:join(["releases", Version, - RelNameStr ++ ".tar.gz"]), + ReleaseLink = filename:join([ + "releases", + Version, + RelNameStr ++ ".tar.gz" + ]), ReleaseNamePattern = case IsEnterprise of false -> RelNameStr; @@ -240,14 +268,18 @@ find_and_link_release_package(Version, RelName, IsEnterprise) -> make_symlink_or_copy(filename:absname(Filename), ReleaseLink), {Filename, ReleaseHandlerPackageLink}; Files -> - ?ERROR("Found more than one package for version: '~s', " - "files: ~p", [Version, Files]), + ?ERROR( + "Found more than one package for version: '~s', " + "files: ~p", + [Version, Files] + ), erlang:halt(47) end. make_symlink_or_copy(Filename, ReleaseLink) -> case file:make_symlink(Filename, ReleaseLink) of - ok -> ok; + ok -> + ok; {error, eexist} -> ?INFO("Symlink ~p already exists, recreate it", [ReleaseLink]), ok = file:delete(ReleaseLink), @@ -260,36 +292,55 @@ make_symlink_or_copy(Filename, ReleaseLink) -> end. parse_version(V) when is_list(V) -> - hd(string:tokens(V,"/")). + hd(string:tokens(V, "/")). check_and_install(TargetNode, Vsn) -> %% Backup the sys.config, this will be used when we check and install release %% NOTE: We cannot backup the old sys.config directly, because the %% configs for plugins are only in app-envs, not in the old sys.config Configs0 = - [{AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} - || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT)], + [ + {AppName, rpc:call(TargetNode, application, get_all_env, [AppName], ?TIMEOUT)} + || {AppName, _, _} <- rpc:call(TargetNode, application, which_applications, [], ?TIMEOUT) + ], Configs1 = [{AppName, Conf} || {AppName, Conf} <- Configs0, Conf =/= []], - ok = file:write_file(filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1])), + ok = file:write_file( + filename:join(["releases", Vsn, "sys.config"]), io_lib:format("~p.", [Configs1]) + ), %% check and install release - case rpc:call(TargetNode, release_handler, - check_install_release, [Vsn], ?TIMEOUT) of + case + rpc:call( + TargetNode, + release_handler, + check_install_release, + [Vsn], + ?TIMEOUT + ) + of {ok, _OtherVsn, _Desc} -> ok; {error, Reason} -> ?ERROR("Call release_handler:check_install_release failed: ~p.", [Reason]), erlang:halt(3) end, - case rpc:call(TargetNode, release_handler, install_release, - [Vsn, [{update_paths, true}]], ?TIMEOUT) of + case + rpc:call( + TargetNode, + release_handler, + install_release, + [Vsn, [{update_paths, true}]], + ?TIMEOUT + ) + of {ok, _, _} -> ?INFO("Installed Release: ~s.", [Vsn]), ok; {error, {no_such_release, Vsn}} -> VerList = iolist_to_binary( - [io_lib:format("* ~s\t~s~n",[V,S]) || {V,S} <- which_releases(TargetNode)]), + [io_lib:format("* ~s\t~s~n", [V, S]) || {V, S} <- which_releases(TargetNode)] + ), ?INFO("Installed versions:~n~s", [VerList]), ?ERROR("Unable to revert to '~s' - not installed.", [Vsn]), erlang:halt(2); @@ -298,11 +349,13 @@ check_and_install(TargetNode, Vsn) -> %% If the value is soft_purge, release_handler:install_release/1 %% returns {error,{old_processes,Mod}} {error, {old_processes, Mod}} -> - ?ERROR("Unable to install '~s' - old processes still running code from module ~p", - [Vsn, Mod]), + ?ERROR( + "Unable to install '~s' - old processes still running code from module ~p", + [Vsn, Mod] + ), erlang:halt(3); {error, Reason1} -> - ?ERROR("Call release_handler:install_release failed: ~p",[Reason1]), + ?ERROR("Call release_handler:install_release failed: ~p", [Reason1]), erlang:halt(4) end. @@ -310,22 +363,34 @@ maybe_permafy(TargetNode, RelName, Vsn, Opts) -> case proplists:get_value(permanent, Opts, true) of true -> permafy(TargetNode, RelName, Vsn); - false -> ok + false -> + ok end. permafy(TargetNode, RelName, Vsn) -> RelNameStr = atom_to_list(RelName), - ok = rpc:call(TargetNode, release_handler, - make_permanent, [Vsn], ?TIMEOUT), + ok = rpc:call( + TargetNode, + release_handler, + make_permanent, + [Vsn], + ?TIMEOUT + ), ?INFO("Made release permanent: ~p", [Vsn]), %% upgrade/downgrade the scripts by replacing them - Scripts = [RelNameStr, RelNameStr++"_ctl", "nodetool", "install_upgrade.escript"], - [{ok, _} = file:copy(filename:join(["bin", File++"-"++Vsn]), - filename:join(["bin", File])) - || File <- Scripts], + Scripts = [RelNameStr, RelNameStr ++ "_ctl", "nodetool", "install_upgrade.escript"], + [ + {ok, _} = file:copy( + filename:join(["bin", File ++ "-" ++ Vsn]), + filename:join(["bin", File]) + ) + || File <- Scripts + ], %% update the vars UpdatedVars = io_lib:format("REL_VSN=\"~s\"~nERTS_VSN=\"~s\"~n", [Vsn, erts_vsn()]), - file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [append]). + file:write_file(filename:absname(filename:join(["releases", "emqx_vars"])), UpdatedVars, [ + append + ]). remove_release(TargetNode, Vsn) -> case rpc:call(TargetNode, release_handler, remove_release, [Vsn], ?TIMEOUT) of @@ -339,22 +404,31 @@ remove_release(TargetNode, Vsn) -> which_releases(TargetNode) -> R = rpc:call(TargetNode, release_handler, which_releases, [], ?TIMEOUT), - [ {V, S} || {_,V,_, S} <- R ]. + [{V, S} || {_, V, _, S} <- R]. %% the running release version is either the only one marked `current´ %% or, if none exists, the one marked `permanent` current_release_version(TargetNode) -> - R = rpc:call(TargetNode, release_handler, which_releases, - [], ?TIMEOUT), - Versions = [ {S, V} || {_,V,_, S} <- R ], + R = rpc:call( + TargetNode, + release_handler, + which_releases, + [], + ?TIMEOUT + ), + Versions = [{S, V} || {_, V, _, S} <- R], %% current version takes priority over the permanent - proplists:get_value(current, Versions, - proplists:get_value(permanent, Versions)). + proplists:get_value( + current, + Versions, + proplists:get_value(permanent, Versions) + ). print_existing_versions(TargetNode) -> VerList = iolist_to_binary([ - io_lib:format("* ~s\t~s~n",[V,S]) - || {V,S} <- which_releases(TargetNode) ]), + io_lib:format("* ~s\t~s~n", [V, S]) + || {V, S} <- which_releases(TargetNode) + ]), ?INFO("Installed versions:~n~s", [VerList]). start_distribution(TargetNode, NameTypeArg, Cookie) -> @@ -378,12 +452,12 @@ make_script_node(Node) -> %% get name type from arg get_name_type(NameTypeArg) -> - case NameTypeArg of - "-sname" -> - shortnames; - _ -> - longnames - end. + case NameTypeArg of + "-sname" -> + shortnames; + _ -> + longnames + end. erts_vsn() -> {ok, Str} = file:read_file(filename:join(["releases", "start_erl.data"])), @@ -393,11 +467,14 @@ erts_vsn() -> validate_target_version(TargetVersion, TargetNode) -> CurrentVersion = current_release_version(TargetNode), case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of - {{Major, Minor}, {Major, Minor}} -> ok; + {{Major, Minor}, {Major, Minor}} -> + ok; _ -> - ?ERROR("Cannot upgrade/downgrade from '~s' to '~s'~n" - "Hot upgrade is only supported between patch releases.", - [CurrentVersion, TargetVersion]), + ?ERROR( + "Cannot upgrade/downgrade from '~s' to '~s'~n" + "Hot upgrade is only supported between patch releases.", + [CurrentVersion, TargetVersion] + ), erlang:halt(48) end. @@ -409,7 +486,8 @@ get_major_minor_vsn(Version) -> parse_semver(Version) -> case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of - {match, Parts} -> Parts; + {match, Parts} -> + Parts; nomatch -> ?ERROR("Invalid semantic version: '~s'~n", [Version]), erlang:halt(22) diff --git a/bin/nodetool b/bin/nodetool index ab2210aa5..c0d5b0025 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -21,12 +21,13 @@ main(Args) -> ok end end, - ok = add_libs_dir(), case Args of ["hocon" | Rest] -> + ok = add_libs_dir(), %% forward the call to hocon_cli hocon_cli:main(Rest); ["check_license_key", Key0] -> + ok = add_libs_dir(), Key = cleanup_key(Key0), check_license(#{key => Key}); _ -> diff --git a/build b/build index 8b485f3b6..c70f91b60 100755 --- a/build +++ b/build @@ -12,6 +12,12 @@ if [ "${DEBUG:-0}" -eq 1 ]; then export DIAGNOSTIC=1 fi +log_red() { + local RED='\033[0;31m' # Red + local NC='\033[0m' # No Color + echo -e "${RED}${1}${NC}" +} + PROFILE_ARG="$1" ARTIFACT="$2" @@ -34,7 +40,7 @@ case "$(is_enterprise "$PROFILE_ARG"),$(is_enterprise "$PROFILE_ENV")" in true ;; *) - echo "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'" + log_red "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'" exit 1 ;; esac @@ -133,6 +139,14 @@ make_docs() { erl -noshell -eval \ "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." + local desc="$docdir/desc.en.hocon" + if command -v jq &> /dev/null; then + log "Generating $desc" + scripts/merge-i18n.escript | jq --sort-keys . > "$desc" + else + # it is not a big deal if we cannot generate the desc + log_red "NOT Generated: $desc" + fi } ## arg1 is the profile for which the following args (as app names) should be excluded @@ -149,8 +163,8 @@ assert_no_excluded_deps() { for app in "${excluded_apps[@]}"; do found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")" if [ -n "${found}" ]; then - echo "ERROR: ${app} should not be included in ${PROFILE}" - echo "ERROR: found ${app} in ${rel_dir}" + log_red "ERROR: ${app} should not be included in ${PROFILE}" + log_red "ERROR: found ${app} in ${rel_dir}" exit 1 fi done @@ -291,7 +305,7 @@ make_tgz() { mkdir -p "${tard}/emqx" mkdir -p "${pkgpath}" if [ ! -f "$src_tarball" ]; then - log "ERROR: $src_tarball is not found" + log_red "ERROR: $src_tarball is not found" fi $TAR zxf "${src_tarball}" -C "${tard}/emqx" if [ -f "${tard}/emqx/releases/${PKG_VSN}/relup" ]; then diff --git a/changes/ce/feat-11725.en.md b/changes/ce/feat-11725.en.md new file mode 100644 index 000000000..ce5b08169 --- /dev/null +++ b/changes/ce/feat-11725.en.md @@ -0,0 +1 @@ +Introduced the LDAP as a new authentication and authorization backend. diff --git a/changes/ce/feat-11752.en.md b/changes/ce/feat-11752.en.md new file mode 100644 index 000000000..0cf0d5f6f --- /dev/null +++ b/changes/ce/feat-11752.en.md @@ -0,0 +1,3 @@ +Change default RPC driver from 'gen_rpc' to 'rpc' for core-replica database sync. + +This improves core-replica data replication latency. diff --git a/changes/ce/feat-11785.en.md b/changes/ce/feat-11785.en.md new file mode 100644 index 000000000..765ce6ea0 --- /dev/null +++ b/changes/ce/feat-11785.en.md @@ -0,0 +1 @@ +Allow viewer to change their own passwords, viewer can't change other's password. diff --git a/changes/ce/feat-11787.en.md b/changes/ce/feat-11787.en.md new file mode 100644 index 000000000..2dc3efc73 --- /dev/null +++ b/changes/ce/feat-11787.en.md @@ -0,0 +1,3 @@ +Improve `emqx` command performance. + +Avoid loading EMQX application code in `nodetool` script unless necessary. diff --git a/changes/ce/feat-11790.en.md b/changes/ce/feat-11790.en.md new file mode 100644 index 000000000..c2ceee216 --- /dev/null +++ b/changes/ce/feat-11790.en.md @@ -0,0 +1,3 @@ +Added validation of Redis commands configured in Redis authorization source. +Also, improved Redis command parsing in authentication and authorization +so that it is `redis-cli` compatible and supports quoted arguments. diff --git a/changes/ce/fix-11757.en.md b/changes/ce/fix-11757.en.md new file mode 100644 index 000000000..cc57e70a8 --- /dev/null +++ b/changes/ce/fix-11757.en.md @@ -0,0 +1 @@ +Fixed 500 error response when downloading non-existent trace files, now returns 404. diff --git a/changes/ce/fix-11762.en.md b/changes/ce/fix-11762.en.md new file mode 100644 index 000000000..b2276d08c --- /dev/null +++ b/changes/ce/fix-11762.en.md @@ -0,0 +1 @@ +Fixed destruction of built_in_database authorization source. Now all the ACL records are removed when the authorization source is destroyed. Previosly, old records were left in the database, which could cause problems when creating authorization source back. diff --git a/changes/ce/fix-11771.en.md b/changes/ce/fix-11771.en.md new file mode 100644 index 000000000..bc91dfaa5 --- /dev/null +++ b/changes/ce/fix-11771.en.md @@ -0,0 +1 @@ +Fixed validation of Bcrypt salt rounds in authentication management through the API/Dashboard. diff --git a/changes/ce/fix-11780.en.md b/changes/ce/fix-11780.en.md new file mode 100644 index 000000000..549707ffb --- /dev/null +++ b/changes/ce/fix-11780.en.md @@ -0,0 +1 @@ +Fixed validation of the `iterations` field of the `pbkdf2` password hashing algorithm. Now, `iterations` must be strictly positive. Previously, it could be set to 0, which led to a nonfunctional authenticator. diff --git a/changes/ce/fix-11791.en.md b/changes/ce/fix-11791.en.md new file mode 100644 index 000000000..983347605 --- /dev/null +++ b/changes/ce/fix-11791.en.md @@ -0,0 +1 @@ +Fixed an issue that prevented heartbeats from correctly keeping the CoAP Gateway connections alive. diff --git a/changes/ce/fix-11797.en.md b/changes/ce/fix-11797.en.md new file mode 100644 index 000000000..6227e079c --- /dev/null +++ b/changes/ce/fix-11797.en.md @@ -0,0 +1 @@ +Modified HTTP API behavior for APIs managing the `built_in_database` authorization source: They will now return a `404` status code if `built_in_database` is not set as the authorization source, replacing the former `20X` response. diff --git a/changes/ee/feat-11541.en.md b/changes/ee/feat-11541.en.md new file mode 100644 index 000000000..dee06609d --- /dev/null +++ b/changes/ee/feat-11541.en.md @@ -0,0 +1,3 @@ +Introduced additional way of file transfer interactions. Now client may send file transfer commands to `$file-async/...` topic instead of `$file/...` and receive command execution results as messages to `$file-response/{clientId}` topic. +This simplifies file transfer feature usage in certain cases, for example, when a client uses MQTTv3 or when the broker is behind an MQTT bridge. +See the [EIP-0021](https://github.com/emqx/eip) for more details. diff --git a/changes/ee/fix-11731.en.md b/changes/ee/fix-11731.en.md new file mode 100644 index 000000000..5584da337 --- /dev/null +++ b/changes/ee/fix-11731.en.md @@ -0,0 +1 @@ +Add file_transfer feature configs to hot-config schema. diff --git a/changes/ee/fix-11754.en.md b/changes/ee/fix-11754.en.md new file mode 100644 index 000000000..5aa2bed15 --- /dev/null +++ b/changes/ee/fix-11754.en.md @@ -0,0 +1 @@ +Improved log formatting for Postgres bridge when there are unicode characters in the error messages returned by the driver. diff --git a/dev b/dev index 7622c72fc..38ab7bee3 100755 --- a/dev +++ b/dev @@ -227,8 +227,6 @@ prepare_erl_libs() { for app in "_build/${profile}/checkouts"/*; do erl_libs="${erl_libs}${sep}${app}" done - else - echo "no checkouts" fi export ERL_LIBS="$erl_libs" } @@ -336,13 +334,6 @@ copy_other_conf_files() { is_current_profile_app() { local app="$1" case "$app" in - lib-ee*) - if [ "$PROFILE" = 'emqx-enterprise' ]; then - return 0 - else - return 1 - fi - ;; *emqx_telemetry*) if [ "$PROFILE" = 'emqx-enterprise' ]; then return 1 diff --git a/elvis.config b/elvis.config index ee7eaeaee..87d739865 100644 --- a/elvis.config +++ b/elvis.config @@ -5,7 +5,7 @@ [ {config, [ - #{dirs => ["src", "apps/**/src", "lib-ee/**/src"], + #{dirs => ["src", "apps/**/src"], filter => "*.erl", ruleset => erl_files, rules => [ diff --git a/git-blame-ignore-revs b/git-blame-ignore-revs index 41c6e5e49..c1bb2f5ea 100644 --- a/git-blame-ignore-revs +++ b/git-blame-ignore-revs @@ -33,5 +33,7 @@ b4451823350ec46126c49ca915b4b169dd4cf49e a4feb3e6e95c18cb531416112e57520c5ba00d40 # reformat apps/emqx_dashboard 07444e3da53c408695630bc0f57340f557106942 -# reformat all remaning apps +# reformat all remaining apps 02c3f87b316e8370287d5cd46de4f103ffe48433 +# erlfmt all remaining escripts +72eb34658d31fb38130421949cff262efab51139 diff --git a/lib-ee/BSL.txt b/lib-ee/BSL.txt deleted file mode 100644 index 0acc0e696..000000000 --- a/lib-ee/BSL.txt +++ /dev/null @@ -1,94 +0,0 @@ -Business Source License 1.1 - -Licensor: Hangzhou EMQ Technologies Co., Ltd. -Licensed Work: EMQX Enterprise Edition - The Licensed Work is (c) 2023 - Hangzhou EMQ Technologies Co., Ltd. -Additional Use Grant: Students and educators are granted right to copy, - modify, and create derivative work for research - or education. -Change Date: 2027-02-01 -Change License: Apache License, Version 2.0 - -For information about alternative licensing arrangements for the Software, -please contact Licensor: https://www.emqx.com/en/contact - -Notice - -The Business Source License (this document, or the “License”) is not an Open -Source license. However, the Licensed Work will eventually be made available -under an Open Source License, as stated in this License. - -License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. -“Business Source License” is a trademark of MariaDB Corporation Ab. - ------------------------------------------------------------------------------ - -Business Source License 1.1 - -Terms - -The Licensor hereby grants you the right to copy, modify, create derivative -works, redistribute, and make non-production use of the Licensed Work. The -Licensor may make an Additional Use Grant, above, permitting limited -production use. - -Effective on the Change Date, or the fourth anniversary of the first publicly -available distribution of a specific version of the Licensed Work under this -License, whichever comes first, the Licensor hereby grants you rights under -the terms of the Change License, and the rights granted in the paragraph -above terminate. - -If your use of the Licensed Work does not comply with the requirements -currently in effect as described in this License, you must purchase a -commercial license from the Licensor, its affiliated entities, or authorized -resellers, or you must refrain from using the Licensed Work. - -All copies of the original and modified Licensed Work, and derivative works -of the Licensed Work, are subject to this License. This License applies -separately for each version of the Licensed Work and the Change Date may vary -for each version of the Licensed Work released by Licensor. - -You must conspicuously display this License on each original or modified copy -of the Licensed Work. If you receive the Licensed Work in original or -modified form from a third party, the terms and conditions set forth in this -License apply to your use of that work. - -Any use of the Licensed Work in violation of this License will automatically -terminate your rights under this License for the current and all other -versions of the Licensed Work. - -This License does not grant you any right in any trademark or logo of -Licensor or its affiliates (provided that you may use a trademark or logo of -Licensor as expressly required by this License). - -TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON -AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, -EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND -TITLE. - -MariaDB hereby grants you permission to use this License’s text to license -your works, and to refer to it using the trademark “Business Source License”, -as long as you comply with the Covenants of Licensor below. - -Covenants of Licensor - -In consideration of the right to use this License’s text and the “Business -Source License” name and trademark, Licensor covenants to MariaDB, and to all -other recipients of the licensed work to be provided by Licensor: - -1. To specify as the Change License the GPL Version 2.0 or any later version, - or a license that is compatible with GPL Version 2.0 or a later version, - where “compatible” means that software provided under the Change License can - be included in a program with software provided under GPL Version 2.0 or a - later version. Licensor may specify additional Change Licenses without - limitation. - -2. To either: (a) specify an additional grant of rights to use that does not - impose any additional restriction on the right granted in this License, as - the Additional Use Grant; or (b) insert the text “None”. - -3. To specify a Change Date. - -4. Not to modify this License in any other way. diff --git a/mix.exs b/mix.exs index 80bf57875..4a2c2f377 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.19", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.40.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -169,24 +169,10 @@ defmodule EMQXUmbrella.MixProject do end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do - umbrella_apps = - Enum.map(enterprise_umbrella_apps(), fn app_name -> - path = "apps/#{app_name}" - {app_name, path: path, manager: :rebar3, override: true} - end) - - "lib-ee/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) - |> Enum.map(fn path -> - app = - path - |> Path.basename() - |> String.to_atom() - - {app, path: path, manager: :rebar3, override: true} + Enum.map(enterprise_umbrella_apps(), fn app_name -> + path = "apps/#{app_name}" + {app_name, path: path, manager: :rebar3, override: true} end) - |> Enum.concat(umbrella_apps) end defp enterprise_apps(_profile_info) do @@ -220,13 +206,12 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_rabbitmq, :emqx_bridge_clickhouse, :emqx_ft, + :emqx_license, :emqx_s3, :emqx_schema_registry, :emqx_enterprise, :emqx_bridge_kinesis, :emqx_bridge_azure_event_hub, - :emqx_ldap, - :emqx_auth_ldap, :emqx_gcp_device, :emqx_dashboard_rbac, :emqx_dashboard_sso @@ -353,6 +338,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_management, :emqx_retainer, :emqx_prometheus, + :emqx_rule_engine, :emqx_auto_subscribe, :emqx_slow_subs, :emqx_plugins, diff --git a/rebar.config b/rebar.config index b0d365f39..151b115f6 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.19"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} @@ -106,6 +106,10 @@ emqx_exproto_pb % generated code for protobuf ]}. +{eunit_opts, + [ verbose + ]}. + {project_plugins, [ erlfmt, {rebar3_hex, "7.0.2"}, diff --git a/rebar.config.erl b/rebar.config.erl index 6d2c05a34..2d9ec926a 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -107,8 +107,6 @@ is_community_umbrella_app("apps/emqx_schema_registry") -> false; is_community_umbrella_app("apps/emqx_enterprise") -> false; is_community_umbrella_app("apps/emqx_bridge_kinesis") -> false; is_community_umbrella_app("apps/emqx_bridge_azure_event_hub") -> false; -is_community_umbrella_app("apps/emqx_ldap") -> false; -is_community_umbrella_app("apps/emqx_auth_ldap") -> false; is_community_umbrella_app("apps/emqx_gcp_device") -> false; is_community_umbrella_app("apps/emqx_dashboard_rbac") -> false; is_community_umbrella_app("apps/emqx_dashboard_sso") -> false; @@ -164,11 +162,7 @@ project_app_dirs(Edition) -> || Path <- filelib:wildcard("apps/*"), is_community_umbrella_app(Path) orelse IsEnterprise ], - UmbrellaApps ++ - case IsEnterprise of - true -> ["lib-ee/*"]; - false -> [] - end. + UmbrellaApps. plugins() -> [ @@ -539,8 +533,7 @@ provide_bcrypt_release(ReleaseType) -> erl_opts_i() -> [{i, "apps"}] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))]. + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))]. dialyzer(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), @@ -597,7 +590,7 @@ coveralls() -> [] end. -app_names() -> list_dir("apps") ++ list_dir("lib-ee"). +app_names() -> list_dir("apps"). list_dir(Dir) -> case filelib:is_dir(Dir) of diff --git a/rel/i18n/emqx_bridge_api.hocon b/rel/i18n/emqx_bridge_api.hocon index 8b7950cdc..3567f03cc 100644 --- a/rel/i18n/emqx_bridge_api.hocon +++ b/rel/i18n/emqx_bridge_api.hocon @@ -49,7 +49,7 @@ desc_api8.label: """Node Bridge Operate""" desc_api9.desc: -"""Test creating a new bridge by given ID
+"""Test creating a new bridge by given ID
The ID must be of format '{type}:{name}'""" desc_api9.label: diff --git a/rel/i18n/emqx_bridge_cassandra.hocon b/rel/i18n/emqx_bridge_cassandra.hocon index d598d3921..a96315340 100644 --- a/rel/i18n/emqx_bridge_cassandra.hocon +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -32,7 +32,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_clickhouse.hocon b/rel/i18n/emqx_bridge_clickhouse.hocon index 726d1eb7c..7d1961f98 100644 --- a/rel/i18n/emqx_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_bridge_clickhouse.hocon @@ -32,7 +32,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to Clickhouse. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo.hocon index 417b43c0c..a014aae9f 100644 --- a/rel/i18n/emqx_bridge_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to DynamoDB. All MQTT `PUBLISH` messages with the topic -matching the `local_topic` will be forwarded.
+matching the `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also `local_topic` is configured, then both the data got from the rule and the MQTT messages that match `local_topic` will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_bridge_gcp_pubsub.hocon index b5dffec1f..68a6f8578 100644 --- a/rel/i18n/emqx_bridge_gcp_pubsub.hocon +++ b/rel/i18n/emqx_bridge_gcp_pubsub.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to GCP PubSub. All MQTT 'PUBLISH' messages with the topic -matching `local_topic` will be forwarded.
+matching `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_greptimedb.hocon b/rel/i18n/emqx_bridge_greptimedb.hocon index 93d783332..977e6e064 100644 --- a/rel/i18n/emqx_bridge_greptimedb.hocon +++ b/rel/i18n/emqx_bridge_greptimedb.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to the GreptimeDB. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" @@ -37,8 +37,8 @@ local_topic.label: write_syntax.desc: """Conf of GreptimeDB gRPC protocol to write data points. Write syntax is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported, which is the same as InfluxDB line protocol. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and -[GreptimeDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR:
+[GreptimeDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR:
``` [,=[,=]] =[,=] [] ``` diff --git a/rel/i18n/emqx_bridge_greptimedb_connector.hocon b/rel/i18n/emqx_bridge_greptimedb_connector.hocon index 9cb10951f..0a509ebfc 100644 --- a/rel/i18n/emqx_bridge_greptimedb_connector.hocon +++ b/rel/i18n/emqx_bridge_greptimedb_connector.hocon @@ -31,8 +31,8 @@ protocol.label: """Protocol""" server.desc: -"""The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The GreptimeDB default port 8086 is used if `[:Port]` is not specified.""" server.label: diff --git a/rel/i18n/emqx_bridge_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb.hocon index 809c60588..de9989953 100644 --- a/rel/i18n/emqx_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb.hocon @@ -32,7 +32,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_influxdb.hocon b/rel/i18n/emqx_bridge_influxdb.hocon index 4299f41ab..48454bbd3 100644 --- a/rel/i18n/emqx_bridge_influxdb.hocon +++ b/rel/i18n/emqx_bridge_influxdb.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" @@ -37,8 +37,8 @@ local_topic.label: write_syntax.desc: """Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and -[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR:
+[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR:
``` [,=[,=]] =[,=] [] ``` diff --git a/rel/i18n/emqx_bridge_influxdb_connector.hocon b/rel/i18n/emqx_bridge_influxdb_connector.hocon index 4169ce065..ce79c2a93 100644 --- a/rel/i18n/emqx_bridge_influxdb_connector.hocon +++ b/rel/i18n/emqx_bridge_influxdb_connector.hocon @@ -49,8 +49,8 @@ protocol.label: """Protocol""" server.desc: -"""The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port]`.
+"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
The InfluxDB default port 8086 is used if `[:Port]` is not specified.""" server.label: diff --git a/rel/i18n/emqx_bridge_kinesis.hocon b/rel/i18n/emqx_bridge_kinesis.hocon index 42329bcd6..188ab82f3 100644 --- a/rel/i18n/emqx_bridge_kinesis.hocon +++ b/rel/i18n/emqx_bridge_kinesis.hocon @@ -32,7 +32,7 @@ pool_size.label: local_topic.desc: """The MQTT topic filter to be forwarded to Amazon Kinesis. All MQTT `PUBLISH` messages with the topic -matching the `local_topic` will be forwarded.
+matching the `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also `local_topic` is configured, then both the data got from the rule and the MQTT messages that match `local_topic` will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_mysql.hocon b/rel/i18n/emqx_bridge_mysql.hocon index 10a02589c..37326be81 100644 --- a/rel/i18n/emqx_bridge_mysql.hocon +++ b/rel/i18n/emqx_bridge_mysql.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_oracle.hocon b/rel/i18n/emqx_bridge_oracle.hocon index c0c8142e6..bcf41ea2c 100644 --- a/rel/i18n/emqx_bridge_oracle.hocon +++ b/rel/i18n/emqx_bridge_oracle.hocon @@ -2,7 +2,7 @@ emqx_bridge_oracle { local_topic { desc = "The MQTT topic filter to be forwarded to Oracle Database. All MQTT 'PUBLISH' messages with the topic" - " matching the local_topic will be forwarded.
" + " matching the local_topic will be forwarded.
" "NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is" " configured, then both the data got from the rule and the MQTT messages that match local_topic" " will be forwarded." diff --git a/rel/i18n/emqx_bridge_pgsql.hocon b/rel/i18n/emqx_bridge_pgsql.hocon index 5295abb35..0a5ca2b04 100644 --- a/rel/i18n/emqx_bridge_pgsql.hocon +++ b/rel/i18n/emqx_bridge_pgsql.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_redis.hocon b/rel/i18n/emqx_bridge_redis.hocon index 8e8c18de0..05c8d95a6 100644 --- a/rel/i18n/emqx_bridge_redis.hocon +++ b/rel/i18n/emqx_bridge_redis.hocon @@ -34,7 +34,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to Redis. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_rocketmq.hocon b/rel/i18n/emqx_bridge_rocketmq.hocon index ac5deb757..a2449c1a9 100644 --- a/rel/i18n/emqx_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_bridge_rocketmq.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic -matching the `local_topic` will be forwarded.
+matching the `local_topic` will be forwarded.
NOTE: if the bridge is used as a rule action, `local_topic` should be left empty otherwise the messages will be duplicated.""" local_topic.label: diff --git a/rel/i18n/emqx_bridge_sqlserver.hocon b/rel/i18n/emqx_bridge_sqlserver.hocon index 0e0801f42..24e4615f3 100644 --- a/rel/i18n/emqx_bridge_sqlserver.hocon +++ b/rel/i18n/emqx_bridge_sqlserver.hocon @@ -32,7 +32,7 @@ driver.label: local_topic.desc: """The MQTT topic filter to be forwarded to Microsoft SQL Server. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_bridge_tdengine.hocon b/rel/i18n/emqx_bridge_tdengine.hocon index 2d1059d28..ec6c10779 100644 --- a/rel/i18n/emqx_bridge_tdengine.hocon +++ b/rel/i18n/emqx_bridge_tdengine.hocon @@ -26,7 +26,7 @@ desc_type.label: local_topic.desc: """The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
+matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index b2958ce90..1a0ed5273 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -2,114 +2,33 @@ emqx_limiter_schema { max_conn_rate.desc: """Maximum connection rate.
-This is used to limit the connection rate for this node, -once the limit is reached, new connections will be deferred or refused""" +This is used to limit the connection rate for this node. +Once the limit is reached, new connections will be deferred or refused.
+For example:
+- 1000/s :: Only accepts 1000 connections per second
+- 1000/10s :: Only accepts 1000 connections every 10 seconds""" max_conn_rate.label: """Maximum Connection Rate""" messages_rate.desc: """Messages publish rate.
-This is used to limit the inbound message numbers for this node, -once the limit is reached, the restricted client will slow down and even be hung for a while.""" +This is used to limit the inbound message numbers for this node. +Once the limit is reached, the restricted client will slow down and even be hung for a while.
+For example:
+- 500/s :: Only the first 500 messages are sent per second and other messages are buffered.
+- 500/10s :: Only the first 500 messages are sent even 10 second and other messages are buffered.""" messages_rate.label: """Messages Publish Rate""" bytes_rate.desc: """Data publish rate.
-This is used to limit the inbound bytes rate for this node, -once the limit is reached, the restricted client will slow down and even be hung for a while.""" +This is used to limit the inbound bytes rate for this node. +Once the limit is reached, the restricted client will slow down and even be hung for a while.
+The unit of the bytes could be:KB MB GB.
+For example:
+- 500KB/s :: Only the first 500 kilobytes are sent per second and other messages are buffered.
+- 500MB/10s :: Only the first 500 megabytes are sent even 10 second and other messages are buffered.""" bytes_rate.label: """Data Publish Rate""" -bucket_cfg.desc: -"""Bucket Configs""" - -bucket_cfg.label: -"""Buckets""" - -burst.desc: -"""The burst, This value is based on rate.
- This value + rate = the maximum limit that can be achieved when limiter burst.""" - -burst.label: -"""Burst""" - -bytes.desc: -"""The `bytes` limiter. -This is used to limit the inbound bytes rate for this EMQX node. -Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - -bytes.label: -"""Bytes""" - -client.desc: -"""The rate limit for each user of the bucket""" - -client.label: -"""Per Client""" - -connection.desc: -"""The connection limiter. -This is used to limit the connection rate for this EMQX node. -Once the limit is reached, new connections will be refused""" - -connection.label: -"""Connection""" - -divisible.desc: -"""Is it possible to split the number of requested tokens?""" - -divisible.label: -"""Divisible""" - -failure_strategy.desc: -"""The strategy when all the retries failed.""" - -failure_strategy.label: -"""Failure Strategy""" - -initial.desc: -"""The initial number of tokens for this bucket.""" - -initial.label: -"""Initial""" - -internal.desc: -"""Limiter for EMQX internal app.""" - -low_watermark.desc: -"""If the remaining tokens are lower than this value, -the check/consume will succeed, but it will be forced to wait for a short period of time.""" - -low_watermark.label: -"""Low Watermark""" - -max_retry_time.desc: -"""The maximum retry time when acquire failed.""" - -max_retry_time.label: -"""Max Retry Time""" - -message_routing.desc: -"""The message routing limiter. -This is used to limit the forwarding rate for this EMQX node. -Once the limit is reached, new publish will be refused""" - -message_routing.label: -"""Message Routing""" - -messages.desc: -"""The `messages` limiter. -This is used to limit the inbound message numbers for this EMQX node -Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - -messages.label: -"""Messages""" - -rate.desc: -"""Rate for this bucket.""" - -rate.label: -"""Rate""" - } diff --git a/rel/i18n/emqx_connector_pgsql.hocon b/rel/i18n/emqx_postgresql.hocon similarity index 91% rename from rel/i18n/emqx_connector_pgsql.hocon rename to rel/i18n/emqx_postgresql.hocon index 485e666a0..c6d2581c1 100644 --- a/rel/i18n/emqx_connector_pgsql.hocon +++ b/rel/i18n/emqx_postgresql.hocon @@ -1,4 +1,4 @@ -emqx_connector_pgsql { +emqx_postgresql { server.desc: """The IPv4 or IPv6 address or the hostname to connect to.
diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 9ed579994..3eb816f3b 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -92,7 +92,7 @@ mqtt_max_topic_alias.label: """Max Topic Alias""" common_ssl_opts_schema_user_lookup_fun.desc: -"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.
+"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.
Has no effect when TLS version is configured (or negotiated) to 1.3""" common_ssl_opts_schema_user_lookup_fun.label: @@ -1039,27 +1039,6 @@ base_listener_limiter.desc: base_listener_limiter.label: """Type of the rate limit.""" -max_conn_rate.desc: -"""Maximum connection rate.
-This is used to limit the connection rate for this listener, -once the limit is reached, new connections will be deferred or refused""" -max_conn_rate.label: -"""Maximum Connection Rate""" - -messages_rate.desc: -"""Messages publish rate.
-This is used to limit the inbound message numbers for each client connected to this listener, -once the limit is reached, the restricted client will slow down and even be hung for a while.""" -messages_rate.label: -"""Messages Publish Rate""" - -bytes_rate.desc: -"""Data publish rate.
-This is used to limit the inbound bytes rate for each client connected to this listener, -once the limit is reached, the restricted client will slow down and even be hung for a while.""" -bytes_rate.label: -"""Data Publish Rate""" - persistent_session_store_backend.desc: """Database management system used to store information about persistent sessions and messages. - `builtin`: Use the embedded database (mria)""" @@ -1228,7 +1207,7 @@ The SSL application already takes measures to counter-act such attempts, but client-initiated renegotiation can be strictly disabled by setting this option to false. The default value is true. Note that disabling renegotiation can result in long-lived connections becoming unusable due to limits on -the number of messages the underlying cipher suite can encipher.
+the number of messages the underlying cipher suite can encipher.
Has no effect when TLS version is configured (or negotiated) to 1.3""" server_ssl_opts_schema_client_renegotiation.label: @@ -1315,7 +1294,7 @@ common_ssl_opts_schema_secure_renegotiate.desc: """SSL parameter renegotiation is a feature that allows a client and a server to renegotiate the parameters of the SSL connection on the fly. RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -you drop support for the insecure renegotiation, prone to MitM attacks.
+you drop support for the insecure renegotiation, prone to MitM attacks.
Has no effect when TLS version is configured (or negotiated) to 1.3""" common_ssl_opts_schema_secure_renegotiate.label: @@ -1351,7 +1330,7 @@ mqtt_max_packet_size.label: """Max Packet Size""" common_ssl_opts_schema_reuse_sessions.desc: -"""Enable TLS session reuse.
+"""Enable TLS session reuse.
Has no effect when TLS version is configured (or negotiated) to 1.3""" common_ssl_opts_schema_reuse_sessions.label: diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index b32b39fd1..b76e8d345 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -4,6 +4,12 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +log_red() { + local RED='\033[0;31m' # Red + local NC='\033[0m' # No Color + echo -e "${RED}${1}${NC}" +} + # match any official release tag 'e*' and 'v*' latest_release="$(env PREV_TAG_MATCH_PATTERN='*' ./scripts/find-prev-rel-tag.sh)" echo "Version check compare base: $latest_release" @@ -47,7 +53,7 @@ for app in ${APPS}; do -- "$app_path/priv" \ -- "$app_path/c_src" | wc -l ) " if [ "$changed_lines" -gt 0 ]; then - echo "ERROR: $src_file needs a vsn bump" + log_red "ERROR: $src_file needs a vsn bump" bad_app_count=$(( bad_app_count + 1)) fi else diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript index 738aeec31..304c771fd 100755 --- a/scripts/check-deps-integrity.escript +++ b/scripts/check-deps-integrity.escript @@ -5,9 +5,7 @@ -mode(compile). main([]) -> - Files = ["rebar.config"] ++ - apps_rebar_config("apps") ++ - apps_rebar_config("lib-ee"), + Files = ["rebar.config"] ++ apps_rebar_config("apps"), Deps = collect_deps(Files, #{}), case count_bad_deps(Deps) of 0 -> @@ -22,7 +20,8 @@ apps_rebar_config(Dir) -> %% collect a kv-list of {DepName, [{DepReference, RebarConfigFile}]} %% the value part should have unique DepReference -collect_deps([], Acc) -> maps:to_list(Acc); +collect_deps([], Acc) -> + maps:to_list(Acc); collect_deps([File | Files], Acc) -> Deps = try @@ -30,12 +29,13 @@ collect_deps([File | Files], Acc) -> {deps, Deps0} = lists:keyfind(deps, 1, Config), Deps0 catch - C : E : St -> + C:E:St -> erlang:raise(C, {E, {failed_to_find_deps_in_rebar_config, File}}, St) end, collect_deps(Files, do_collect_deps(Deps, File, Acc)). -do_collect_deps([], _File, Acc) -> Acc; +do_collect_deps([], _File, Acc) -> + Acc; %% ignore relative app dependencies do_collect_deps([{_Name, {path, _Path}} | Deps], File, Acc) -> do_collect_deps(Deps, File, Acc); @@ -43,7 +43,8 @@ do_collect_deps([{Name, Ref} | Deps], File, Acc) -> Refs = maps:get(Name, Acc, []), do_collect_deps(Deps, File, Acc#{Name => [{Ref, File} | Refs]}). -count_bad_deps([]) -> 0; +count_bad_deps([]) -> + 0; count_bad_deps([{Name, Refs0} | Rest]) -> Refs = lists:keysort(1, Refs0), case is_unique_ref(Refs) andalso not_branch_ref(Refs) of @@ -55,10 +56,8 @@ count_bad_deps([{Name, Refs0} | Rest]) -> end. is_unique_ref([_]) -> true; -is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> - is_unique_ref([{Ref, File2} | Rest]); -is_unique_ref(_) -> - false. +is_unique_ref([{Ref, _File1}, {Ref, File2} | Rest]) -> is_unique_ref([{Ref, File2} | Rest]); +is_unique_ref(_) -> false. not_branch_ref([]) -> true; not_branch_ref([{{git, _Repo, {branch, _Branch}}, _File} | _Rest]) -> false; diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index f48e5a513..4aea1b6d5 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -8,10 +8,9 @@ -define(RED, "\e[31m"). -define(RESET, "\e[39m"). -main([Files0]) -> +main(Files) -> io:format(user, "checking i18n file styles~n", []), _ = put(errors, 0), - Files = string:tokens(Files0, "\n"), ok = load_hocon(), ok = lists:foreach(fun check/1, Files), case get(errors) of @@ -47,7 +46,6 @@ logerr(Fmt, Args) -> _ = put(errors, N + 1), ok. - check(File) -> io:format(user, ".", []), {ok, C} = hocon:load(File), @@ -55,9 +53,12 @@ check(File) -> ok. check_one_field(Name, Field) -> - maps:foreach(fun(SubName, DescAndLabel) -> - check_desc_and_label([Name, ".", SubName], DescAndLabel) - end, Field). + maps:foreach( + fun(SubName, DescAndLabel) -> + check_desc_and_label([Name, ".", SubName], DescAndLabel) + end, + Field + ). check_desc_and_label(Name, D) -> case maps:keys(D) -- [<<"desc">>, <<"label">>] of @@ -85,8 +86,8 @@ check_desc_string(Name, <<>>) -> check_desc_string(Name, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> - logerr("~s: ~s~n", [Name, Reason]) - end, + logerr("~s: ~s~n", [Name, Reason]) + end, case Str of [$\s | _] -> Err("remove leading whitespace"); diff --git a/scripts/check-i18n-style.sh b/scripts/check-i18n-style.sh index d21f43a72..b7d4d2113 100755 --- a/scripts/check-i18n-style.sh +++ b/scripts/check-i18n-style.sh @@ -3,6 +3,4 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")/.." -all_files="$(git ls-files 'rel/i18n/*.hocon')" - -./scripts/check-i18n-style.escript "$all_files" +./scripts/check-i18n-style.escript rel/i18n/*.hocon diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 9f8f852b2..b6cec5d74 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -98,17 +98,16 @@ if [ ! -d "${WHICH_APP}" ]; then exit 1 fi -if [[ "${WHICH_APP}" == lib-ee* && (-z "${PROFILE+x}" || "${PROFILE}" != emqx-enterprise) ]]; then - echo 'You are trying to run an enterprise test case without the emqx-enterprise profile.' - echo 'This will most likely not work.' - echo '' - echo 'Run "export PROFILE=emqx-enterprise" and "make" to fix this' - exit 1 -fi - ERLANG_CONTAINER='erlang' DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" +if [ -f "${WHICH_APP}/BSL.txt" ]; then + if [ -n "${PROFILE:-}" ] && [ "${PROFILE}" != 'emqx-enterprise' ]; then + echo "bad_profile: PROFILE=${PROFILE} will not work for app ${WHICH_APP}" + exit 1 + fi +fi + if [ -z "${PROFILE+x}" ]; then case "${WHICH_APP}" in apps/emqx) @@ -127,10 +126,6 @@ if [ -z "${PROFILE+x}" ]; then apps/emqx_rule_engine) export PROFILE='emqx-enterprise' ;; - lib-ee*) - ## ensure enterprise profile when testing lib-ee applications - export PROFILE='emqx-enterprise' - ;; apps/*) if [[ -f "${WHICH_APP}/BSL.txt" ]]; then export PROFILE='emqx-enterprise' diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 30c4e4716..89f0a66e5 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -41,9 +41,7 @@ find_app() { "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } -CE="$(find_app 'apps')" -EE="$(find_app 'lib-ee')" -APPS_ALL="$(echo -e "${CE}\n${EE}")" +APPS_ALL="$(find_app 'apps')" if [ "$MODE" = 'list' ]; then echo "${APPS_ALL}" @@ -111,9 +109,6 @@ matrix() { fi entries+=("$(format_app_entry "$app" 1 "$profile" "$runner")") ;; - lib-ee/*) - entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")") - ;; *) echo "unknown app: $app" exit 1 diff --git a/scripts/gen-erlang-ls-config.sh b/scripts/gen-erlang-ls-config.sh index 0f5bcdecd..d1e6e44b7 100755 --- a/scripts/gen-erlang-ls-config.sh +++ b/scripts/gen-erlang-ls-config.sh @@ -73,7 +73,7 @@ deps_dirs() for dir in _build/default/lib/*; do app=$(basename "${dir}") ## Only add applications that are not part of EMQX umbrella project: - [ -d "apps/${app}" ] || [ -d "lib-ce/${app}" ] || [ -d "lib-ee/${app}" ] || + [ -d "apps/${app}" ] || echo " - \"${dir}\"" done } @@ -91,8 +91,6 @@ EOF cat < do_merge_desc_files(BaseConf, Cfgs) -> lists:foldl( - fun(CfgFile, Acc) -> - case filelib:is_regular(CfgFile) of - true -> - {ok, Bin1} = file:read_file(CfgFile), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Cfgs). + fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + [Acc, io_lib:nl(), Bin1]; + false -> + Acc + end + end, + BaseConf, + Cfgs + ). get_all_desc_files() -> Dir = filename:join(["rel", "i18n"]), diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript new file mode 100755 index 000000000..dfd76f01f --- /dev/null +++ b/scripts/merge-i18n.escript @@ -0,0 +1,41 @@ +#!/usr/bin/env escript + +%% This script is only used at build time to generate the merged desc.en.hocon in JSON format +%% but NOT the file generated to _build/$PROFILE/lib/emqx_dashboard/priv (which is HOCON format). +%% +%% The generated JSON file is used as the source of truth when translating to other languages. + +-mode(compile). + +-define(RED, "\e[31m"). +-define(RESET, "\e[39m"). + +main(_) -> + try + _ = hocon:module_info() + catch + _:_ -> + fail("hocon module not found, please make sure the project is compiled") + end, + %% wildcard all .hocon files in rel/i18n + Files = filelib:wildcard("rel/i18n/*.hocon"), + case Files of + [_ | _] -> + ok; + [] -> + fail("No .hocon files found in rel/i18n") + end, + case hocon:files(Files) of + {ok, Map} -> + JSON = jiffy:encode(Map), + io:format("~s~n", [JSON]); + {error, Reason} -> + fail("~p~n", [Reason]) + end. + +fail(Str) -> + fail(Str, []). + +fail(Str, Args) -> + io:format(standard_error, ?RED ++ "ERROR: " ++ Str ++ ?RESET ++ "~n", Args), + halt(1). diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 632aabfe4..dfad7c869 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -25,10 +25,12 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." # generate merged config files and English translation of the desc (desc.en.hocon) ./scripts/merge-config.escript +I18N_REPO_BRANCH="v$(./pkg-vsn.sh "${PROFILE_STR}" | tr -d '.' | cut -c 1-2)" + # download desc (i18n) translations curl -L --silent --show-error \ --output "apps/emqx_dashboard/priv/desc.zh.hocon" \ - 'https://raw.githubusercontent.com/emqx/emqx-i18n/main/desc.zh.hocon' + "https://raw.githubusercontent.com/emqx/emqx-i18n/${I18N_REPO_BRANCH}/desc.zh.hocon" # TODO # make sbom a build artifcat diff --git a/scripts/relup-build/inject-relup.escript b/scripts/relup-build/inject-relup.escript index b7d905979..7e252f741 100755 --- a/scripts/relup-build/inject-relup.escript +++ b/scripts/relup-build/inject-relup.escript @@ -20,9 +20,9 @@ inject_relup_file(File) -> case file:script(File) of {ok, {CurrRelVsn, UpVsnRUs, DnVsnRUs}} -> ?INFO("Injecting instructions to: ~p", [File]), - UpdatedContent = {CurrRelVsn, - inject_relup_instrs(up, UpVsnRUs), - inject_relup_instrs(down, DnVsnRUs)}, + UpdatedContent = + {CurrRelVsn, inject_relup_instrs(up, UpVsnRUs), + inject_relup_instrs(down, DnVsnRUs)}, file:write_file(File, term_to_text(UpdatedContent)); {ok, _BadFormat} -> ?ERROR("Bad formatted relup file: ~p", [File]), @@ -36,38 +36,49 @@ inject_relup_file(File) -> end. inject_relup_instrs(Type, RUs) -> - lists:map(fun({Vsn, Desc, Instrs}) -> - {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} - end, RUs). + lists:map( + fun({Vsn, Desc, Instrs}) -> + {Vsn, Desc, append_emqx_relup_instrs(Type, Vsn, Instrs)} + end, + RUs + ). append_emqx_relup_instrs(up, FromRelVsn, Instrs0) -> - {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), - Instrs1 ++ - [ {load, {emqx_release, brutal_purge, soft_purge}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} + {{UpExtra, _}, Instrs1} = filter_and_check_instrs(up, Instrs0), + Instrs1 ++ + [ + {load, {emqx_release, brutal_purge, soft_purge}}, + {load, {emqx_relup, brutal_purge, soft_purge}}, + {apply, {emqx_relup, post_release_upgrade, [FromRelVsn, UpExtra]}} ]; - append_emqx_relup_instrs(down, ToRelVsn, Instrs0) -> {{_, DnExtra}, Instrs1} = filter_and_check_instrs(down, Instrs0), %% NOTE: When downgrading, we apply emqx_relup:post_release_downgrade/2 before reloading %% or removing the emqx_relup module. - Instrs2 = Instrs1 ++ - [ {load, {emqx_release, brutal_purge, soft_purge}} - , {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}} - , {load, {emqx_relup, brutal_purge, soft_purge}} - ], + Instrs2 = + Instrs1 ++ + [ + {load, {emqx_release, brutal_purge, soft_purge}}, + {apply, {emqx_relup, post_release_downgrade, [ToRelVsn, DnExtra]}}, + {load, {emqx_relup, brutal_purge, soft_purge}} + ], Instrs2. filter_and_check_instrs(Type, Instrs) -> case filter_fetch_emqx_mods_and_extra(Instrs) of {_, DnExtra, _, _} when Type =:= up, DnExtra =/= undefined -> - ?ERROR("Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" - " from the upgrade instruction list, should be 'post_release_upgrade'", []), + ?ERROR( + "Got '{apply,{emqx_relup,post_release_downgrade,[_,Extra]}}'" + " from the upgrade instruction list, should be 'post_release_upgrade'", + [] + ), error({instruction_not_found, load_object_code}); {UpExtra, _, _, _} when Type =:= down, UpExtra =/= undefined -> - ?ERROR("Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" - " from the downgrade instruction list, should be 'post_release_downgrade'", []), + ?ERROR( + "Got '{apply,{emqx_relup,post_release_upgrade,[_,Extra]}}'" + " from the downgrade instruction list, should be 'post_release_downgrade'", + [] + ), error({instruction_not_found, load_object_code}); {_, _, [], _} -> ?ERROR("Cannot find any 'load_object_code' instructions for app emqx", []), @@ -81,12 +92,15 @@ filter_fetch_emqx_mods_and_extra(Instrs) -> lists:foldl(fun do_filter_and_get/2, {undefined, undefined, [], []}, Instrs). %% collect modules for emqx app -do_filter_and_get({load_object_code, {emqx, _AppVsn, Mods}} = Instr, - {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {load_object_code, {emqx, _AppVsn, Mods}} = Instr, + {UpExtra, DnExtra, EmqxMods, RemainInstrs} +) -> {UpExtra, DnExtra, EmqxMods ++ Mods, RemainInstrs ++ [Instr]}; %% remove 'load' instrs for emqx_relup and emqx_release -do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) - when Mod =:= emqx_relup; Mod =:= emqx_release -> +do_filter_and_get({load, {Mod, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) when + Mod =:= emqx_relup; Mod =:= emqx_release +-> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; %% remove 'remove' and 'purge' instrs for emqx_relup do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> @@ -94,22 +108,31 @@ do_filter_and_get({remove, {emqx_relup, _, _}}, {UpExtra, DnExtra, EmqxMods, Rem do_filter_and_get({purge, [emqx_relup]}, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods, RemainInstrs}; %% remove 'apply' instrs for upgrade, and collect the 'Extra' parameter -do_filter_and_get({apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, - {_, DnExtra, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {apply, {emqx_relup, post_release_upgrade, [_, UpExtra0]}}, + {_, DnExtra, EmqxMods, RemainInstrs} +) -> {UpExtra0, DnExtra, EmqxMods, RemainInstrs}; %% remove 'apply' instrs for downgrade, and collect the 'Extra' parameter -do_filter_and_get({apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, - {UpExtra, _, EmqxMods, RemainInstrs}) -> +do_filter_and_get( + {apply, {emqx_relup, post_release_downgrade, [_, DnExtra0]}}, + {UpExtra, _, EmqxMods, RemainInstrs} +) -> {UpExtra, DnExtra0, EmqxMods, RemainInstrs}; %% keep all other instrs unchanged do_filter_and_get(Instr, {UpExtra, DnExtra, EmqxMods, RemainInstrs}) -> {UpExtra, DnExtra, EmqxMods, RemainInstrs ++ [Instr]}. assert_mandatory_modules(_, Mods) -> - MandInstrs = [{load_module,emqx_release,brutal_purge,soft_purge,[]}, - {load_module,emqx_relup}], - assert(lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods), - "The following instructions are mandatory in every clause of the emqx.appup.src: ~p", [MandInstrs]). + MandInstrs = [ + {load_module, emqx_release, brutal_purge, soft_purge, []}, + {load_module, emqx_relup} + ], + assert( + lists:member(emqx_relup, Mods) andalso lists:member(emqx_release, Mods), + "The following instructions are mandatory in every clause of the emqx.appup.src: ~p", + [MandInstrs] + ). assert(true, _, _) -> ok; diff --git a/scripts/rerun-failed-checks.py b/scripts/rerun-failed-checks.py deleted file mode 100644 index ff9b9f33e..000000000 --- a/scripts/rerun-failed-checks.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -# Usage: python3 rerun-failed-checks.py -t -r -b -# -# Description: This script will fetch the latest commit from a branch, and check the status of all check runs of the commit. -# If any check run is not successful, it will trigger a rerun of the failed jobs. -# -# Default branch is master, default repo is emqx/emqx -# -# Limitation: only works for upstream repo, not for forked. -import requests -import http.client -import json -import os -import sys -import time -import math -from optparse import OptionParser - -job_black_list = [ - 'windows', - 'publish_artifacts', - 'stale' -] - -def fetch_latest_commit(token: str, repo: str, branch: str): - url = f'https://api.github.com/repos/{repo}/commits/{branch}' - headers = {'Accept': 'application/vnd.github+json', - 'Authorization': f'Bearer {token}', - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'python3' - } - r = requests.get(url, headers=headers) - if r.status_code == 200: - res = r.json() - return res - else: - print( - f'Failed to fetch latest commit from {branch} branch, code: {r.status_code}') - sys.exit(1) - - -''' -fetch check runs of a commit. -@note, only works for public repos -''' -def fetch_check_runs(token: str, repo: str, ref: str): - all_checks = [] - page = 1 - total_pages = 1 - per_page = 100 - failed_checks = [] - while page <= total_pages: - print(f'Fetching check runs for page {page} of {total_pages} pages') - url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page={per_page}&page={page}' - headers = {'Accept': 'application/vnd.github.v3+json', - 'Authorization': f'Bearer {token}' - } - r = requests.get(url, headers=headers) - if r.status_code == 200: - resp = r.json() - all_checks.extend(resp['check_runs']) - - page += 1 - if 'total_count' in resp and resp['total_count'] > per_page: - total_pages = math.ceil(resp['total_count'] / per_page) - else: - print(f'Failed to fetch check runs {r.status_code}') - sys.exit(1) - - - for crun in all_checks: - if crun['status'] == 'completed' and crun['conclusion'] != 'success': - print('Failed check: ', crun['name']) - failed_checks.append( - {'id': crun['id'], 'name': crun['name'], 'url': crun['url']}) - else: - # pretty print crun - # print(json.dumps(crun, indent=4)) - print('successed:', crun['id'], crun['name'], - crun['status'], crun['conclusion']) - - return failed_checks - -''' -rerquest a check-run -''' -def trigger_build(failed_checks: list, repo: str, token: str): - reruns = [] - for crun in failed_checks: - if crun['name'].strip() in job_black_list: - print(f'Skip black listed job {crun["name"]}') - continue - - r = requests.get(crun['url'], headers={'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'python3', - 'Authorization': f'Bearer {token}'} - ) - if r.status_code == 200: - # url example: https://github.com/qzhuyan/emqx/actions/runs/4469557961/jobs/7852858687 - run_id = r.json()['details_url'].split('/')[-3] - reruns.append(run_id) - else: - print(f'failed to fetch check run {crun["name"]}') - - # remove duplicates - for run_id in set(reruns): - url = f'https://api.github.com/repos/{repo}/actions/runs/{run_id}/rerun-failed-jobs' - - r = requests.post(url, headers={'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'python3', - 'Authorization': f'Bearer {token}'} - ) - if r.status_code == 201: - print(f'Successfully triggered build for {crun["name"]}') - - else: - # Only complain but not exit. - print( - f'Failed to trigger rerun for {run_id}, {crun["name"]}: {r.status_code} : {r.text}') - - -def main(): - parser = OptionParser() - parser.add_option("-r", "--repo", dest="repo", - help="github repo", default="emqx/emqx") - parser.add_option("-t", "--token", dest="gh_token", - help="github API token") - parser.add_option("-b", "--branch", dest="branch", default='master', - help="Branch that workflow runs on") - (options, args) = parser.parse_args() - - # Get gh token from env var GITHUB_TOKEN if provided, else use the one from command line - token = os.environ['GITHUB_TOKEN'] if 'GITHUB_TOKEN' in os.environ else options.gh_token - - target_commit = fetch_latest_commit(token, options.repo, options.branch) - - failed_checks = fetch_check_runs(token, options.repo, target_commit['sha']) - - trigger_build(failed_checks, options.repo, token) - - -if __name__ == '__main__': - main() diff --git a/scripts/test/check-example-configs.sh b/scripts/test/check-example-configs.sh index f71fb15eb..cffea24ce 100755 --- a/scripts/test/check-example-configs.sh +++ b/scripts/test/check-example-configs.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -PROJ_DIR="$(git rev-parse --show-toplevel)" + +cd -P -- "$(dirname -- "$0")/../.." PROFILE="${PROFILE:-emqx}" DIR_NAME='examples' @@ -11,7 +12,7 @@ if [ "${PROFILE}" = 'emqx-enterprise' ]; then SCHEMA_MOD='emqx_enterprise_schema' fi -IFS=$'\n' read -r -d '' -a FILES < <(find "${PROJ_DIR}/rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') +IFS=$'\n' read -r -d '' -a FILES < <(find "rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') prepare_erl_libs() { local libs_dir="$1" @@ -30,7 +31,7 @@ prepare_erl_libs() { } # This is needed when checking schema -export EMQX_ETC_DIR="${PROJ_DIR}/apps/emqx/etc" +export EMQX_ETC_DIR="apps/emqx/etc" prepare_erl_libs "_build/$PROFILE/lib" diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 2c99c641d..e7dbff89a 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -72,12 +72,6 @@ while [ "$#" -gt 0 ]; do esac done -if [ "$TAG_PREFIX" = 'v' ]; then - SRC_DIRS="{apps}" -else - SRC_DIRS="{apps,lib-ee}" -fi - ## make sure we build here in bash and always pass --skip-build to escript if [ "${SKIP_BUILD:-}" != 'yes' ]; then make "${PROFILE}" @@ -114,7 +108,7 @@ PREV_REL_DIR="${PREV_DIR_BASE}/${PREV_TAG}/_build/${PROFILE}/lib" # this in turn makes quoting "${ESCRIPT_ARGS[@]}" problematic, hence disable SC2068 check here # shellcheck disable=SC2068 ./scripts/update_appup.escript \ - --src-dirs "${SRC_DIRS}/**" \ + --src-dirs "apps/**" \ --release-dir "_build/${PROFILE}/lib" \ --prev-release-dir "${PREV_REL_DIR}" \ --skip-build \ diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index f18659d2b..945a948b0 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -1,6 +1,7 @@ #!/usr/bin/env -S escript -c %% -*- erlang-indent-level:4 -*- +%% erlfmt-ignore usage() -> "A script that fills in boilerplate for appup files. @@ -32,54 +33,55 @@ Options: --make-command A command used to assemble the release --prev-release-dir Previous version's release dir (if already built/extracted) --release-dir Release directory - --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' + --src-dirs Directories where source code is found. Defaults to '{src,apps}/**/' ". --record(app, - { modules :: #{module() => binary()} - , version :: string() - }). +-record(app, { + modules :: #{module() => binary()}, + version :: string() +}). default_options() -> - #{ clone_url => find_upstream_repo("origin") - , make_command => "make emqx-rel" - , beams_dir => "_build/emqx/rel/emqx/lib/" - , check => false - , prev_tag => undefined - , src_dirs => "{src,apps,lib-*}/**/" - , prev_beams_dir => undefined - }. + #{ + clone_url => find_upstream_repo("origin"), + make_command => "make emqx-rel", + beams_dir => "_build/emqx/rel/emqx/lib/", + check => false, + prev_tag => undefined, + src_dirs => "{src,apps}/**/", + prev_beams_dir => undefined + }. %% App-specific actions that should be added unconditionally to any update/downgrade: app_specific_actions(_) -> []. ignored_apps() -> - [gpb %% only a build tool - ] ++ otp_standard_apps(). + %% only a build tool + [gpb] ++ otp_standard_apps(). main(Args) -> #{prev_tag := Baseline} = Options = parse_args(Args, default_options()), init_globals(Options), main(Options, Baseline). -parse_args([PrevTag = [A|_]], State) when A =/= $- -> +parse_args([PrevTag = [A | _]], State) when A =/= $- -> State#{prev_tag => PrevTag}; -parse_args(["--check"|Rest], State) -> +parse_args(["--check" | Rest], State) -> parse_args(Rest, State#{check => true}); -parse_args(["--skip-build"|Rest], State) -> +parse_args(["--skip-build" | Rest], State) -> parse_args(Rest, State#{make_command => undefined}); -parse_args(["--repo", Repo|Rest], State) -> +parse_args(["--repo", Repo | Rest], State) -> parse_args(Rest, State#{clone_url => Repo}); -parse_args(["--remote", Remote|Rest], State) -> +parse_args(["--remote", Remote | Rest], State) -> parse_args(Rest, State#{clone_url => find_upstream_repo(Remote)}); -parse_args(["--make-command", Command|Rest], State) -> +parse_args(["--make-command", Command | Rest], State) -> parse_args(Rest, State#{make_command => Command}); -parse_args(["--release-dir", Dir|Rest], State) -> +parse_args(["--release-dir", Dir | Rest], State) -> parse_args(Rest, State#{beams_dir => Dir}); -parse_args(["--prev-release-dir", Dir|Rest], State) -> +parse_args(["--prev-release-dir", Dir | Rest], State) -> parse_args(Rest, State#{prev_beams_dir => Dir}); -parse_args(["--src-dirs", Pattern|Rest], State) -> +parse_args(["--src-dirs", Pattern | Rest], State) -> parse_args(Rest, State#{src_dirs => Pattern}); parse_args(_, _) -> fail(usage()). @@ -87,9 +89,11 @@ parse_args(_, _) -> main(Options, Baseline) -> {CurrRelDir, PrevRelDir} = prepare(Baseline, Options), putopt(prev_beams_dir, PrevRelDir), - log("~n===================================~n" + log( + "~n===================================~n" "Processing changes..." - "~n===================================~n"), + "~n===================================~n" + ), CurrAppsIdx = index_apps(CurrRelDir), PrevAppsIdx = index_apps(PrevRelDir), %% log("Curr: ~p~nPrev: ~p~n", [CurrAppsIdx, PrevAppsIdx]), @@ -98,6 +102,7 @@ main(Options, Baseline) -> ok = check_appup_files(), ok = warn_and_exit(is_valid()). +%% erlfmt-ignore warn_and_exit(true) -> log(" NOTE: Please review the changes manually. This script does not know about NIF @@ -109,9 +114,12 @@ warn_and_exit(false) -> halt(1). prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> - log("~n===================================~n" + log( + "~n===================================~n" "Baseline: ~s" - "~n===================================~n", [Baseline]), + "~n===================================~n", + [Baseline] + ), log("Building the current version...~n"), ok = bash(MakeCommand), PrevRelDir = @@ -126,6 +134,7 @@ prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir} end, {BeamDir, PrevRelDir}. +%% erlfmt-ignore build_prev_release(Baseline, #{clone_url := Repo, make_command := MakeCommand}) -> BaseDir = "/tmp/emqx-appup-base/", Dir = filename:basename(Repo, ".git") ++ [$-|Baseline], @@ -146,24 +155,27 @@ find_upstream_repo(Remote) -> find_appup_actions(CurrApps, PrevApps) -> maps:fold( - fun(App, CurrAppIdx, Acc) -> - case PrevApps of - #{App := PrevAppIdx} -> - find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc; - _ -> - %% New app, nothing to upgrade here. - Acc - end - end, - [], - CurrApps). + fun(App, CurrAppIdx, Acc) -> + case PrevApps of + #{App := PrevAppIdx} -> + find_appup_actions(App, CurrAppIdx, PrevAppIdx) ++ Acc; + _ -> + %% New app, nothing to upgrade here. + Acc + end + end, + [], + CurrApps + ). find_appup_actions(_App, AppIdx, AppIdx) -> %% No changes to the app, ignore: []; -find_appup_actions(App, - CurrAppIdx = #app{version = CurrVersion}, - PrevAppIdx = #app{version = PrevVersion}) -> +find_appup_actions( + App, + CurrAppIdx = #app{version = CurrVersion}, + PrevAppIdx = #app{version = PrevVersion} +) -> {OldUpgrade0, OldDowngrade0} = find_base_appup_actions(App, PrevVersion), OldUpgrade = ensure_all_patch_versions(App, CurrVersion, OldUpgrade0), OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), @@ -195,7 +207,10 @@ do_ensure_all_patch_versions(App, CurrVsn, OldActions) -> {ok, ExpectedVsns} -> CoveredVsns = [V || {V, _} <- OldActions, V =/= <<".*">>], ExpectedVsnStrs = [vsn_number_to_string(V) || V <- ExpectedVsns], - MissingActions = [{V, []} || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns)], + MissingActions = [ + {V, []} + || V <- ExpectedVsnStrs, not contains_version(V, CoveredVsns) + ], MissingActions ++ OldActions; {error, bad_version} -> log("WARN: Could not infer expected versions to upgrade from for ~p~n", [App]), @@ -206,23 +221,24 @@ do_ensure_all_patch_versions(App, CurrVsn, OldActions) -> %% in their current appup. diff_appup_instructions(ComputedChanges, PresentChanges) -> lists:foldr( - fun({VsnOrRegex, ComputedActions}, Acc) -> - case find_matching_version(VsnOrRegex, PresentChanges) of - undefined -> - [{VsnOrRegex, ComputedActions} | Acc]; - PresentActions -> - DiffActions = ComputedActions -- PresentActions, - case DiffActions of - [] -> - %% no diff - Acc; - _ -> - [{VsnOrRegex, DiffActions} | Acc] - end - end - end, - [], - ComputedChanges). + fun({VsnOrRegex, ComputedActions}, Acc) -> + case find_matching_version(VsnOrRegex, PresentChanges) of + undefined -> + [{VsnOrRegex, ComputedActions} | Acc]; + PresentActions -> + DiffActions = ComputedActions -- PresentActions, + case DiffActions of + [] -> + %% no diff + Acc; + _ -> + [{VsnOrRegex, DiffActions} | Acc] + end + end + end, + [], + ComputedChanges + ). %% checks if any missing diffs are present %% and groups them by `up' and `down' types. @@ -234,9 +250,10 @@ parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) -> %% no diff for external dependency; ignore ok; _ -> - Diffs = #{ up => DiffUp - , down => DiffDown - }, + Diffs = #{ + up => DiffUp, + down => DiffDown + }, {diffs, Diffs} end. @@ -260,18 +277,21 @@ find_base_appup_actions(App, PrevVersion) -> {ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}. merge_update_actions(App, Changes, Vsns, PrevVersion) -> - lists:map(fun(Ret = {<<".*">>, _}) -> - Ret; - ({Vsn, Actions}) -> - case is_skipped_version(App, Vsn, PrevVersion) of - true -> - log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), - {Vsn, Actions}; - false -> - {Vsn, do_merge_update_actions(App, Changes, Actions)} - end - end, - Vsns). + lists:map( + fun + (Ret = {<<".*">>, _}) -> + Ret; + ({Vsn, Actions}) -> + case is_skipped_version(App, Vsn, PrevVersion) of + true -> + log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), + {Vsn, Actions}; + false -> + {Vsn, do_merge_update_actions(App, Changes, Actions)} + end + end, + Vsns + ). %% say current version is 1.1.3, and the compare base is version 1.1.1, %% but there is a 1.1.2 in appup we may skip merging instructions for @@ -306,7 +326,7 @@ do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) -> []; false -> [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed] ++ - [{add_module, M} || M <- New] + [{add_module, M} || M <- New] end, {OldActionsWithStop, OldActionsAfterStop} = find_application_stop_instruction(App, OldActions), @@ -331,11 +351,14 @@ contains_restart_application(Application, Actions) -> find_application_stop_instruction(Application, Actions) -> {Before, After0} = lists:splitwith( - fun({apply, {application, stop, [App]}}) when App =:= Application -> - false; - (_) -> - true - end, Actions), + fun + ({apply, {application, stop, [App]}}) when App =:= Application -> + false; + (_) -> + true + end, + Actions + ), case After0 of [StopInst | After] -> {Before ++ [StopInst], After}; @@ -353,8 +376,10 @@ process_old_action({delete_module, Module}) -> [Module]; process_old_action({update, Module, _Change}) -> [Module]; -process_old_action(LoadModule) when is_tuple(LoadModule) andalso - element(1, LoadModule) =:= load_module -> +process_old_action(LoadModule) when + is_tuple(LoadModule) andalso + element(1, LoadModule) =:= load_module +-> element(2, LoadModule); process_old_action(_) -> []. @@ -370,17 +395,19 @@ ensure_version(Version, OldInstructions) -> contains_version(Needle, Haystack) when is_list(Needle) -> lists:any( - fun(Regex) when is_binary(Regex) -> - case re:run(Needle, Regex) of - {match, _} -> - true; - nomatch -> - false - end; - (Vsn) -> - Vsn =:= Needle - end, - Haystack). + fun + (Regex) when is_binary(Regex) -> + case re:run(Needle, Regex) of + {match, _} -> + true; + nomatch -> + false + end; + (Vsn) -> + Vsn =:= Needle + end, + Haystack + ). %% As a best effort approach, we assume that we only bump patch %% version numbers between release upgrades for our dependencies and @@ -413,9 +440,9 @@ vsn_number_to_string({Major, Minor, Patch}) -> read_appup(File) -> %% NOTE: appup file is a script, it may contain variables or functions. - case do_read_appup(File) of - {ok, {U, D}} -> {U, D}; - {error, Reason} -> fail("Failed to parse appup file ~p~n~p", [File, Reason]) + case do_read_appup(File) of + {ok, {U, D}} -> {U, D}; + {error, Reason} -> fail("Failed to parse appup file ~p~n~p", [File, Reason]) end. do_read_appup(File) -> @@ -434,10 +461,11 @@ check_appup_files() -> update_appups(Changes) -> lists:foreach( - fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> - do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) - end, - Changes). + fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) -> + do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) + end, + Changes + ). do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> case locate_current_src(App, ".appup.src") of @@ -469,8 +497,11 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> ok; {diffs, Diffs} -> set_invalid(), - log("ERROR: Appup file for '~p' is not complete.~n" - "Missing:~100p~n", [App, Diffs]), + log( + "ERROR: Appup file for '~p' is not complete.~n" + "Missing:~100p~n", + [App, Diffs] + ), notok end. @@ -496,9 +527,12 @@ render_appup(App, File, Up, Down) -> end. do_render_appup(File, Up, Down) -> - IOList = io_lib:format("%% -*- mode: erlang -*-~n" - "%% Unless you know what you are doing, DO NOT edit manually!!~n" - "{VSN,~n ~p,~n ~p}.~n", [Up, Down]), + IOList = io_lib:format( + "%% -*- mode: erlang -*-~n" + "%% Unless you know what you are doing, DO NOT edit manually!!~n" + "{VSN,~n ~p,~n ~p}.~n", + [Up, Down] + ), ok = file:write_file(File, IOList). create_stub(App) -> @@ -544,30 +578,37 @@ index_app(AppFile) -> %% Note: assuming that beams are always located in the same directory where app file is: EbinDir = filename:dirname(AppFile), Modules = hashsums(EbinDir), - {App, #app{ version = Vsn - , modules = Modules - }}. + {App, #app{ + version = Vsn, + modules = Modules + }}. -diff_app(UpOrDown, App, - #app{version = NewVersion, modules = NewModules}, - #app{version = OldVersion, modules = OldModules}) -> +diff_app( + UpOrDown, + App, + #app{version = NewVersion, modules = NewModules}, + #app{version = OldVersion, modules = OldModules} +) -> {New, Changed} = - maps:fold( fun(Mod, MD5, {New, Changed}) -> - case OldModules of - #{Mod := OldMD5} when MD5 =:= OldMD5 -> - {New, Changed}; - #{Mod := _} -> - {New, [Mod | Changed]}; - _ -> - {[Mod | New], Changed} - end - end - , {[], []} - , NewModules - ), + maps:fold( + fun(Mod, MD5, {New, Changed}) -> + case OldModules of + #{Mod := OldMD5} when MD5 =:= OldMD5 -> + {New, Changed}; + #{Mod := _} -> + {New, [Mod | Changed]}; + _ -> + {[Mod | New], Changed} + end + end, + {[], []}, + NewModules + ), Deleted = maps:keys(maps:without(maps:keys(NewModules), OldModules)), - Changes = lists:filter(fun({_T, L}) -> length(L) > 0 end, - [{added, New}, {changed, Changed}, {deleted, Deleted}]), + Changes = lists:filter( + fun({_T, L}) -> length(L) > 0 end, + [{added, New}, {changed, Changed}, {deleted, Deleted}] + ), case NewVersion =:= OldVersion of true when Changes =:= [] -> %% no change @@ -577,13 +618,17 @@ diff_app(UpOrDown, App, case UpOrDown =:= up of true -> %% only log for the upgrade case because it would be the same result - log("ERROR: Application '~p' contains changes, but its version is not updated. ~s", - [App, format_changes(Changes)]); + log( + "ERROR: Application '~p' contains changes, but its version is not updated. ~s", + [App, format_changes(Changes)] + ); false -> ok end; false -> - log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [App, OldVersion, UpOrDown, NewVersion]), + log("INFO: Application '~p' has been updated: ~p --[~p]--> ~p~n", [ + App, OldVersion, UpOrDown, NewVersion + ]), log("INFO: changes [~p]: ~p~n", [UpOrDown, Changes]), ok end, @@ -594,14 +639,16 @@ format_changes(Changes) -> -spec hashsums(file:filename()) -> #{module() => binary()}. hashsums(EbinDir) -> - maps:from_list(lists:map( - fun(Beam) -> - File = filename:join(EbinDir, Beam), - {ok, Ret = {_Module, _MD5}} = beam_lib:md5(File), - Ret - end, - filelib:wildcard("*.beam", EbinDir) - )). + maps:from_list( + lists:map( + fun(Beam) -> + File = filename:join(EbinDir, Beam), + {ok, Ret = {_Module, _MD5}} = beam_lib:md5(File), + Ret + end, + filelib:wildcard("*.beam", EbinDir) + ) + ). is_app_external(App) -> Ext = ".app.src", @@ -674,12 +721,13 @@ do_locate(Dir, App, Suffix) -> end. find_app(Pattern) -> - lists:filter(fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end, - filelib:wildcard(Pattern)). + lists:filter( + fun(D) -> re:run(D, "apps/.*/_build") =:= nomatch end, + filelib:wildcard(Pattern) + ). bash(undefined) -> ok; -bash(Script) -> - bash(Script, []). +bash(Script) -> bash(Script, []). bash(Script, Env) -> log("+ ~s~n+ Env: ~p~n", [Script, Env]), @@ -695,12 +743,14 @@ cmd(Exec, Params) -> fail("Executable not found in $PATH: ~s", [Exec]); Path -> Params1 = maps:to_list(maps:with([env, args, cd], Params)), - Port = erlang:open_port( {spawn_executable, Path} - , [ exit_status - , nouse_stdio - | Params1 - ] - ), + Port = erlang:open_port( + {spawn_executable, Path}, + [ + exit_status, + nouse_stdio + | Params1 + ] + ), receive {Port, {exit_status, Status}} -> Status