diff --git a/.ci/docker-compose-file/docker-compose-hstreamdb.yaml b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml new file mode 100644 index 000000000..f3c4dbd4c --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml @@ -0,0 +1,123 @@ +version: "3.5" + +services: + hserver: + image: hstreamdb/hstream:v0.15.0 + container_name: hstreamdb + depends_on: + - zookeeper + - hstore + # ports: + # - "127.0.0.1:6570:6570" + expose: + - 6570 + networks: + - emqx_bridge + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /tmp:/tmp + - data_store:/data/store + command: + - bash + - "-c" + - | + set -e + /usr/local/script/wait-for-storage.sh hstore 6440 zookeeper 2181 600 \ + /usr/local/bin/hstream-server \ + --bind-address 0.0.0.0 --port 6570 \ + --internal-port 6571 \ + --server-id 100 \ + --seed-nodes "$$(hostname -I | awk '{print $$1}'):6571" \ + --advertised-address $$(hostname -I | awk '{print $$1}') \ + --metastore-uri zk://zookeeper:2181 \ + --store-config /data/store/logdevice.conf \ + --store-admin-host hstore --store-admin-port 6440 \ + --store-log-level warning \ + --io-tasks-path /tmp/io/tasks \ + --io-tasks-network emqx_bridge + + hstore: + image: hstreamdb/hstream:v0.15.0 + networks: + - emqx_bridge + volumes: + - data_store:/data/store + command: + - bash + - "-c" + - | + set -ex + # N.B. "enable-dscp-reflection=false" is required for linux kernel which + # doesn't support dscp reflection, e.g. centos7. + /usr/local/bin/ld-dev-cluster --root /data/store \ + --use-tcp --tcp-host $$(hostname -I | awk '{print $$1}') \ + --user-admin-port 6440 \ + --param enable-dscp-reflection=false \ + --no-interactive + + zookeeper: + image: zookeeper + expose: + - 2181 + networks: + - emqx_bridge + volumes: + - data_zk_data:/data + - data_zk_datalog:/datalog + + ## The three container `hstream-exporter`, `prometheus`, `console` + ## is for HStreamDB Web Console + ## But HStreamDB Console is not supported in v0.15.0 + ## because of HStreamApi proto changed + # hstream-exporter: + # depends_on: + # hserver: + # condition: service_completed_successfully + # image: hstreamdb/hstream-exporter + # networks: + # - hstream-quickstart + # command: + # - bash + # - "-c" + # - | + # set -ex + # hstream-exporter --addr hstream://hserver:6570 + + # prometheus: + # image: prom/prometheus + # expose: + # - 9097 + # networks: + # - hstream-quickstart + # ports: + # - "9097:9090" + # volumes: + # - $PWD/prometheus:/etc/prometheus + + # console: + # image: hstreamdb/hstream-console + # depends_on: + # - hserver + # expose: + # - 5177 + # networks: + # - hstream-quickstart + # environment: + # - SERVER_PORT=5177 + # - PROMETHEUS_URL=http://prometheus:9097 + # - HSTREAM_PUBLIC_ADDRESS=hstream.example.com + # - HSTREAM_PRIVATE_ADDRESS=hserver:6570 + # ports: + # - "5177:5177" + +# networks: +# hstream-quickstart: +# name: hstream-quickstart + +volumes: + data_store: + name: quickstart_data_store + data_zk_data: + name: quickstart_data_zk_data + data_zk_datalog: + name: quickstart_data_zk_datalog diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index f15e779db..c0c88aef0 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -43,10 +43,12 @@ services: - 19000:19000 # S3 TLS - 19100:19100 - # IOTDB + # IOTDB (3 total) - 14242:4242 - 28080:18080 - 38080:38080 + # HStreamDB + - 15670:5670 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 87878ac92..d5576108f 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -155,5 +155,11 @@ "listen": "0.0.0.0:8085", "upstream": "gcp_emulator:8085", "enabled": true + }, + { + "name": "hstreamdb", + "listen": "0.0.0.0:6570", + "upstream": "hstreamdb:6570", + "enabled": true } ] diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 8615a433a..6b47ceafa 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -33,7 +33,6 @@ runs: HOMEBREW_NO_INSTALL_UPGRADE: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 run: | - brew update brew install curl zip unzip coreutils openssl@1.1 echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 26a3cc5fc..a12aeb012 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,7 +10,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] Added tests for the changes - [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-.en.md` files +- [ ] Change log has been added to `changes/(ce|ee)/(feat|perf|fix)-.en.md` files - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 3b47150c2..3ee9b79c7 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -182,6 +182,7 @@ jobs: images: | ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} flavor: | + latest=${{ matrix.elixir == 'no_elixir' }} suffix=${{ steps.pre-meta.outputs.img_suffix }} tags: | type=semver,pattern={{major}}.{{minor}},value=${{ needs.prepare.outputs.VERSION }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 30de6f3b1..586142bbe 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -101,3 +101,21 @@ jobs: push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm" push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm" push "el/9" "packages/$PROFILE-$VERSION-el9-arm64.rpm" + + rerun-apps-version-check: + runs-on: ubuntu-22.04 + if: github.repository_owner == 'emqx' && github.event_name == 'release' + needs: + - upload + permissions: + pull-requests: read + checks: read + actions: write + steps: + - uses: actions/checkout@v3 + - name: trigger re-run of app versions check on open PRs + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/rerun-apps-version-check.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 272a602e9..6337034eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,6 @@ You are welcome to submit any bugs, issues and feature requests on this repository. - ## Commit Message Guidelines We have very precise rules over how our git commit messages can be formatted. This leads to **more readable messages** that are easy to follow when looking through the **project history**. @@ -80,3 +79,14 @@ Just as in the **subject**, use the imperative, present tense: "change" not "cha The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**. **Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this. + +## Changelog + +Changes affecting EMQX functionality shall be described in a separate markdown file under `changes` directory. + +File name pattern: `changes/(ce|ee)/(feat|perf|fix)-.en.md`, where: + +- `ce,ee`: Indicates whether given change affects community and enterprise edition (`ce`), or enterprise edition only (`ee`); for any change only one file is needed as enterprise edition absorbs all changes from the community edition automatically. When in doubts, one could consult [documentation](https://www.emqx.io/docs/en/latest/). Enterprise features have a corresponding "Tip" banner, see for example [here](https://www.emqx.io/docs/en/v5.1/data-integration/data-bridge-influxdb.html). +- `feat|perf|fix`: Whether the change is a new functionality (`feat`), performance improvement (`perf`), or a bug fix (`fix`). +- `PR-id`: Github pull request id. Since pull request id cannot be known before the PR is actually created, it's common to add change log entry in a separate commit. +- `en`: ISO 639-1 language code indicating the language the change log entry is written in. Right now we are only accepting entries in English. diff --git a/README.md b/README.md index dfec5d71a..87d611c5e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -EMQX is the world's most scalable open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency. +EMQX is the world's most scalable open-source [MQTT broker](https://www.emqx.com/en/blog/the-ultimate-guide-to-mqtt-broker-comparison) with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency. EMQX supports multiple open standard protocols like MQTT, HTTP, QUIC, and WebSocket. It’s 100% compliant with MQTT 5.0 and 3.x standard, and secures bi-directional communication with MQTT over TLS/SSL and various authentication mechanisms. diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index 693bc91b5..e840d2b4a 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -18,3 +18,17 @@ -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization"). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY, <<"authorization">>). + +-define(DEFAULT_ACTION_QOS, 0). +-define(DEFAULT_ACTION_RETAIN, false). + +-define(AUTHZ_SUBSCRIBE(QOS), #{action_type => subscribe, qos => QOS}). +-define(AUTHZ_SUBSCRIBE, ?AUTHZ_SUBSCRIBE(?DEFAULT_ACTION_QOS)). + +-define(AUTHZ_PUBLISH(QOS, RETAIN), #{action_type => publish, qos => QOS, retain => RETAIN}). +-define(AUTHZ_PUBLISH(QOS), ?AUTHZ_PUBLISH(QOS, ?DEFAULT_ACTION_RETAIN)). +-define(AUTHZ_PUBLISH, ?AUTHZ_PUBLISH(?DEFAULT_ACTION_QOS)). + +-define(authz_action(PUBSUB, QOS), #{action_type := PUBSUB, qos := QOS}). +-define(authz_action(PUBSUB), ?authz_action(PUBSUB, _)). +-define(authz_action, ?authz_action(_)). diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index d5da3fb18..7b2ce6c6b 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -21,7 +21,7 @@ -define(PH(Type), <<"${", Type/binary, "}">>). -%% action: publish/subscribe/all +%% action: publish/subscribe -define(PH_ACTION, <<"${action}">>). %% cert @@ -79,6 +79,7 @@ -define(PH_REASON, <<"${reason}">>). -define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>). +-define(PH_RETAIN, <<"${retain}">>). %% sync change these place holder with binary def. -define(PH_S_ACTION, "${action}"). @@ -113,5 +114,6 @@ -define(PH_S_NODE, "${node}"). -define(PH_S_REASON, "${reason}"). -define(PH_S_ENDPOINT_NAME, "${endpoint_name}"). +-define(PH_S_RETAIN, "${retain}"). -endif. diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index c83444efc..5446975d3 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.1.0"). +-define(EMQX_RELEASE_CE, "5.1.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.1.0"). +-define(EMQX_RELEASE_EE, "5.1.1-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 0278a1b1d..78a8b76e0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.11"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 007c0e72a..928539f46 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.1"}, + {vsn, "5.1.2"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index efe9bee37..43669bf6c 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -77,10 +77,10 @@ authenticate(Credential) -> %% @doc Check Authorization -spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) -> allow | deny. -authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) -> +authorize(ClientInfo, Action, <<"$delayed/", Data/binary>> = RawTopic) -> case binary:split(Data, <<"/">>) of [_, Topic] -> - authorize(ClientInfo, PubSub, Topic); + authorize(ClientInfo, Action, Topic); _ -> ?SLOG(warning, #{ msg => "invalid_delayed_topic_format", @@ -90,39 +90,39 @@ authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) -> inc_authz_metrics(deny), deny end; -authorize(ClientInfo, PubSub, Topic) -> +authorize(ClientInfo, Action, Topic) -> Result = case emqx_authz_cache:is_enabled() of - true -> check_authorization_cache(ClientInfo, PubSub, Topic); - false -> do_authorize(ClientInfo, PubSub, Topic) + true -> check_authorization_cache(ClientInfo, Action, Topic); + false -> do_authorize(ClientInfo, Action, Topic) end, inc_authz_metrics(Result), Result. -check_authorization_cache(ClientInfo, PubSub, Topic) -> - case emqx_authz_cache:get_authz_cache(PubSub, Topic) of +check_authorization_cache(ClientInfo, Action, Topic) -> + case emqx_authz_cache:get_authz_cache(Action, Topic) of not_found -> - AuthzResult = do_authorize(ClientInfo, PubSub, Topic), - emqx_authz_cache:put_authz_cache(PubSub, Topic, AuthzResult), + AuthzResult = do_authorize(ClientInfo, Action, Topic), + emqx_authz_cache:put_authz_cache(Action, Topic, AuthzResult), AuthzResult; AuthzResult -> emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, AuthzResult, cache] + [ClientInfo, Action, Topic, AuthzResult, cache] ), inc_authz_metrics(cache_hit), AuthzResult end. -do_authorize(ClientInfo, PubSub, Topic) -> +do_authorize(ClientInfo, Action, Topic) -> NoMatch = emqx:get_config([authorization, no_match], allow), Default = #{result => NoMatch, from => default}, - case run_hooks('client.authorize', [ClientInfo, PubSub, Topic], Default) of + case run_hooks('client.authorize', [ClientInfo, Action, Topic], Default) of AuthzResult = #{result := Result} when Result == allow; Result == deny -> From = maps:get(from, AuthzResult, unknown), emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, Result, From] + [ClientInfo, Action, Topic, Result, From] ), Result; Other -> @@ -133,7 +133,7 @@ do_authorize(ClientInfo, PubSub, Topic) -> }), emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, deny, unknown_return_format] + [ClientInfo, Action, Topic, deny, unknown_return_format] ), deny end. diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index ffb4e3d1e..cb72986e7 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -26,8 +26,8 @@ get_release/0, set_config_loader/1, get_config_loader/0, - set_init_tnx_id/1, - get_init_tnx_id/0 + unset_config_loaded/0, + init_load_done/0 ]). -include("logger.hrl"). @@ -56,24 +56,22 @@ prep_stop(_State) -> stop(_State) -> ok. +-define(CONFIG_LOADER, config_loader). +-define(DEFAULT_LOADER, emqx). %% @doc Call this function to make emqx boot without loading config, %% in case we want to delegate the config load to a higher level app %% which manages emqx app. set_config_loader(Module) when is_atom(Module) -> - application:set_env(emqx, config_loader, Module). + application:set_env(emqx, ?CONFIG_LOADER, Module). get_config_loader() -> - application:get_env(emqx, config_loader, emqx). + application:get_env(emqx, ?CONFIG_LOADER, ?DEFAULT_LOADER). -%% @doc Set the transaction id from which this node should start applying after boot. -%% The transaction ID is received from the core node which we just copied the latest -%% config from. -set_init_tnx_id(TnxId) -> - application:set_env(emqx, cluster_rpc_init_tnx_id, TnxId). +unset_config_loaded() -> + application:unset_env(emqx, ?CONFIG_LOADER). -%% @doc Get the transaction id from which this node should start applying after boot. -get_init_tnx_id() -> - application:get_env(emqx, cluster_rpc_init_tnx_id, -1). +init_load_done() -> + get_config_loader() =/= ?DEFAULT_LOADER. maybe_load_config() -> case get_config_loader() of diff --git a/apps/emqx/src/emqx_authz_cache.erl b/apps/emqx/src/emqx_authz_cache.erl index 6555266a5..af19ecf8f 100644 --- a/apps/emqx/src/emqx_authz_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -16,7 +16,7 @@ -module(emqx_authz_cache). --include("emqx.hrl"). +-include("emqx_access_control.hrl"). -export([ list_authz_cache/0, @@ -159,8 +159,7 @@ dump_authz_cache() -> map_authz_cache(Fun) -> [ Fun(R) - || R = {{SubPub, _T}, _Authz} <- erlang:get(), - SubPub =:= publish orelse SubPub =:= subscribe + || R = {{?authz_action, _T}, _Authz} <- erlang:get() ]. foreach_authz_cache(Fun) -> _ = map_authz_cache(Fun), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 93bb6535e..5e594d35f 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -20,6 +20,7 @@ -include("emqx.hrl"). -include("emqx_channel.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_access_control.hrl"). -include("logger.hrl"). -include("types.hrl"). @@ -491,7 +492,7 @@ handle_in( ok -> TopicFilters0 = parse_topic_filters(TopicFilters), TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0), - TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel), + TupleTopicFilters0 = check_sub_authzs(SubPkt, TopicFilters1, Channel), HasAuthzDeny = lists:any( fun({_TopicFilter, ReasonCode}) -> ReasonCode =:= ?RC_NOT_AUTHORIZED @@ -1838,14 +1839,34 @@ check_pub_alias( check_pub_alias(_Packet, _Channel) -> ok. +%%-------------------------------------------------------------------- +%% Authorization action + +authz_action(#mqtt_packet{ + header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{} +}) -> + ?AUTHZ_PUBLISH(QoS, Retain); +authz_action(#mqtt_packet{ + header = #mqtt_packet_header{qos = QoS}, variable = #mqtt_packet_subscribe{} +}) -> + ?AUTHZ_SUBSCRIBE(QoS); +%% Will message +authz_action(#message{qos = QoS, flags = #{retain := Retain}}) -> + ?AUTHZ_PUBLISH(QoS, Retain); +authz_action(#message{qos = QoS}) -> + ?AUTHZ_PUBLISH(QoS). + %%-------------------------------------------------------------------- %% Check Pub Authorization check_pub_authz( - #mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}}, + #mqtt_packet{ + variable = #mqtt_packet_publish{topic_name = Topic} + } = Packet, #channel{clientinfo = ClientInfo} ) -> - case emqx_access_control:authorize(ClientInfo, publish, Topic) of + Action = authz_action(Packet), + case emqx_access_control:authorize(ClientInfo, Action, Topic) of allow -> ok; deny -> {error, ?RC_NOT_AUTHORIZED} end. @@ -1868,24 +1889,23 @@ check_pub_caps( %%-------------------------------------------------------------------- %% Check Sub Authorization -%% TODO: not only check topic filter. Qos chould be checked too. -%% Not implemented yet: -%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7] -check_sub_authzs(TopicFilters, Channel) -> - check_sub_authzs(TopicFilters, Channel, []). +check_sub_authzs(Packet, TopicFilters, Channel) -> + Action = authz_action(Packet), + check_sub_authzs(Action, TopicFilters, Channel, []). check_sub_authzs( + Action, [TopicFilter = {Topic, _} | More], Channel = #channel{clientinfo = ClientInfo}, Acc ) -> - case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of + case emqx_access_control:authorize(ClientInfo, Action, Topic) of allow -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); + check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); deny -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) end; -check_sub_authzs([], _Channel, Acc) -> +check_sub_authzs(_Action, [], _Channel, Acc) -> lists:reverse(Acc). %%-------------------------------------------------------------------- @@ -2149,7 +2169,8 @@ publish_will_msg( ClientInfo = #{mountpoint := MountPoint}, Msg = #message{topic = Topic} ) -> - PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow, + Action = authz_action(Msg), + PublishingDisallowed = emqx_access_control:authorize(ClientInfo, Action, Topic) =/= allow, ClientBanned = emqx_banned:check(ClientInfo), case PublishingDisallowed orelse ClientBanned of true -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index aaee3b64e..b1bb29159 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -96,7 +96,9 @@ format_list(Listener) -> do_list_raw() -> %% GET /listeners from other nodes returns [] when init config is not loaded. - case emqx_app:get_config_loader() =/= emqx of + %% FIXME This is a workaround for the issue: + %% mria:running_nodes() sometime return node which not ready to accept rpc call. + case emqx_app:init_load_done() of true -> Key = <<"listeners">>, Raw = emqx_config:get_raw([Key], #{}), @@ -368,7 +370,7 @@ console_print(_Fmt, _Args) -> ok. %% Start MQTT/TCP listener -spec do_start_listener(atom(), atom(), map()) -> {ok, pid() | {skipped, atom()}} | {error, term()}. -do_start_listener(_Type, _ListenerName, #{enabled := false}) -> +do_start_listener(_Type, _ListenerName, #{enable := false}) -> {ok, {skipped, listener_disabled}}; do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == tcp; Type == ssl @@ -499,8 +501,8 @@ post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) -> remove_listener(Type, Name, OldConf); post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) -> - #{enabled := NewEnabled} = NewConf, - #{enabled := OldEnabled} = OldConf, + #{enable := NewEnabled} = NewConf, + #{enable := OldEnabled} = OldConf, case {NewEnabled, OldEnabled} of {true, true} -> ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), @@ -810,7 +812,7 @@ has_enabled_listener_conf_by_type(Type) -> lists:any( fun({Id, LConf}) when is_map(LConf) -> {ok, #{type := Type0}} = parse_listener_id(Id), - Type =:= Type0 andalso maps:get(enabled, LConf, true) + Type =:= Type0 andalso maps:get(enable, LConf, true) end, list() ). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8509dc245..ce1af66af 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -30,6 +30,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("logger.hrl"). +-define(MAX_INT_MQTT_PACKET_SIZE, 268435456). -define(MAX_INT_TIMEOUT_MS, 4294967295). %% floor(?MAX_INT_TIMEOUT_MS / 1000). -define(MAX_INT_TIMEOUT_S, 4294967). @@ -45,6 +46,7 @@ -type timeout_duration_s() :: 0..?MAX_INT_TIMEOUT_S. -type timeout_duration_ms() :: 0..?MAX_INT_TIMEOUT_MS. -type bytesize() :: integer(). +-type mqtt_max_packet_size() :: 1..?MAX_INT_MQTT_PACKET_SIZE. -type wordsize() :: bytesize(). -type percent() :: float(). -type file() :: string(). @@ -71,6 +73,7 @@ -typerefl_from_string({timeout_duration_s/0, emqx_schema, to_timeout_duration_s}). -typerefl_from_string({timeout_duration_ms/0, emqx_schema, to_timeout_duration_ms}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({mqtt_max_packet_size/0, emqx_schema, to_bytesize}). -typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}). -typerefl_from_string({percent/0, emqx_schema, to_percent}). -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). @@ -151,6 +154,7 @@ timeout_duration_s/0, timeout_duration_ms/0, bytesize/0, + mqtt_max_packet_size/0, wordsize/0, percent/0, file/0, @@ -1746,13 +1750,12 @@ mqtt_listener(Bind) -> base_listener(Bind) -> [ - {"enabled", + {"enable", sc( boolean(), #{ default => true, - %% TODO(5.2): change field name to 'enable' and keep 'enabled' as an alias - aliases => [enable], + aliases => [enabled], desc => ?DESC(fields_listener_enabled) } )}, @@ -3357,7 +3360,7 @@ mqtt_general() -> )}, {"max_packet_size", sc( - bytesize(), + mqtt_max_packet_size(), #{ default => <<"1MB">>, desc => ?DESC(mqtt_max_packet_size) diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index c6a567318..cc937f81c 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -29,6 +29,7 @@ -export_type([ zone/0, pubsub/0, + pubsub_action/0, subid/0 ]). @@ -127,7 +128,12 @@ | exactly_once. -type zone() :: atom(). --type pubsub() :: publish | subscribe. +-type pubsub_action() :: publish | subscribe. + +-type pubsub() :: + #{action_type := subscribe, qos := qos()} + | #{action_type := publish, qos := qos(), retain := boolean()}. + -type subid() :: binary() | atom(). -type group() :: binary() | undefined. diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 305eaf5eb..5d4344de6 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -19,8 +19,8 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -44,8 +44,7 @@ t_authenticate(_) -> ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). t_authorize(_) -> - Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), - ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)). + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, <<"t">>)). t_delayed_authorize(_) -> RawTopic = <<"$delayed/1/foo/2">>, @@ -54,11 +53,11 @@ t_delayed_authorize(_) -> ok = emqx_hooks:put('client.authorize', {?MODULE, authz_stub, [Topic]}, ?HP_AUTHZ), - Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>), - ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)), + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, RawTopic)), - Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>), - ?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)), + ?assertEqual( + deny, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, InvalidTopic) + ), ok. t_quick_deny_anonymous(_) -> @@ -96,8 +95,8 @@ t_quick_deny_anonymous(_) -> %% Helper functions %%-------------------------------------------------------------------- -authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}}; -authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}. +authz_stub(_Client, _Action, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}}; +authz_stub(_Client, _Action, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}. quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) -> {stop, {error, not_authorized}}; diff --git a/apps/emqx/test/emqx_authz_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl index 5497422af..09d1e1522 100644 --- a/apps/emqx/test/emqx_authz_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -43,8 +43,6 @@ t_clean_authz_cache(_) -> ct:sleep(100), ClientPid = case emqx_cm:lookup_channels(<<"emqx_c">>) of - [Pid] when is_pid(Pid) -> - Pid; Pids when is_list(Pids) -> lists:last(Pids); _ -> diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 5653cd2d2..f266dbcfa 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -908,7 +908,8 @@ t_check_pub_alias(_) -> t_check_sub_authzs(_) -> emqx_config:put_zone_conf(default, [authorization, enable], true), TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, - [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()). + Subscribe = ?SUBSCRIBE_PACKET(1, [TopicFilter]), + [{TopicFilter, 0}] = emqx_channel:check_sub_authzs(Subscribe, [TopicFilter], channel()). t_enrich_connack_caps(_) -> ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]), diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index b004b139e..7f1fe4628 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -618,7 +618,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256" ], - enabled => true, + enable => true, idle_timeout => 15000, ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index caae62f4a..5e8bd4103 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -206,11 +206,6 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> base_port := BasePort, work_dir := WorkDir } = Spec, - Listeners = [ - #{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}} - || Type <- [tcp, ssl, ws, wss], - Port <- [listener_port(BasePort, Type)] - ], Cluster = case get_cluster_seeds(Spec) of [_ | _] = Seeds -> @@ -239,7 +234,7 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> tcp_server_port => gen_rpc_port(BasePort), port_discovery => manual }, - listeners => lists:foldl(fun maps:merge/2, #{}, Listeners) + listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec) } }; default_appspec(_App, _, _) -> @@ -252,6 +247,13 @@ get_cluster_seeds(#{join_to := Node}) -> get_cluster_seeds(#{core_nodes := CoreNodes}) -> CoreNodes. +allocate_listener_port(Type, #{base_port := BasePort}) -> + Port = listener_port(BasePort, Type), + #{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}}. + +allocate_listener_ports(Types, Spec) -> + lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]). + start_node_init(Spec = #{name := Node}) -> Node = start_bare_node(Node, Spec), pong = net_adm:ping(Node), diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index aef0fc5e5..1ae6ceded 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -305,7 +305,7 @@ default_appspec(emqx_conf, SuiteOpts) -> #{ config => SharedConfig, % NOTE - % We inform `emqx` of our config loader before starting `emqx_conf` sothat it won't + % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't % overwrite everything with a default configuration. before_start => fun() -> emqx_app:set_config_loader(?MODULE) diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index fa0713cf0..b8d0c39f6 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -229,7 +229,7 @@ t_ssl_password_cert(Config) -> keyfile => filename:join(DataDir, "server-password.key") }, LConf = #{ - enabled => true, + enable => true, bind => {{127, 0, 0, 1}, Port}, mountpoint => <<>>, zone => default, diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 95cf29bee..6d1ced486 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -20,6 +20,7 @@ -include_lib("proper/include/proper.hrl"). -include("emqx.hrl"). +-include("emqx_access_control.hrl"). %% High level Types -export([ @@ -34,7 +35,8 @@ subopts/0, nodename/0, normal_topic/0, - normal_topic_filter/0 + normal_topic_filter/0, + pubsub/0 ]). %% Basic Types @@ -482,6 +484,23 @@ normal_topic_filter() -> end ). +subscribe_action() -> + ?LET( + Qos, + qos(), + ?AUTHZ_SUBSCRIBE(Qos) + ). + +publish_action() -> + ?LET( + {Qos, Retain}, + {qos(), boolean()}, + ?AUTHZ_PUBLISH(Qos, Retain) + ). + +pubsub() -> + oneof([publish_action(), subscribe_action()]). + %%-------------------------------------------------------------------- %% Basic Types %%-------------------------------------------------------------------- diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf index 9964dc7ba..dbeec6852 100644 --- a/apps/emqx_authz/etc/acl.conf +++ b/apps/emqx_authz/etc/acl.conf @@ -20,7 +20,7 @@ %% %% -type(permission() :: allow | deny). %% -%% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}). +%% -type(rule() :: {permission(), who(), action(), topics()} | {permission(), all}). %%-------------------------------------------------------------------- {allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index b43a2cdab..5cab24fab 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -18,16 +18,6 @@ -define(APP, emqx_authz). --define(ALLOW_DENY(A), - ((A =:= allow) orelse (A =:= <<"allow">>) orelse - (A =:= deny) orelse (A =:= <<"deny">>)) -). --define(PUBSUB(A), - ((A =:= subscribe) orelse (A =:= <<"subscribe">>) orelse - (A =:= publish) orelse (A =:= <<"publish">>) orelse - (A =:= all) orelse (A =:= <<"all">>)) -). - %% authz_mnesia -define(ACL_TABLE, emqx_acl). @@ -59,12 +49,12 @@ username => user1, rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -72,6 +62,20 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/topic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/topic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). @@ -79,12 +83,12 @@ clientid => client1, rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -92,18 +96,32 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/topic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/topic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). -define(ALL_RULES_EXAMPLE, #{ rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -111,9 +129,28 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/topic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/topic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). + +-define(USERNAME_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?USERNAME_RULES_EXAMPLE))). +-define(CLIENTID_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?CLIENTID_RULES_EXAMPLE))). +-define(ALL_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?ALL_RULES_EXAMPLE))). + -define(META_EXAMPLE, #{ page => 1, limit => 100, @@ -121,3 +158,8 @@ }). -define(RESOURCE_GROUP, <<"emqx_authz">>). + +-define(AUTHZ_FEATURES, [rich_actions]). + +-define(DEFAULT_RULE_QOS, [0, 1, 2]). +-define(DEFAULT_RULE_RETAIN, all). diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index e9654557e..29a035ab3 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 6f45a88b7..0419bcf72 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -39,6 +39,11 @@ get_enabled_authzs/0 ]). +-export([ + feature_available/1, + set_feature_available/2 +]). + -export([post_config_update/5, pre_config_update/3]). -export([acl_conf_file/0]). @@ -519,6 +524,20 @@ read_acl_file(#{<<"path">> := Path} = Source) -> {ok, Rules} = emqx_authz_file:read_file(Path), maps:remove(<<"path">>, Source#{<<"rules">> => Rules}). +%%------------------------------------------------------------------------------ +%% Extended Features +%%------------------------------------------------------------------------------ + +-define(DEFAULT_RICH_ACTIONS, true). + +-define(FEATURE_KEY(_NAME_), {?MODULE, _NAME_}). + +feature_available(rich_actions) -> + persistent_term:get(?FEATURE_KEY(rich_actions), ?DEFAULT_RICH_ACTIONS). + +set_feature_available(Feature, Enable) when is_boolean(Enable) -> + persistent_term:put(?FEATURE_KEY(Feature), Enable). + %%------------------------------------------------------------------------------ %% Internal function %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index f5ac40f5e..a2a8f2525 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -359,6 +359,22 @@ fields(rule_item) -> required => true, example => publish } + )}, + {qos, + mk( + array(emqx_schema:qos()), + #{ + desc => ?DESC(qos), + default => ?DEFAULT_RULE_QOS + } + )}, + {retain, + mk( + hoconsc:union([all, boolean()]), + #{ + desc => ?DESC(retain), + default => ?DEFAULT_RULE_RETAIN + } )} ]; fields(clientid) -> @@ -434,7 +450,7 @@ users(post, #{body := Body}) when is_list(Body) -> [] -> lists:foreach( fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> - emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)) + emqx_authz_mnesia:store_rules({username, Username}, Rules) end, Body ), @@ -470,7 +486,7 @@ clients(post, #{body := Body}) when is_list(Body) -> [] -> lists:foreach( fun(#{<<"clientid">> := ClientID, <<"rules">> := Rules}) -> - emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules)) + emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules) end, Body ), @@ -489,21 +505,14 @@ user(get, #{bindings := #{username := Username}}) -> {ok, Rules} -> {200, #{ username => Username, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; user(put, #{ bindings := #{username := Username}, body := #{<<"username">> := Username, <<"rules">> := Rules} }) -> - emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)), + emqx_authz_mnesia:store_rules({username, Username}, Rules), {204}; user(delete, #{bindings := #{username := Username}}) -> case emqx_authz_mnesia:get_rules({username, Username}) of @@ -521,21 +530,14 @@ client(get, #{bindings := #{clientid := ClientID}}) -> {ok, Rules} -> {200, #{ clientid => ClientID, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; client(put, #{ bindings := #{clientid := ClientID}, body := #{<<"clientid">> := ClientID, <<"rules">> := Rules} }) -> - emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules)), + emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules), {204}; client(delete, #{bindings := #{clientid := ClientID}}) -> case emqx_authz_mnesia:get_rules({clientid, ClientID}) of @@ -552,18 +554,11 @@ all(get, _) -> {200, #{rules => []}}; {ok, Rules} -> {200, #{ - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; all(post, #{body := #{<<"rules">> := Rules}}) -> - emqx_authz_mnesia:store_rules(all, format_rules(Rules)), + emqx_authz_mnesia:store_rules(all, Rules), {204}; all(delete, _) -> emqx_authz_mnesia:store_rules(all, []), @@ -626,58 +621,20 @@ run_fuzzy_filter( %%-------------------------------------------------------------------- %% format funcs -%% format rule from api -format_rules(Rules) when is_list(Rules) -> - lists:foldl( - fun( - #{ - <<"topic">> := Topic, - <<"action">> := Action, - <<"permission">> := Permission - }, - AccIn - ) when - ?PUBSUB(Action) andalso - ?ALLOW_DENY(Permission) - -> - AccIn ++ [{atom(Permission), atom(Action), Topic}] - end, - [], - Rules - ). - %% format result from mnesia tab format_result([{username, Username}, {rules, Rules}]) -> #{ username => Username, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }; format_result([{clientid, ClientID}, {rules, Rules}]) -> #{ clientid => ClientID, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }. -atom(B) when is_binary(B) -> - try - binary_to_existing_atom(B, utf8) - catch - _Error:_Expection -> binary_to_atom(B) - end; -atom(A) when is_atom(A) -> A. + +format_rules(Rules) -> + [emqx_authz_rule_raw:format_rule(Rule) || Rule <- Rules]. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 317395a45..7d421d39b 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -16,7 +16,6 @@ -module(emqx_authz_file). --include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(emqx_authz). diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 5747e6eeb..aafbe25ad 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -51,6 +51,11 @@ ?PH_CERT_CN_NAME ]). +-define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ + ?PH_QOS, + ?PH_RETAIN +]). + description() -> "AuthZ with http". @@ -72,7 +77,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ type := http, @@ -81,7 +86,7 @@ authorize( request_timeout := RequestTimeout } = Config ) -> - Request = generate_request(PubSub, Topic, Client, Config), + Request = generate_request(Action, Topic, Client, Config), case emqx_resource:simple_sync_query(ResourceID, {Method, Request, RequestTimeout}) of {ok, 204, _Headers} -> {matched, allow}; @@ -139,14 +144,14 @@ parse_config( method => Method, base_url => BaseUrl, headers => Headers, - base_path_templete => emqx_authz_utils:parse_str(Path, ?PLACEHOLDERS), + base_path_templete => emqx_authz_utils:parse_str(Path, placeholders()), base_query_template => emqx_authz_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)), - ?PLACEHOLDERS + placeholders() ), body_template => emqx_authz_utils:parse_deep( maps:to_list(maps:get(body, Conf, #{})), - ?PLACEHOLDERS + placeholders() ), request_timeout => ReqTimeout, %% pool_type default value `random` @@ -173,7 +178,7 @@ parse_url(Url) -> end. generate_request( - PubSub, + Action, Topic, Client, #{ @@ -184,7 +189,7 @@ generate_request( body_template := BodyTemplate } ) -> - Values = client_vars(Client, PubSub, Topic), + Values = client_vars(Client, Action, Topic), Path = emqx_authz_utils:render_urlencoded_str(BasePathTemplate, Values), Query = emqx_authz_utils:render_deep(BaseQueryTemplate, Values), Body = emqx_authz_utils:render_deep(BodyTemplate, Values), @@ -227,11 +232,9 @@ serialize_body(<<"application/json">>, Body) -> serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). -client_vars(Client, PubSub, Topic) -> - Client#{ - action => PubSub, - topic => Topic - }. +client_vars(Client, Action, Topic) -> + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + Vars#{topic => Topic}. to_list(A) when is_atom(A) -> atom_to_list(A); @@ -243,3 +246,11 @@ to_list(L) when is_list(L) -> to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L); to_bin(X) -> X. + +placeholders() -> + placeholders(emqx_authz:feature_available(rich_actions)). + +placeholders(true) -> + ?PLACEHOLDERS ++ ?PLACEHOLDERS_FOR_RICH_ACTIONS; +placeholders(false) -> + ?PLACEHOLDERS. diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index bdb4877c0..2cecd0c71 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -16,7 +16,6 @@ -module(emqx_authz_mnesia). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -202,25 +201,16 @@ record_count() -> %%-------------------------------------------------------------------- normalize_rules(Rules) -> - lists:map(fun normalize_rule/1, Rules). + lists:flatmap(fun normalize_rule/1, Rules). -normalize_rule({Permission, Action, Topic}) -> - {normalize_permission(Permission), normalize_action(Action), normalize_topic(Topic)}; -normalize_rule(Rule) -> - error({invalid_rule, Rule}). - -normalize_topic(Topic) when is_list(Topic) -> list_to_binary(Topic); -normalize_topic(Topic) when is_binary(Topic) -> Topic; -normalize_topic(Topic) -> error({invalid_rule_topic, Topic}). - -normalize_action(publish) -> publish; -normalize_action(subscribe) -> subscribe; -normalize_action(all) -> all; -normalize_action(Action) -> error({invalid_rule_action, Action}). - -normalize_permission(allow) -> allow; -normalize_permission(deny) -> deny; -normalize_permission(Permission) -> error({invalid_rule_permission, Permission}). +normalize_rule(RuleRaw) -> + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + %% For backward compatibility + {ok, {Permission, Action, TopicFilters}} -> + [{Permission, Action, TopicFilter} || TopicFilter <- TopicFilters]; + {error, Reason} -> + error(Reason) + end. do_get_rules(Key) -> case mnesia:dirty_read(?ACL_TABLE, Key) of diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index e82ff64e1..52a920d3a 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -68,7 +68,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ collection := Collection, @@ -77,14 +77,7 @@ authorize( } ) -> RenderedFilter = emqx_authz_utils:render_deep(FilterTemplate, Client), - Result = - try - emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}}) - catch - error:Error -> {error, Error} - end, - - case Result of + case emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}}) of {error, Reason} -> ?SLOG(error, #{ msg => "query_mongo_error", @@ -94,18 +87,22 @@ authorize( resource_id => ResourceID }), nomatch; - {ok, []} -> - nomatch; {ok, Rows} -> - Rules = [ - emqx_authz_rule:compile({Permission, all, Action, Topics}) - || #{ - <<"topics">> := Topics, - <<"permission">> := Permission, - <<"action">> := Action - } <- Rows - ], - do_authorize(Client, PubSub, Topic, Rules) + Rules = lists:flatmap(fun parse_rule/1, Rows), + do_authorize(Client, Action, Topic, Rules) + end. + +parse_rule(Row) -> + case emqx_authz_rule_raw:parse_rule(Row) of + {ok, {Permission, Action, Topics}} -> + [emqx_authz_rule:compile({Permission, all, Action, Topics})]; + {error, Reason} -> + ?SLOG(error, #{ + msg => "parse_rule_error", + reason => Reason, + row => Row + }), + [] end. do_authorize(_Client, _PubSub, _Topic, []) -> diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index fb6a29c3d..a724e451c 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -55,7 +55,7 @@ create(#{query := SQL} = Source0) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), - Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}. + Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), @@ -64,7 +64,7 @@ update(#{query := SQL} = Source0) -> {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> - Source#{annotations => #{id => Id, tmpl_oken => TmplToken}} + Source#{annotations => #{id => Id, tmpl_token => TmplToken}} end. destroy(#{annotations := #{id := Id}}) -> @@ -72,57 +72,49 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ annotations := #{ id := ResourceID, - tmpl_oken := TmplToken + tmpl_token := TmplToken } } ) -> - RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Vars), case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ?PREPARE_KEY, RenderParams}) of - {ok, _Columns, []} -> - nomatch; - {ok, Columns, Rows} -> - do_authorize(Client, PubSub, Topic, Columns, Rows); + {ok, ColumnNames, Rows} -> + do_authorize(Client, Action, Topic, ColumnNames, Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_mysql_error", reason => Reason, - tmpl_oken => TmplToken, + tmpl_token => TmplToken, params => RenderParams, resource_id => ResourceID }), nomatch end. -do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; -do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> - case +do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> + try emqx_authz_rule:match( - Client, - PubSub, - Topic, - emqx_authz_rule:compile(format_result(Columns, Row)) + Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) + catch + error:Reason -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule => Row + }), + do_authorize(Client, Action, Topic, ColumnNames, Tail) end. - -format_result(Columns, Row) -> - Permission = lists:nth(index(<<"permission">>, Columns), Row), - Action = lists:nth(index(<<"action">>, Columns), Row), - Topic = lists:nth(index(<<"topic">>, Columns), Row), - {Permission, all, Action, [Topic]}. - -index(Elem, List) -> - index(Elem, List, 1). -index(_Elem, [], _Index) -> {error, not_found}; -index(Elem, [Elem | _List], Index) -> Index; -index(Elem, [_ | List], Index) -> index(Elem, List, Index + 1). diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index 05f2315a6..f0bdf77be 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -21,6 +21,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("epgsql/include/epgsql.hrl"). + -behaviour(emqx_authz). %% AuthZ Callbacks @@ -77,7 +79,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ annotations := #{ @@ -86,14 +88,13 @@ authorize( } } ) -> - RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Vars), case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ResourceID, RenderedParams}) of - {ok, _Columns, []} -> - nomatch; {ok, Columns, Rows} -> - do_authorize(Client, PubSub, Topic, Columns, Rows); + do_authorize(Client, Action, Topic, column_names(Columns), Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_postgresql_error", @@ -104,33 +105,29 @@ authorize( nomatch end. -do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; -do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> - case +do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> + try emqx_authz_rule:match( - Client, - PubSub, - Topic, - emqx_authz_rule:compile(format_result(Columns, Row)) + Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) + catch + error:Reason:Stack -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule => Row, + stack => Stack + }), + do_authorize(Client, Action, Topic, ColumnNames, Tail) end. -format_result(Columns, Row) -> - Permission = lists:nth(index(<<"permission">>, 2, Columns), erlang:tuple_to_list(Row)), - Action = lists:nth(index(<<"action">>, 2, Columns), erlang:tuple_to_list(Row)), - Topic = lists:nth(index(<<"topic">>, 2, Columns), erlang:tuple_to_list(Row)), - {Permission, all, Action, [Topic]}. - -index(Key, N, TupleList) when is_integer(N) -> - Tuple = lists:keyfind(Key, N, TupleList), - index(Tuple, TupleList, 1); -index(_Tuple, [], _Index) -> - {error, not_found}; -index(Tuple, [Tuple | _TupleList], Index) -> - Index; -index(Tuple, [_ | TupleList], Index) -> - index(Tuple, TupleList, Index + 1). +column_names(Columns) -> + lists:map( + fun(#column{name = Name}) -> Name end, + Columns + ). diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 3b19db832..d163c0d16 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -70,19 +70,18 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ cmd_template := CmdTemplate, annotations := #{id := ResourceID} } ) -> - Cmd = emqx_authz_utils:render_deep(CmdTemplate, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + Cmd = emqx_authz_utils:render_deep(CmdTemplate, Vars), case emqx_resource:simple_sync_query(ResourceID, {cmd, Cmd}) of - {ok, []} -> - nomatch; {ok, Rows} -> - do_authorize(Client, PubSub, Topic, Rows); + do_authorize(Client, Action, Topic, Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_redis_error", @@ -93,21 +92,63 @@ authorize( nomatch end. -do_authorize(_Client, _PubSub, _Topic, []) -> +do_authorize(_Client, _Action, _Topic, []) -> nomatch; -do_authorize(Client, PubSub, Topic, [TopicFilter, Action | Tail]) -> - case +do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) -> + try emqx_authz_rule:match( Client, - PubSub, + Action, Topic, - emqx_authz_rule:compile({allow, all, Action, [TopicFilter]}) + compile_rule(RuleEncoded, TopicFilterRaw) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Tail) + nomatch -> do_authorize(Client, Action, Topic, Tail) + catch + error:Reason:Stack -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule_encoded => RuleEncoded, + topic_filter_raw => TopicFilterRaw, + stacktrace => Stack + }), + do_authorize(Client, Action, Topic, Tail) + end. + +compile_rule(RuleBin, TopicFilterRaw) -> + RuleRaw = + maps:merge( + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => TopicFilterRaw + }, + parse_rule(RuleBin) + ), + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + {ok, {Permission, Action, Topics}} -> + emqx_authz_rule:compile({Permission, all, Action, Topics}); + {error, Reason} -> + error(Reason) end. tokens(Query) -> Tokens = binary:split(Query, <<" ">>, [global]), [Token || Token <- Tokens, size(Token) > 0]. + +parse_rule(<<"publish">>) -> + #{<<"action">> => <<"publish">>}; +parse_rule(<<"subscribe">>) -> + #{<<"action">> => <<"subscribe">>}; +parse_rule(<<"all">>) -> + #{<<"action">> => <<"all">>}; +parse_rule(Bin) when is_binary(Bin) -> + case emqx_utils_json:safe_decode(Bin, [return_maps]) of + {ok, Map} when is_map(Map) -> + maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map); + {ok, _} -> + error({invalid_topic_rule, Bin, notamap}); + {error, Error} -> + error({invalid_topic_rule, Bin, Error}) + end. diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index ec1a8c5de..6e13cac91 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -16,9 +16,9 @@ -module(emqx_authz_rule). --include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include("emqx_authz.hrl"). -ifdef(TEST). -compile(export_all). @@ -32,50 +32,123 @@ compile/1 ]). --type ipaddress() :: - {ipaddr, esockd_cidr:cidr_string()} - | {ipaddrs, list(esockd_cidr:cidr_string())}. +-type permission() :: allow | deny. --type username() :: {username, binary()}. - --type clientid() :: {clientid, binary()}. - --type who() :: +-type who_condition() :: ipaddress() | username() | clientid() | {'and', [ipaddress() | username() | clientid()]} | {'or', [ipaddress() | username() | clientid()]} | all. +-type ipaddress() :: + {ipaddr, esockd_cidr:cidr_string()} + | {ipaddrs, list(esockd_cidr:cidr_string())}. +-type username() :: {username, binary()}. +-type clientid() :: {clientid, binary()}. --type action() :: subscribe | publish | all. --type permission() :: allow | deny. +-type action_condition() :: + subscribe + | publish + | #{action_type := subscribe, qos := qos_condition()} + | #{action_type := publish | all, qos := qos_condition(), retain := retain_condition()} + | all. +-type qos_condition() :: [qos()]. +-type retain_condition() :: retain() | all. --type rule() :: {permission(), who(), action(), list(emqx_types:topic())}. +-type topic_condition() :: list(emqx_types:topic() | {eq, emqx_types:topic()}). + +-type rule() :: {permission(), who_condition(), action_condition(), topic_condition()}. + +-type qos() :: emqx_types:qos(). +-type retain() :: boolean(). +-type action() :: + #{action_type := subscribe, qos := qos()} + | #{action_type := publish, qos := qos(), retain := retain()}. -export_type([ - action/0, - permission/0 + permission/0, + who_condition/0, + action_condition/0, + topic_condition/0 ]). +-define(IS_PERMISSION(Permission), (Permission =:= allow orelse Permission =:= deny)). + compile({Permission, all}) when - ?ALLOW_DENY(Permission) + ?IS_PERMISSION(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; compile({Permission, Who, Action, TopicFilters}) when - ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(TopicFilters) + ?IS_PERMISSION(Permission) andalso is_list(TopicFilters) -> - {atom(Permission), compile_who(Who), atom(Action), [ + {Permission, compile_who(Who), compile_action(Action), [ compile_topic(Topic) || Topic <- TopicFilters ]}; -compile({Permission, _Who, _Action, _TopicFilter}) when not ?ALLOW_DENY(Permission) -> +compile({Permission, _Who, _Action, _TopicFilter}) when not ?IS_PERMISSION(Permission) -> throw({invalid_authorization_permission, Permission}); -compile({_Permission, _Who, Action, _TopicFilter}) when not ?PUBSUB(Action) -> - throw({invalid_authorization_action, Action}); compile(BadRule) -> throw({invalid_authorization_rule, BadRule}). +compile_action(Action) -> + compile_action(emqx_authz:feature_available(rich_actions), Action). + +-define(IS_ACTION_WITH_RETAIN(Action), (Action =:= publish orelse Action =:= all)). + +compile_action(_RichActionsOn, subscribe) -> + subscribe; +compile_action(_RichActionsOn, Action) when ?IS_ACTION_WITH_RETAIN(Action) -> + Action; +compile_action(true = _RichActionsOn, {subscribe, Opts}) when is_list(Opts) -> + #{ + action_type => subscribe, + qos => qos_from_opts(Opts) + }; +compile_action(true = _RichActionsOn, {Action, Opts}) when + ?IS_ACTION_WITH_RETAIN(Action) andalso is_list(Opts) +-> + #{ + action_type => Action, + qos => qos_from_opts(Opts), + retain => retain_from_opts(Opts) + }; +compile_action(_RichActionsOn, Action) -> + throw({invalid_authorization_action, Action}). + +qos_from_opts(Opts) -> + try + case proplists:get_all_values(qos, Opts) of + [] -> + ?DEFAULT_RULE_QOS; + QoSs -> + lists:flatmap( + fun + (QoS) when is_integer(QoS) -> + [validate_qos(QoS)]; + (QoS) when is_list(QoS) -> + lists:map(fun validate_qos/1, QoS) + end, + QoSs + ) + end + catch + bad_qos -> + throw({invalid_authorization_qos, Opts}) + end. + +validate_qos(QoS) when is_integer(QoS), QoS >= 0, QoS =< 2 -> + QoS; +validate_qos(_) -> + throw(bad_qos). + +retain_from_opts(Opts) -> + case proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN) of + all -> all; + Retain when is_boolean(Retain) -> Retain; + _ -> throw({invalid_authorization_retain, Opts}) + end. + compile_who(all) -> all; compile_who({user, Username}) -> @@ -99,8 +172,12 @@ compile_who({ipaddrs, CIDRs}) -> compile_who({'and', L}) when is_list(L) -> {'and', [compile_who(Who) || Who <- L]}; compile_who({'or', L}) when is_list(L) -> - {'or', [compile_who(Who) || Who <- L]}. + {'or', [compile_who(Who) || Who <- L]}; +compile_who(Who) -> + throw({invalid_who, Who}). +compile_topic("eq " ++ Topic) -> + {eq, emqx_topic:words(bin(Topic))}; compile_topic(<<"eq ", Topic/binary>>) -> {eq, emqx_topic:words(Topic)}; compile_topic({eq, Topic}) -> @@ -117,45 +194,65 @@ compile_topic(Topic) -> Tokens -> {pattern, Tokens} end. -atom(B) when is_binary(B) -> - try - binary_to_existing_atom(B, utf8) - catch - _E:_S -> binary_to_atom(B) - end; -atom(A) when is_atom(A) -> A. - bin(L) when is_list(L) -> list_to_binary(L); bin(B) when is_binary(B) -> B. --spec matches(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), [rule()]) -> +-spec matches(emqx_types:clientinfo(), action(), emqx_types:topic(), [rule()]) -> {matched, allow} | {matched, deny} | nomatch. -matches(_Client, _PubSub, _Topic, []) -> +matches(_Client, _Action, _Topic, []) -> nomatch; -matches(Client, PubSub, Topic, [{Permission, Who, Action, TopicFilters} | Tail]) -> - case match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) of - nomatch -> matches(Client, PubSub, Topic, Tail); +matches(Client, Action, Topic, [{Permission, WhoCond, ActionCond, TopicCond} | Tail]) -> + case match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) of + nomatch -> matches(Client, Action, Topic, Tail); Matched -> Matched end. --spec match(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), rule()) -> +-spec match(emqx_types:clientinfo(), action(), emqx_types:topic(), rule()) -> {matched, allow} | {matched, deny} | nomatch. -match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) -> +match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) -> case - match_action(PubSub, Action) andalso - match_who(Client, Who) andalso - match_topics(Client, Topic, TopicFilters) + match_action(Action, ActionCond) andalso + match_who(Client, WhoCond) andalso + match_topics(Client, Topic, TopicCond) of true -> {matched, Permission}; _ -> nomatch end. -match_action(publish, publish) -> true; -match_action(subscribe, subscribe) -> true; -match_action(_, all) -> true; -match_action(_, _) -> false. +-spec match_action(action(), action_condition()) -> boolean(). +match_action(#{action_type := publish}, PubSubCond) when is_atom(PubSubCond) -> + match_pubsub(publish, PubSubCond); +match_action( + #{action_type := publish, qos := QoS, retain := Retain}, #{ + action_type := publish, qos := QoSCond, retain := RetainCond + } +) -> + match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond); +match_action(#{action_type := publish, qos := QoS, retain := Retain}, #{ + action_type := all, qos := QoSCond, retain := RetainCond +}) -> + match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond); +match_action(#{action_type := subscribe}, PubSubCond) when is_atom(PubSubCond) -> + match_pubsub(subscribe, PubSubCond); +match_action(#{action_type := subscribe, qos := QoS}, #{action_type := subscribe, qos := QoSCond}) -> + match_qos(QoS, QoSCond); +match_action(#{action_type := subscribe, qos := QoS}, #{action_type := all, qos := QoSCond}) -> + match_qos(QoS, QoSCond); +match_action(_, _) -> + false. + +match_pubsub(publish, publish) -> true; +match_pubsub(subscribe, subscribe) -> true; +match_pubsub(_, all) -> true; +match_pubsub(_, _) -> false. + +match_qos(QoS, QoSs) -> lists:member(QoS, QoSs). + +match_retain(_, all) -> true; +match_retain(Retain, Retain) when is_boolean(Retain) -> true; +match_retain(_, _) -> false. match_who(_, all) -> true; diff --git a/apps/emqx_authz/src/emqx_authz_rule_raw.erl b/apps/emqx_authz/src/emqx_authz_rule_raw.erl new file mode 100644 index 000000000..1fbe2ca45 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_rule_raw.erl @@ -0,0 +1,197 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc +%% This module converts authz rule fields obtained from +%% external sources like database or API to the format +%% accepted by emqx_authz_rule module. +%%-------------------------------------------------------------------- + +-module(emqx_authz_rule_raw). + +-export([parse_rule/1, format_rule/1]). + +-include("emqx_authz.hrl"). + +-type rule_raw() :: #{binary() => binary() | [binary()]}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec parse_rule(rule_raw()) -> + {ok, { + emqx_authz_rule:permission(), + emqx_authz_rule:action_condition(), + emqx_authz_rule:topic_condition() + }} + | {error, term()}. +parse_rule( + #{ + <<"permission">> := PermissionRaw, + <<"action">> := ActionTypeRaw + } = RuleRaw +) -> + try + Topics = validate_rule_topics(RuleRaw), + Permission = validate_rule_permission(PermissionRaw), + ActionType = validate_rule_action_type(ActionTypeRaw), + Action = validate_rule_action(ActionType, RuleRaw), + {ok, {Permission, Action, Topics}} + catch + throw:ValidationError -> + {error, ValidationError} + end; +parse_rule(RuleRaw) -> + {error, {invalid_rule, RuleRaw}}. + +-spec format_rule({ + emqx_authz_rule:permission(), + emqx_authz_rule:action_condition(), + emqx_authz_rule:topic_condition() +}) -> map(). +format_rule({Permission, Action, Topics}) when is_list(Topics) -> + maps:merge( + #{ + topic => lists:map(fun format_topic/1, Topics), + permission => Permission + }, + format_action(Action) + ); +format_rule({Permission, Action, Topic}) -> + maps:merge( + #{ + topic => format_topic(Topic), + permission => Permission + }, + format_action(Action) + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +validate_rule_topics(#{<<"topic">> := TopicRaw}) when is_binary(TopicRaw) -> + [validate_rule_topic(TopicRaw)]; +validate_rule_topics(#{<<"topics">> := TopicsRaw}) when is_list(TopicsRaw) -> + lists:map(fun validate_rule_topic/1, TopicsRaw); +validate_rule_topics(RuleRaw) -> + throw({invalid_topics, RuleRaw}). + +validate_rule_topic(<<"eq ", TopicRaw/binary>>) -> + {eq, validate_rule_topic(TopicRaw)}; +validate_rule_topic(TopicRaw) when is_binary(TopicRaw) -> TopicRaw. + +validate_rule_permission(<<"allow">>) -> allow; +validate_rule_permission(<<"deny">>) -> deny; +validate_rule_permission(PermissionRaw) -> throw({invalid_permission, PermissionRaw}). + +validate_rule_action_type(<<"publish">>) -> publish; +validate_rule_action_type(<<"subscribe">>) -> subscribe; +validate_rule_action_type(<<"all">>) -> all; +validate_rule_action_type(ActionRaw) -> throw({invalid_action, ActionRaw}). + +validate_rule_action(ActionType, RuleRaw) -> + validate_rule_action(emqx_authz:feature_available(rich_actions), ActionType, RuleRaw). + +%% rich_actions disabled +validate_rule_action(false, ActionType, _RuleRaw) -> + ActionType; +%% rich_actions enabled +validate_rule_action(true, publish, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)), + {publish, [{qos, Qos}, {retain, Retain}]}; +validate_rule_action(true, subscribe, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + {subscribe, [{qos, Qos}]}; +validate_rule_action(true, all, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)), + {all, [{qos, Qos}, {retain, Retain}]}. + +validate_rule_qos(QosInt) when is_integer(QosInt) andalso QosInt >= 0 andalso QosInt =< 2 -> + [QosInt]; +validate_rule_qos(QosBin) when is_binary(QosBin) -> + try + QosRawList = binary:split(QosBin, <<",">>, [global]), + lists:map(fun validate_rule_qos_atomic/1, QosRawList) + catch + _:_ -> + throw({invalid_qos, QosBin}) + end; +validate_rule_qos(QosList) when is_list(QosList) -> + try + lists:map(fun validate_rule_qos_atomic/1, QosList) + catch + invalid_qos -> + throw({invalid_qos, QosList}) + end; +validate_rule_qos(undefined) -> + ?DEFAULT_RULE_QOS; +validate_rule_qos(null) -> + ?DEFAULT_RULE_QOS; +validate_rule_qos(QosRaw) -> + throw({invalid_qos, QosRaw}). + +validate_rule_qos_atomic(<<"0">>) -> 0; +validate_rule_qos_atomic(<<"1">>) -> 1; +validate_rule_qos_atomic(<<"2">>) -> 2; +validate_rule_qos_atomic(0) -> 0; +validate_rule_qos_atomic(1) -> 1; +validate_rule_qos_atomic(2) -> 2; +validate_rule_qos_atomic(_) -> throw(invalid_qos). + +validate_rule_retain(<<"0">>) -> false; +validate_rule_retain(<<"1">>) -> true; +validate_rule_retain(0) -> false; +validate_rule_retain(1) -> true; +validate_rule_retain(<<"true">>) -> true; +validate_rule_retain(<<"false">>) -> false; +validate_rule_retain(true) -> true; +validate_rule_retain(false) -> false; +validate_rule_retain(undefined) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(null) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(<<"all">>) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(Retain) -> throw({invalid_retain, Retain}). + +format_action(Action) -> + format_action(emqx_authz:feature_available(rich_actions), Action). + +%% rich_actions disabled +format_action(false, Action) when is_atom(Action) -> + #{ + action => Action + }; +format_action(false, {ActionType, _Opts}) -> + #{ + action => ActionType + }; +%% rich_actions enabled +format_action(true, Action) when is_atom(Action) -> + #{ + action => Action + }; +format_action(true, {ActionType, Opts}) -> + #{ + action => ActionType, + qos => proplists:get_value(qos, Opts, ?DEFAULT_RULE_QOS), + retain => proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN) + }. + +format_topic({eq, Topic}) when is_binary(Topic) -> + <<"eq ", Topic/binary>>; +format_topic(Topic) when is_binary(Topic) -> + Topic. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index ec112070e..d903ae027 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -31,7 +31,10 @@ parse_sql/3, render_deep/2, render_str/2, - render_sql_params/2 + render_sql_params/2, + client_vars/1, + vars_for_rule_query/2, + parse_rule_from_row/2 ]). -export([ @@ -43,6 +46,8 @@ start_after_created => false }). +-include_lib("emqx/include/logger.hrl"). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -171,6 +176,24 @@ content_type(Headers) when is_list(Headers) -> <<"application/json">> ). +-define(RAW_RULE_KEYS, [<<"permission">>, <<"action">>, <<"topic">>, <<"qos">>, <<"retain">>]). + +parse_rule_from_row(ColumnNames, Row) -> + RuleRaw = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + {ok, {Permission, Action, Topics}} -> + emqx_authz_rule:compile({Permission, all, Action, Topics}); + {error, Reason} -> + error(Reason) + end. + +vars_for_rule_query(Client, ?authz_action(PubSub, Qos) = Action) -> + Client#{ + action => PubSub, + qos => Qos, + retain => maps:get(retain, Action, false) + }. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -208,3 +231,8 @@ handle_sql_var(_Name, Value) -> bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); bin(X) -> X. + +to_list(Tuple) when is_tuple(Tuple) -> + tuple_to_list(Tuple); +to_list(List) when is_list(List) -> + List. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 3775b9a1c..7f03a38a2 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -96,7 +96,7 @@ t_api(_) -> <<"hasnext">> := false } } = emqx_utils_json:decode(Request1), - ?assertEqual(3, length(Rules1)), + ?assertEqual(?USERNAME_RULES_EXAMPLE_COUNT, length(Rules1)), {ok, 200, Request1_1} = request( @@ -204,7 +204,7 @@ t_api(_) -> } = emqx_utils_json:decode(Request4), #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), - ?assertEqual(3, length(Rules3)), + ?assertEqual(?CLIENTID_RULES_EXAMPLE_COUNT, length(Rules3)), {ok, 204, _} = request( @@ -253,7 +253,7 @@ t_api(_) -> [] ), #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), - ?assertEqual(3, length(Rules5)), + ?assertEqual(?ALL_RULES_EXAMPLE_COUNT, length(Rules5)), {ok, 204, _} = request( diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index ec96522a5..396679783 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -38,34 +38,26 @@ all() -> groups() -> []. -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - init_per_testcase(TestCase, Config) -> Apps = emqx_cth_suite:start( - [{emqx_conf, "authorization.no_match = deny"}, emqx_authz], + [ + {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"}, + emqx_authz + ], #{work_dir => filename:join(?config(priv_dir, Config), TestCase)} ), [{tc_apps, Apps} | Config]. end_per_testcase(_TestCase, Config) -> - emqx_cth_suite:stop(?config(tc_apps, Config)). + emqx_cth_suite:stop(?config(tc_apps, Config)), + _ = emqx_authz:set_feature_available(rich_actions, true). %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ t_ok(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), ok = setup_config(?RAW_SOURCE#{ <<"rules">> => <<"{allow, {user, \"username\"}, publish, [\"t\"]}.">> @@ -73,23 +65,52 @@ t_ok(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) + ). + +t_rich_actions(_Config) -> + ClientInfo = emqx_authz_test_lib:base_client_info(), + + ok = setup_config(?RAW_SOURCE#{ + <<"rules">> => + <<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">> + }), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) + ), + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(0, false), <<"t">>) + ), + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) + ). + +t_no_rich_actions(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + ?assertMatch( + {error, {pre_config_update, emqx_authz, {invalid_authorization_action, _}}}, + emqx_authz:update(?CMD_REPLACE, [ + ?RAW_SOURCE#{ + <<"rules">> => + <<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">> + } + ]) ). t_superuser(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - is_superuser => true, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = + emqx_authz_test_lib:client_info(#{is_superuser => true}), %% no rules apply to superuser ok = setup_config(?RAW_SOURCE#{ @@ -98,12 +119,12 @@ t_superuser(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) ). t_invalid_file(_Config) -> diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 702bf2756..6cf4b5bc0 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -65,6 +65,7 @@ init_per_testcase(_Case, Config) -> Config. end_per_testcase(_Case, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), try ok = emqx_authz_http_test_server:stop() catch @@ -97,7 +98,7 @@ t_response_handling(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, no body @@ -109,7 +110,7 @@ t_response_handling(_Config) -> #{} ), - deny = emqx_access_control:authorize(ClientInfo, publish, <<"t">>), + deny = emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>), %% OK, get, 204 ok = setup_handler_and_config( @@ -122,7 +123,7 @@ t_response_handling(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, 400 @@ -136,7 +137,7 @@ t_response_handling(_Config) -> ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, 400 + body & headers @@ -155,7 +156,7 @@ t_response_handling(_Config) -> ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% the server cannot be reached; should skip to the next @@ -165,7 +166,7 @@ t_response_handling(_Config) -> ?check_trace( ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), fun(Trace) -> ?assertMatch( @@ -200,7 +201,9 @@ t_query_params(_Config) -> proto_name := <<"MQTT">>, mountpoint := <<"MOUNTPOINT">>, topic := <<"t/1">>, - action := <<"publish">> + action := <<"publish">>, + qos := <<"1">>, + retain := <<"false">> } = cowboy_req:match_qs( [ username, @@ -209,7 +212,9 @@ t_query_params(_Config) -> proto_name, mountpoint, topic, - action + action, + qos, + retain ], Req0 ), @@ -224,7 +229,9 @@ t_query_params(_Config) -> "proto_name=${proto_name}&" "mountpoint=${mountpoint}&" "topic=${topic}&" - "action=${action}" + "action=${action}&" + "qos=${qos}&" + "retain=${retain}" >> } ), @@ -241,7 +248,7 @@ t_query_params(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>) ). t_path(_Config) -> @@ -256,7 +263,9 @@ t_path(_Config) -> "MQTT/" "MOUNTPOINT/" "t%2F1/" - "publish" + "publish/" + "1/" + "false" >>, cowboy_req:path(Req0) ), @@ -271,7 +280,9 @@ t_path(_Config) -> "${proto_name}/" "${mountpoint}/" "${topic}/" - "${action}" + "${action}/" + "${qos}/" + "${retain}" >> } ), @@ -288,7 +299,7 @@ t_path(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>) ). t_json_body(_Config) -> @@ -309,7 +320,9 @@ t_json_body(_Config) -> <<"proto_name">> := <<"MQTT">>, <<"mountpoint">> := <<"MOUNTPOINT">>, <<"topic">> := <<"t">>, - <<"action">> := <<"publish">> + <<"action">> := <<"publish">>, + <<"qos">> := <<"1">>, + <<"retain">> := <<"false">> }, emqx_utils_json:decode(RawBody, [return_maps]) ), @@ -324,7 +337,9 @@ t_json_body(_Config) -> <<"proto_name">> => <<"${proto_name}">>, <<"mountpoint">> => <<"${mountpoint}">>, <<"topic">> => <<"${topic}">>, - <<"action">> => <<"${action}">> + <<"action">> => <<"${action}">>, + <<"qos">> => <<"${qos}">>, + <<"retain">> => <<"${retain}">> } } ), @@ -341,7 +356,45 @@ t_json_body(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) + ). + +t_no_rich_actions(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + ok = setup_handler_and_config( + fun(Req0, State) -> + ?assertEqual( + <<"/authz/users/">>, + cowboy_req:path(Req0) + ), + + {ok, RawBody, Req1} = cowboy_req:read_body(Req0), + + %% No interpolation if rich_actions is disabled + ?assertMatch( + #{ + <<"qos">> := <<"${qos}">>, + <<"retain">> := <<"${retain}">> + }, + emqx_utils_json:decode(RawBody, [return_maps]) + ), + {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"qos">> => <<"${qos}">>, + <<"retain">> => <<"${retain}">> + } + } + ), + + ClientInfo = emqx_authz_test_lib:base_client_info(), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) ). t_placeholder_and_body(_Config) -> @@ -401,7 +454,7 @@ t_placeholder_and_body(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). t_no_value_for_placeholder(_Config) -> @@ -441,7 +494,7 @@ t_no_value_for_placeholder(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). t_create_replace(_Config) -> @@ -466,7 +519,7 @@ t_create_replace(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Changing to valid config @@ -485,7 +538,7 @@ t_create_replace(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl index 56e893c5b..fcaa378c5 100644 --- a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authn/include/emqx_authn.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -341,12 +342,12 @@ t_check_undefined_expire(_Config) -> ?assertMatch( {matched, allow}, - emqx_authz_client_info:authorize(Client, subscribe, <<"a/b">>, undefined) + emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/b">>, undefined) ), ?assertMatch( {matched, deny}, - emqx_authz_client_info:authorize(Client, subscribe, <<"a/bar">>, undefined) + emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/bar">>, undefined) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 2b7fce309..c82bbd56e 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -18,6 +18,8 @@ -compile(nowarn_export_all). -compile(export_all). +-include_lib("emqx_authz.hrl"). + -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -44,6 +46,7 @@ init_per_testcase(_TestCase, Config) -> Config. end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), ok = emqx_authz_mnesia:purge_rules(). set_special_configs(emqx_authz) -> @@ -54,51 +57,135 @@ set_special_configs(_) -> %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ -t_username_topic_rules(_Config) -> - ok = test_topic_rules(username). -t_clientid_topic_rules(_Config) -> - ok = test_topic_rules(clientid). +t_authz(_Config) -> + ClientInfo = emqx_authz_test_lib:base_client_info(), -t_all_topic_rules(_Config) -> - ok = test_topic_rules(all). + test_authz( + allow, + allow, + {all, #{ + <<"permission">> => <<"allow">>, <<"action">> => <<"subscribe">>, <<"topic">> => <<"t">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>} + ), + test_authz( + allow, + allow, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ), + test_authz( + allow, + allow, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>} + ), + test_authz( + deny, + deny, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ), + test_authz( + allow, + allow, + {{clientid, <<"clientid">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>} + ), + test_authz( + allow, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(1, true), <<"t">>} + ), + test_authz( + deny, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(0, true), <<"t">>} + ), + test_authz( + deny, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>} + ). -test_topic_rules(Key) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +test_authz(Expected, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic}) -> + test_authz_with_rich_actions(true, Expected, {Who, Rule}, {ClientInfo, Action, Topic}), + test_authz_with_rich_actions( + false, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic} + ). - SetupSamples = fun(CInfo, Samples) -> - setup_client_samples(CInfo, Samples, Key) - end, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, SetupSamples), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, SetupSamples), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, SetupSamples). +test_authz_with_rich_actions( + RichActionsEnabled, Expected, {Who, Rule}, {ClientInfo, Action, Topic} +) -> + ct:pal("Test authz rich_actions:~p~nwho:~p~nrule:~p~nattempt:~p~nexpected ~p", [ + RichActionsEnabled, Who, Rule, {ClientInfo, Action, Topic}, Expected + ]), + try + _ = emqx_authz:set_feature_available(rich_actions, RichActionsEnabled), + ok = emqx_authz_mnesia:store_rules(Who, [Rule]), + ?assertEqual(Expected, emqx_access_control:authorize(ClientInfo, Action, Topic)) + after + ok = emqx_authz_mnesia:purge_rules() + end. t_normalize_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), ok = emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{allow, publish, "t"}] + [#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}] ), ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertException( @@ -106,25 +193,31 @@ t_normalize_rules(_Config) -> {invalid_rule, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [[allow, publish, <<"t">>]] + [[<<"allow">>, <<"publish">>, <<"t">>]] ) ), ?assertException( error, - {invalid_rule_action, _}, + {invalid_action, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{allow, pub, <<"t">>}] + [#{<<"permission">> => <<"allow">>, <<"action">> => <<"pub">>, <<"topic">> => <<"t">>}] ) ), ?assertException( error, - {invalid_rule_permission, _}, + {invalid_permission, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{accept, publish, <<"t">>}] + [ + #{ + <<"permission">> => <<"accept">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">> + } + ] ) ). @@ -138,27 +231,5 @@ raw_mnesia_authz_config() -> <<"type">> => <<"built_in_database">> }. -setup_client_samples(ClientInfo, Samples, Key) -> - ok = emqx_authz_mnesia:purge_rules(), - Rules = lists:flatmap( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:map( - fun(Topic) -> - {binary_to_atom(Permission), binary_to_atom(Action), Topic} - end, - Topics - ) - end, - Samples - ), - #{username := Username, clientid := ClientId} = ClientInfo, - Who = - case Key of - username -> {username, Username}; - clientid -> {clientid, ClientId}; - all -> all - end, - ok = emqx_authz_mnesia:store_rules(Who, Rules). - setup_config() -> emqx_authz_test_lib:setup_config(raw_mnesia_authz_config(), #{}). diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index 9ffeacf45..4476deda2 100644 --- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -28,10 +28,10 @@ -define(MONGO_CLIENT, 'emqx_authz_mongo_SUITE_client'). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -57,12 +57,18 @@ set_special_configs(emqx_authz) -> set_special_configs(_) -> ok. +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> {ok, _} = mc_worker_api:connect(mongo_config()), ok = emqx_authz_test_lib:reset_authorizers(), Config. end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), ok = reset_samples(), ok = mc_worker_api:disconnect(?MONGO_CLIENT). @@ -70,233 +76,313 @@ end_per_testcase(_TestCase, _Config) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_complex_filter(_) -> - %% atom and string values also supported - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - Samples = [ +cases() -> + [ #{ - <<"x">> => #{ - <<"u">> => <<"username">>, - <<"c">> => [#{<<"c">> => <<"clientid">>}], - <<"y">> => 1 - }, - <<"permission">> => <<"allow">>, - <<"action">> => <<"publish">>, - <<"topics">> => [<<"t">>] - } - ], - - ok = setup_samples(Samples), - ok = setup_config( - #{ - <<"filter">> => #{ - <<"x">> => #{ - <<"u">> => <<"${username}">>, - <<"c">> => [#{<<"c">> => <<"${clientid}">>}], - <<"y">> => 1 + name => base_publish, + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"b">>, + <<"permission">> => <<"allow">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"all">>, + <<"topics">> => [<<"c">>, <<"d">>], + <<"permission">> => <<"allow">> } - } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>}, + + {allow, ?AUTHZ_PUBLISH, <<"c">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"c">>}, + {allow, ?AUTHZ_PUBLISH, <<"d">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"d">>} + ] + }, + #{ + name => filter_works, + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_rich_rules, + features => [rich_actions], + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">>, + <<"qos">> => <<"1,2,3">> + }, + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">>, + <<"retain">> => <<"yes">> + } + ], + filter => #{}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_rules, + records => [ + #{ + <<"action">> => <<"publis">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + records => [ + #{ + <<"cn">> => <<"cn">>, + <<"dn">> => <<"dn">>, + <<"clientid">> => <<"clientid">>, + <<"peerhost">> => <<"127.0.0.1">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + filter => #{ + <<"cn">> => <<"${cert_common_name}">>, + <<"dn">> => <<"${cert_subject}">>, + <<"clientid">> => <<"${clientid}">>, + <<"peerhost">> => <<"${peerhost}">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topics">> => [ + <<"t/${username}">>, + <<"t/${clientid}">>, + <<"t1/#">>, + <<"t2/+">>, + <<"eq t3/${username}">> + ] + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>}, + {allow, ?AUTHZ_PUBLISH, <<"t2/a">>}, + {allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>}, + {deny, ?AUTHZ_PUBLISH, <<"t3/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a">>, + <<"qos">> => 1, + <<"retain">> => true + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"b">>, + <<"qos">> => <<"1">>, + <<"retain">> => <<"true">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"c">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => 1 + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"d">>, + <<"qos">> => [1, 2], + <<"retain">> => <<"1">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"e">>, + <<"qos">> => [1, 2], + <<"retain">> => <<"all">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"f">>, + <<"qos">> => null, + <<"retain">> => null + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"a">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"a">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"b">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"c">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"d">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"d">>}, + + {allow, ?AUTHZ_PUBLISH(1, false), <<"e">>}, + {allow, ?AUTHZ_PUBLISH(1, true), <<"e">>}, + {deny, ?AUTHZ_PUBLISH(0, false), <<"e">>}, + + {allow, ?AUTHZ_PUBLISH, <<"f">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"f">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + records => [ + #{ + <<"username">> => <<"username">>, + <<"clientid">> => <<"clientid">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + username => "username", + clientid => clientid + }, + filter => #{<<"username">> => <<"${username}">>, <<"clientid">> => <<"${clientid}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{<<"$in">> => #{<<"a">> => 1}}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => complex_query, + records => [ + #{ + <<"a">> => #{<<"u">> => <<"clientid">>, <<"c">> => [<<"cn">>, <<"dn">>]}, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + filter => #{ + <<"a">> => #{ + <<"u">> => <<"${clientid}">>, + <<"c">> => [<<"${cert_common_name}">>, <<"${cert_subject}">>] + } + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{allow, publish, <<"t">>}] - ). - -t_mongo_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_samples([]), - ok = setup_config( - #{<<"filter">> => #{<<"$badoperator">> => <<"$badoperator">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, publish, <<"t">>}] - ). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ByClientid = #{ - <<"clientid">> => <<"clientid">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByClientid]), - ok = setup_config( - #{<<"filter">> => #{<<"clientid">> => <<"${clientid}">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByPeerhost = #{ - <<"peerhost">> => <<"127.0.0.1">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByPeerhost]), - ok = setup_config( - #{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByCN = #{ - <<"CN">> => <<"cn">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByCN]), - ok = setup_config( - #{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByDN = #{ - <<"DN">> => <<"dn">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByDN]), - ok = setup_config( - #{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_bad_filter(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{<<"filter">> => #{<<"$in">> => #{<<"a">> => 1}}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {deny, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -populate_records(AclRecords, AdditionalData) -> - [maps:merge(Record, AdditionalData) || Record <- AclRecords]. - -setup_samples(AclRecords) -> - ok = reset_samples(), - {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, AclRecords), - ok. - -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - Records = lists:map( - fun(Sample) -> - #{ - topics := Topics, - permission := Permission, - action := Action - } = Sample, - - #{ - <<"topics">> => Topics, - <<"permission">> => Permission, - <<"action">> => Action, - <<"username">> => Username - } - end, - Samples - ), - setup_samples(Records), - setup_config(#{<<"filter">> => #{<<"username">> => <<"${username}">>}}). - reset_samples() -> {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"acl">>, #{}), ok. +setup_source_data(#{records := Records}) -> + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, Records), + ok. + +setup_authz_source(#{filter := Filter}) -> + setup_config( + #{ + <<"filter">> => Filter + } + ). + setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_mongo_authz_config(), diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 06449b3b4..4304dd505 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -27,10 +27,10 @@ -define(MYSQL_RESOURCE, <<"emqx_authz_mysql_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -41,13 +41,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?MYSQL_RESOURCE, - ?RESOURCE_GROUP, - emqx_mysql, - mysql_config(), - #{} - ), + ok = create_mysql_resource(), Config; false -> {skip, no_mysql} @@ -59,9 +53,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok = drop_table(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -72,189 +75,11 @@ set_special_configs(_) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - %% by clientid - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by peerhost - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(peerhost, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE peerhost = ${peerhost}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by cn - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(cn, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE cn = ${cert_common_name}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by dn - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(dn, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE dn = ${cert_subject}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% strip double quote support - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = \"${clientid}\"" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_mysql_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{<<"query">> => <<"SOME INVALID STATEMENT">>} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, subscribe, <<"a">>}] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). t_create_invalid(_Config) -> BadConfig = maps:merge( @@ -265,45 +90,307 @@ t_create_invalid(_Config) -> [_] = emqx_authz:lookup(). -t_nonbinary_values(_Config) -> - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, username, topic, permission, action)" - "VALUES(?, ?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( +cases() -> + [ #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid} AND username = ${username}" - >> - } - ), + name => base_publish, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{username => <<"username">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + "CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255)," + " peerhost VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + "INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)" + " VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic FROM acl WHERE" + " clientid = ${clientid} AND cn = ${cert_common_name}" + " AND dn = ${cert_subject} AND peerhost = ${peerhost}", + client_info => #{ + clientid => <<"clientid">>, + cn => <<"cn">>, + dn => <<"dn">>, + peerhost => {127, 0, 0, 1} + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${clientid}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 'eq t/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/#', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't1/+', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/${username}">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/1/2">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/1">>}, + {deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>}, + {deny, ?AUTHZ_PUBLISH, <<"abc">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_s VARCHAR(255), retain_s VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't1', 'allow', 'publish', '1', 'true')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't2', 'allow', 'publish', '2', 'false')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't4', 'allow', 'subscribe', '1', null)", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)" + ], + query => + "SELECT permission, action, topic, qos_s as qos, retain_s as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}, + + {allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>}, + {deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>} + ] + }, + #{ + name => qos_retain_in_query_result_as_integer, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_i VARCHAR(255), retain_i VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)" + " VALUES('username', 't1', 'allow', 'publish', 1, 1)" + ], + query => + "SELECT permission, action, topic, qos_i as qos, retain_i as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>} + ] + }, + #{ + name => retain_in_query_result_as_boolean, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255), retain_b BOOLEAN)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't1', 'allow', 'publish', true)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't2', 'allow', 'publish', false)" + ], + query => + "SELECT permission, action, topic, retain_b as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + "CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255))", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('clientid', 't/${clientid}', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic" + " FROM acl WHERE who = ${username} OR who = ${clientid}", + client_info => #{ + %% string, not a binary + username => "username", + %% atom, not a binary + clientid => clientid + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {deny, ?AUTHZ_PUBLISH, <<"t/foo">>} + ] + }, + #{ + name => null_retain_qos, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(qos VARCHAR(255), retain VARCHAR(255)," + " topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", + + "INSERT INTO acl(qos, retain, topic, permission, action)" + " VALUES(NULL, NULL, 'tp', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic, qos FROM acl", + checks => [ + {allow, ?AUTHZ_PUBLISH(0, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"tp">>}, + + {deny, ?AUTHZ_PUBLISH(0, true), <<"xxx">>} + ] + }, + #{ + name => strip_double_quote, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))" + ], + query => "SELECT permission, action, topic FRO", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => runtime_error, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))" + ], + query => + "SELECT permission, action, topic FROM acl WHERE username = ${username}", + checks => [ + fun() -> + _ = q("DROP TABLE IF EXISTS acl"), + {deny, ?AUTHZ_PUBLISH, <<"t">>} + end + ] + }, + #{ + name => invalid_rule, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + %% 'permit' is invalid value for action + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + } + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) + end, + Queries + ). + +setup_authz_source(#{query := Query}) -> + setup_config( + #{ + <<"query">> => Query + } + ). + raw_mysql_authz_config() -> #{ <<"enable">> => <<"true">>, @@ -332,52 +419,9 @@ q(Sql, Params) -> {sql, Sql, Params} ). -init_table() -> - ok = drop_table(), - ok = q( - "CREATE TABLE acl(\n" - " username VARCHAR(255),\n" - " clientid VARCHAR(255),\n" - " peerhost VARCHAR(255),\n" - " cn VARCHAR(255),\n" - " dn VARCHAR(255),\n" - " topic VARCHAR(255),\n" - " permission VARCHAR(255),\n" - " action VARCHAR(255))" - ). - drop_table() -> ok = q("DROP TABLE IF EXISTS acl"). -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - ok = init_table(), - ok = lists:foreach( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:foreach( - fun(Topic) -> - q( - << - "INSERT INTO acl(username, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [Username, Topic, Permission, Action] - ) - end, - Topics - ) - end, - Samples - ), - setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE username = ${username}" - >> - } - ). - setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_mysql_authz_config(), @@ -400,3 +444,13 @@ start_apps(Apps) -> stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). + +create_mysql_resource() -> + {ok, _} = emqx_resource:create_local( + ?MYSQL_RESOURCE, + ?RESOURCE_GROUP, + emqx_mysql, + mysql_config(), + #{} + ), + ok. diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 0ef21360c..a9181879e 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -27,10 +27,10 @@ -define(PGSQL_RESOURCE, <<"emqx_authz_pgsql_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -41,13 +41,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?PGSQL_RESOURCE, - ?RESOURCE_GROUP, - emqx_connector_pgsql, - pgsql_config(), - #{} - ), + {ok, _} = create_pgsql_resource(), Config; false -> {skip, no_pgsql} @@ -59,9 +53,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok = drop_table(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -72,194 +75,11 @@ set_special_configs(_) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - %% by clientid - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by peerhost - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(peerhost, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE peerhost = ${peerhost}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by cn - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(cn, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE cn = ${cert_common_name}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by dn - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(dn, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE dn = ${cert_subject}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% strip double quote support - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = \"${clientid}\"" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_pgsql_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${username}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, subscribe, <<"a">>}] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). t_create_invalid(_Config) -> BadConfig = maps:merge( @@ -270,45 +90,304 @@ t_create_invalid(_Config) -> [_] = emqx_authz:lookup(). -t_nonbinary_values(_Config) -> - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, username, topic, permission, action)" - "VALUES($1, $2, $3, $4, $5)" - >>, - [<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( +cases() -> + [ #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid} AND username = ${username}" - >> - } - ), + name => base_publish, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{username => <<"username">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + "CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255)," + " peerhost VARCHAR(255), topic VARCHAR(255)," + " permission VARCHAR(255), action VARCHAR(255))", - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + "INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)" + " VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic FROM acl WHERE" + " clientid = ${clientid} AND cn = ${cert_common_name}" + " AND dn = ${cert_subject} AND peerhost = ${peerhost}", + client_info => #{ + clientid => <<"clientid">>, + cn => <<"cn">>, + dn => <<"dn">>, + peerhost => {127, 0, 0, 1} + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${clientid}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 'eq t/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/#', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't1/+', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/${username}">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/1/2">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/1">>}, + {deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>}, + {deny, ?AUTHZ_PUBLISH, <<"abc">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_s VARCHAR(255), retain_s VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't1', 'allow', 'publish', '1', 'true')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't2', 'allow', 'publish', '2', 'false')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't4', 'allow', 'subscribe', '1', null)", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)" + ], + query => + "SELECT permission, action, topic, qos_s as qos, retain_s as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}, + + {allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>}, + {deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>} + ] + }, + #{ + name => qos_retain_in_query_result_as_integer, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_i VARCHAR(255), retain_i VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)" + " VALUES('username', 't1', 'allow', 'publish', 1, 1)" + ], + query => + "SELECT permission, action, topic, qos_i as qos, retain_i as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>} + ] + }, + #{ + name => retain_in_query_result_as_boolean, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255), retain_b BOOLEAN)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't1', 'allow', 'publish', true)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't2', 'allow', 'publish', false)" + ], + query => + "SELECT permission, action, topic, retain_b as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + "CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255))", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('clientid', 't/${clientid}', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic" + " FROM acl WHERE who = ${username} OR who = ${clientid}", + client_info => #{ + %% string, not a binary + username => "username", + %% atom, not a binary + clientid => clientid + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {deny, ?AUTHZ_PUBLISH, <<"t/foo">>} + ] + }, + #{ + name => array_null_qos, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(qos INTEGER[], " + " topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", + + "INSERT INTO acl(qos, topic, permission, action)" + " VALUES('{1,2}', 'tp', 'allow', 'publish')", + + "INSERT INTO acl(qos, topic, permission, action)" + " VALUES(NULL, 'ts', 'allow', 'subscribe')" + ], + query => + "SELECT permission, action, topic, qos FROM acl", + checks => [ + {allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"tp">>}, + {deny, ?AUTHZ_PUBLISH(3, false), <<"tp">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"ts">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"ts">>} + ] + }, + #{ + name => strip_double_quote, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [], + query => "SELECT permission, action, topic FROM acl WHER", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => pgsql_error, + setup => [], + query => + "SELECT permission, action, topic FROM table_not_exists WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"t">>} + ] + }, + #{ + name => invalid_rule, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + %% 'permit' is invalid value for action + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + } + %% TODO: add case for unknown variables after fixing EMQX-10400 + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) + end, + Queries + ). + +setup_authz_source(#{query := Query}) -> + setup_config( + #{ + <<"query">> => Query + } + ). + raw_pgsql_authz_config() -> #{ <<"enable">> => <<"true">>, @@ -331,61 +410,10 @@ q(Sql) -> {query, Sql} ). -insert(Sql, Params) -> - {ok, _} = emqx_resource:simple_sync_query( - ?PGSQL_RESOURCE, - {query, Sql, Params} - ), - ok. - -init_table() -> - ok = drop_table(), - {ok, _, _} = q( - "CREATE TABLE acl(\n" - " username VARCHAR(255),\n" - " clientid VARCHAR(255),\n" - " peerhost VARCHAR(255),\n" - " cn VARCHAR(255),\n" - " dn VARCHAR(255),\n" - " topic VARCHAR(255),\n" - " permission VARCHAR(255),\n" - " action VARCHAR(255))" - ), - ok. - drop_table() -> {ok, _, _} = q("DROP TABLE IF EXISTS acl"), ok. -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - ok = init_table(), - ok = lists:foreach( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:foreach( - fun(Topic) -> - insert( - << - "INSERT INTO acl(username, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [Username, Topic, Permission, Action] - ) - end, - Topics - ) - end, - Samples - ), - setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE username = ${username}" - >> - } - ). - setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_pgsql_authz_config(), @@ -403,6 +431,15 @@ pgsql_config() -> ssl => #{enable => false} }. +create_pgsql_resource() -> + emqx_resource:create_local( + ?PGSQL_RESOURCE, + ?RESOURCE_GROUP, + emqx_connector_pgsql, + pgsql_config(), + #{} + ). + start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 29a352970..28110a7a5 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -28,10 +28,10 @@ -define(REDIS_RESOURCE, <<"emqx_authz_redis_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -42,13 +42,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?REDIS_RESOURCE, - ?RESOURCE_GROUP, - emqx_redis, - redis_config(), - #{} - ), + ok = create_redis_resource(), Config; false -> {skip, no_redis} @@ -60,9 +54,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + _ = cleanup_redis(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -73,93 +76,11 @@ set_special_configs(_) -> %% Tests %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"client id">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ByClientid = #{ - <<"mqtt_user:client id">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByClientid), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${clientid}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByPeerhost = #{ - <<"mqtt_user:127.0.0.1">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByPeerhost), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${peerhost}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByCN = #{ - <<"mqtt_user:cn">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByCN), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_common_name}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByDN = #{ - <<"mqtt_user:dn">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByDN), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_subject}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). %% should still succeed to create even if the config will not work, %% because it's not a part of the schema check @@ -181,7 +102,7 @@ t_create_with_config_values_wont_work(_Config) -> InvalidConfigs ). -%% creating without a require field should return error +%% creating without a required field should return error t_create_invalid_config(_Config) -> AuthzConfig = raw_redis_authz_config(), C = maps:without([<<"server">>], AuthzConfig), @@ -196,54 +117,211 @@ t_create_invalid_config(_Config) -> t_redis_error(_Config) -> ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}), - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), - deny = emqx_access_control:authorize(ClientInfo, subscribe, <<"a">>). + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"a">>) + ). + +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ + +cases() -> + [ + #{ + name => base_publish, + setup => [ + [ + "HMSET", + "acl:username", + "a", + "publish", + "b", + "subscribe", + "d", + "all" + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>}, + + {allow, ?AUTHZ_PUBLISH, <<"d">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"d">>} + ] + }, + #{ + name => invalid_rule, + setup => [ + [ + "HMSET", + "acl:username", + "a", + "[]", + "b", + "{invalid:json}", + "c", + "pub", + "d", + emqx_utils_json:encode(#{qos => 1, retain => true}) + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_PUBLISH, <<"c">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"d">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + ["HMSET", "acl:clientid:cn:dn:127.0.0.1", "a", "publish"] + ], + cmd => "HGETALL acl:${clientid}:${cert_common_name}:${cert_subject}:${peerhost}", + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + [ + "HMSET", + "acl:username", + "t/${username}", + "publish", + "t/${clientid}", + "publish", + "t1/#", + "publish", + "t2/+", + "publish", + "eq t3/${username}", + "publish" + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>}, + {allow, ?AUTHZ_PUBLISH, <<"t2/a">>}, + {allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>}, + {deny, ?AUTHZ_PUBLISH, <<"t3/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + [ + "HMSET", + "acl:username", + "a", + emqx_utils_json:encode(#{action => <<"publish">>, qos => 1, retain => true}), + "b", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => <<"1">>, retain => <<"true">> + }), + "c", + emqx_utils_json:encode(#{action => <<"publish">>, qos => <<"1,2">>, retain => 1}), + "d", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => [1, 2], retain => <<"1">> + }), + "e", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => [1, 2], retain => <<"all">> + }), + "f", + emqx_utils_json:encode(#{action => <<"publish">>, qos => null, retain => null}) + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"a">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"a">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"b">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"c">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"d">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"d">>}, + + {allow, ?AUTHZ_PUBLISH(1, false), <<"e">>}, + {allow, ?AUTHZ_PUBLISH(1, true), <<"e">>}, + {deny, ?AUTHZ_PUBLISH(0, false), <<"e">>}, + + {allow, ?AUTHZ_PUBLISH, <<"f">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"f">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + [ + "HMSET", + "acl:username:clientid", + "a", + "publish" + ] + ], + client_info => #{ + username => "username", + clientid => clientid + }, + cmd => "HGETALL acl:${username}:${clientid}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [ + ["SET", "acl:username", 1] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + } + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -setup_sample(AuthzData) -> - {ok, _} = q(["FLUSHDB"]), - ok = lists:foreach( - fun({Key, Values}) -> - lists:foreach( - fun({TopicFilter, Action}) -> - q(["HSET", Key, TopicFilter, Action]) - end, - maps:to_list(Values) - ) +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) end, - maps:to_list(AuthzData) + Queries ). -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - Key = <<"mqtt_user:", Username/binary>>, - lists:foreach( - fun(Sample) -> - #{ - topics := Topics, - permission := <<"allow">>, - action := Action - } = Sample, - lists:foreach( - fun(Topic) -> - q(["HSET", Key, Topic, Action]) - end, - Topics - ) - end, - Samples - ), - setup_config(#{}). +setup_authz_source(#{cmd := Cmd}) -> + setup_config( + #{ + <<"cmd">> => Cmd + } + ). setup_config(SpecialParams) -> Config = maps:merge(raw_redis_authz_config(), SpecialParams), @@ -261,6 +339,9 @@ raw_redis_authz_config() -> <<"server">> => <> }. +cleanup_redis() -> + q([<<"FLUSHALL">>]). + q(Command) -> emqx_resource:simple_sync_query( ?REDIS_RESOURCE, @@ -283,3 +364,13 @@ start_apps(Apps) -> stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). + +create_redis_resource() -> + {ok, _} = emqx_resource:create_local( + ?REDIS_RESOURCE, + ?RESOURCE_GROUP, + emqx_redis, + redis_config(), + #{} + ), + ok. diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl index fbfb84785..c73fe96ea 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl @@ -18,24 +18,17 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_authz.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --define(SOURCE1, {deny, all}). --define(SOURCE2, {allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}). --define(SOURCE3, {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}). --define(SOURCE4, {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}). --define(SOURCE5, - {allow, - {'or', [ - {username, {re, "^test"}}, - {clientid, {re, "test?"}} - ]}, - publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} -). --define(SOURCE6, {allow, {username, "test"}, publish, ["t/foo${username}boo"]}). +-define(CLIENT_INFO_BASE, #{ + clientid => <<"test">>, + username => <<"test">>, + peerhost => {127, 0, 0, 1}, + zone => default, + listener => {tcp, default} +}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -59,6 +52,12 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]), ok. +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok. + set_special_configs(emqx_authz) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -68,11 +67,11 @@ set_special_configs(_App) -> ok. t_compile(_) -> - ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile(?SOURCE1)), + ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( {allow, {ipaddr, {{127, 0, 0, 1}, {127, 0, 0, 1}, 32}}, all, [{eq, ['#']}, {eq, ['+']}]}, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ), ?assertEqual( @@ -82,14 +81,18 @@ t_compile(_) -> {{192, 168, 1, 0}, {192, 168, 1, 255}, 24} ]}, subscribe, [{pattern, [{var, [<<"clientid">>]}]}]}, - emqx_authz_rule:compile(?SOURCE3) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ), - ?assertMatch( + ?assertEqual( {allow, {'and', [{clientid, {eq, <<"test">>}}, {username, {eq, <<"test">>}}]}, publish, [ [<<"topic">>, <<"test">>] ]}, - emqx_authz_rule:compile(?SOURCE4) + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ), ?assertMatch( @@ -101,240 +104,643 @@ t_compile(_) -> publish, [ {pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]} ]}, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ), ?assertEqual( {allow, {username, {eq, <<"test">>}}, publish, [ {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]} ]}, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, + #{action_type => publish, qos => [0, 1, 2], retain => all}, [[<<"topic">>, <<"test">>]]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => publish, qos => [1], retain => true}, + [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1, 2]}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}, {qos, 2}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1]}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => all, qos => [2], retain => true}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ), + ok. +t_compile_ce(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + ?assertThrow( + {invalid_authorization_action, _}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, all, [[<<"topic">>, <<"test">>]]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, all, ["topic/test"]} + ) + ). + t_match(_) -> - ClientInfo1 = #{ - clientid => <<"test">>, - username => <<"test">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - ClientInfo2 = #{ - clientid => <<"test">>, - username => <<"test">>, - peerhost => {192, 168, 1, 10}, - zone => default, - listener => {tcp, default} - }, - ClientInfo3 = #{ - clientid => <<"test">>, - username => <<"fake">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - ClientInfo4 = #{ - clientid => <<"fake">>, - username => <<"test">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ?assertEqual( + {matched, deny}, + emqx_authz_rule:match( + client_info(), + #{action_type => subscribe, qos => 0}, + <<"#">>, + emqx_authz_rule:compile({deny, all}) + ) + ), ?assertEqual( {matched, deny}, emqx_authz_rule:match( - ClientInfo1, - subscribe, - <<"#">>, - emqx_authz_rule:compile(?SOURCE1) - ) - ), - ?assertEqual( - {matched, deny}, - emqx_authz_rule:match( - ClientInfo2, - subscribe, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"+">>, - emqx_authz_rule:compile(?SOURCE1) + emqx_authz_rule:compile({deny, all}) ) ), + ?assertEqual( {matched, deny}, emqx_authz_rule:match( - ClientInfo3, - subscribe, + client_info(#{username => <<"fake">>}), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE1) + emqx_authz_rule:compile({deny, all}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"#">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo2, - subscribe, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"#">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"test">>, - emqx_authz_rule:compile(?SOURCE3) - ) - ), - ?assertEqual( - {matched, allow}, - emqx_authz_rule:match( - ClientInfo2, - subscribe, - <<"test">>, - emqx_authz_rule:compile(?SOURCE3) - ) - ), - ?assertEqual( - nomatch, - emqx_authz_rule:match( - ClientInfo2, - subscribe, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE3) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) - ) - ), - ?assertEqual( - {matched, allow}, - emqx_authz_rule:match( - ClientInfo2, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, + <<"test">>, + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) - ) - ), - ?assertEqual( - nomatch, - emqx_authz_rule:match( - ClientInfo4, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - publish, - <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + client_info(), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo2, - publish, - <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ) ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) + ) + ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(), + #{action_type => publish, qos => 0, retain => false}, <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => publish, qos => 0, retain => false}, + <<"test">>, + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"test">>, + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"fake">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"fake">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo1, - publish, + client_info(), + #{action_type => publish, qos => 0, retain => false}, <<"t/foo${username}boo">>, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"t/footestboo">>, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ) ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, []}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, []}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 1}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, all, publish, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, all, publish, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {username, "user"}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {ipaddrs, []}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {clientid, {re, "^test"}}, all, ["#"]}) + ) + ), + ok. + +t_invalid_rule(_) -> + ?assertThrow( + {invalid_authorization_permission, _}, + emqx_authz_rule:compile({allawww, all, all, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_authorization_rule, _}, + emqx_authz_rule:compile(ooops) + ), + + ?assertThrow( + {invalid_authorization_qos, _}, + emqx_authz_rule:compile({allow, {username, "test"}, {publish, [{qos, 3}]}, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_authorization_retain, _}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, 'FALSE'}]}, ["topic/test"]} + ) + ), + + ?assertThrow( + {invalid_authorization_action, _}, + emqx_authz_rule:compile({allow, all, unsubscribe, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_who, _}, + emqx_authz_rule:compile({allow, who, all, ["topic/test"]}) + ). + +t_matches(_) -> + ?assertEqual( + {matched, allow}, + emqx_authz_rule:matches( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + [ + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ), + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ] + ) + ), + + Rule = emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:matches( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + [Rule, Rule, Rule] + ) + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +client_info() -> + ?CLIENT_INFO_BASE. + +client_info(Overrides) -> + maps:merge(?CLIENT_INFO_BASE, Overrides). diff --git a/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl new file mode 100644 index 000000000..798661b53 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl @@ -0,0 +1,288 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_rule_raw_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok. + +t_parse_ok(_Config) -> + lists:foreach( + fun({Expected, RuleRaw}) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ?assertEqual({ok, Expected}, emqx_authz_rule_raw:parse_rule(RuleRaw)), + _ = emqx_authz:set_feature_available(rich_actions, false), + ?assertEqual({ok, simple_rule(Expected)}, emqx_authz_rule_raw:parse_rule(RuleRaw)) + end, + ok_cases() + ). + +t_parse_error(_Config) -> + emqx_authz:set_feature_available(rich_actions, true), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {error, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_cases() ++ error_rich_action_cases() + ), + + %% without rich actions some fields are not parsed, so they are not errors when invalid + _ = emqx_authz:set_feature_available(rich_actions, false), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {error, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_cases() + ), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {ok, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_rich_action_cases() + ). + +t_format(_Config) -> + ?assertEqual( + #{ + action => subscribe, + permission => allow, + qos => [1, 2], + retain => true, + topic => [<<"a/b/c">>] + }, + emqx_authz_rule_raw:format_rule( + {allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]} + ) + ), + ?assertEqual( + #{ + action => publish, + permission => allow, + topic => [<<"a/b/c">>] + }, + emqx_authz_rule_raw:format_rule( + {allow, publish, [<<"a/b/c">>]} + ) + ). + +t_format_no_rich_action(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + Rule = {allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]}, + + ?assertEqual( + #{action => subscribe, permission => allow, topic => [<<"a/b/c">>]}, + emqx_authz_rule_raw:format_rule(Rule) + ). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +ok_cases() -> + [ + { + {allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a/b/c">>]}, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publish">> + } + }, + { + {deny, {subscribe, [{qos, [1, 2]}]}, [{eq, <<"a/b/c">>}]}, + #{ + <<"permission">> => <<"deny">>, + <<"topic">> => <<"eq a/b/c">>, + <<"action">> => <<"subscribe">>, + <<"retain">> => <<"true">>, + <<"qos">> => <<"1,2">> + } + }, + { + {allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a">>, <<"b">>]}, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [<<"a">>, <<"b">>], + <<"action">> => <<"publish">> + } + }, + { + {allow, {all, [{qos, [0, 1, 2]}, {retain, all}]}, []}, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"all">> + } + }, + %% Retain + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => <<"true">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => true}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => false}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => <<"false">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => <<"all">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => undefined}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => null}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => <<"1">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => <<"0">>}) + }, + %% Qos + { + expected_rule_with_qos_retain([2], all), + rule_with_raw_qos_retain(#{<<"qos">> => <<"2">>}) + }, + { + expected_rule_with_qos_retain([2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [<<"2">>]}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => <<"1,2">>}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [<<"1">>, <<"2">>]}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [1, 2]}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => undefined}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => null}) + } + ]. + +error_cases() -> + [ + #{ + <<"permission">> => <<"allo">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publish">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publis">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => #{}, + <<"action">> => <<"publish">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">> + } + ]. + +error_rich_action_cases() -> + [ + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => 3 + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => <<"three">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"retain">> => 3 + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => [<<"3">>] + } + ]. + +expected_rule_with_qos_retain(QoS, Retain) -> + {allow, {publish, [{qos, QoS}, {retain, Retain}]}, []}. + +rule_with_raw_qos_retain(Overrides) -> + maps:merge(base_raw_rule(), Overrides). + +base_raw_rule() -> + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">> + }. + +simple_rule({Pemission, {Action, _Opts}, Topics}) -> + {Pemission, Action, Topics}. diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl index e308e0de7..33035c766 100644 --- a/apps/emqx_authz/test/emqx_authz_test_lib.erl +++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl @@ -22,8 +22,6 @@ -compile(nowarn_export_all). -compile(export_all). --define(DEFAULT_CHECK_AVAIL_TIMEOUT, 1000). - reset_authorizers() -> reset_authorizers(deny, false, []). @@ -35,7 +33,7 @@ reset_authorizers(Nomatch, CacheEnabled, Source) -> [authorization], #{ <<"no_match">> => atom_to_binary(Nomatch), - <<"cache">> => #{<<"enable">> => atom_to_binary(CacheEnabled)}, + <<"cache">> => #{<<"enable">> => CacheEnabled}, <<"sources">> => Source } ), @@ -53,216 +51,70 @@ setup_config(BaseConfig, SpecialParams) -> {error, Reason} -> {error, Reason} end. -test_samples(ClientInfo, Samples) -> +%%-------------------------------------------------------------------- +%% Table-based test helpers +%%-------------------------------------------------------------------- + +all_with_table_case(Mod, TableCase, Cases) -> + (emqx_common_test_helpers:all(Mod) -- [TableCase]) ++ + [{group, Name} || Name <- case_names(Cases)]. + +table_groups(TableCase, Cases) -> + [{Name, [], [TableCase]} || Name <- case_names(Cases)]. + +case_names(Cases) -> + lists:map(fun(Case) -> maps:get(name, Case) end, Cases). + +get_case(Name, Cases) -> + [Case] = [C || C <- Cases, maps:get(name, C) =:= Name], + Case. + +setup_default_permission(Case) -> + DefaultPermission = maps:get(default_permission, Case, deny), + emqx_authz_test_lib:reset_authorizers(DefaultPermission, false). + +base_client_info() -> + #{ + clientid => <<"clientid">>, + username => <<"username">>, + peerhost => {127, 0, 0, 1}, + zone => default, + listener => {tcp, default} + }. + +client_info(Overrides) -> + maps:merge(base_client_info(), Overrides). + +enable_features(Case) -> + Features = maps:get(features, Case, []), lists:foreach( - fun({Expected, Action, Topic}) -> - ct:pal( - "client_info: ~p, action: ~p, topic: ~p, expected: ~p", - [ClientInfo, Action, Topic, Expected] - ), - ?assertEqual( - Expected, - emqx_access_control:authorize( - ClientInfo, - Action, - Topic - ) - ) + fun(Feature) -> + Enable = lists:member(Feature, Features), + emqx_authz:set_feature_available(Feature, Enable) end, - Samples + ?AUTHZ_FEATURES ). -test_no_topic_rules(ClientInfo, SetupSamples) -> - %% No rules - - ok = reset_authorizers(deny, false), - ok = SetupSamples(ClientInfo, []), - - ok = test_samples( - ClientInfo, - [ - {deny, subscribe, <<"#">>}, - {deny, subscribe, <<"subs">>}, - {deny, publish, <<"pub">>} - ] +run_checks(#{checks := Checks} = Case) -> + _ = setup_default_permission(Case), + _ = enable_features(Case), + ClientInfoOverrides = maps:get(client_info, Case, #{}), + ClientInfo = client_info(ClientInfoOverrides), + lists:foreach( + fun(Check) -> + run_check(ClientInfo, Check) + end, + Checks ). -test_allow_topic_rules(ClientInfo, SetupSamples) -> - Samples = [ - #{ - topics => [ - <<"eq testpub1/${username}">>, - <<"testpub2/${clientid}">>, - <<"testpub3/#">> - ], - permission => <<"allow">>, - action => <<"publish">> - }, - #{ - topics => [ - <<"eq testsub1/${username}">>, - <<"testsub2/${clientid}">>, - <<"testsub3/#">> - ], - permission => <<"allow">>, - action => <<"subscribe">> - }, - - #{ - topics => [ - <<"eq testall1/${username}">>, - <<"testall2/${clientid}">>, - <<"testall3/#">> - ], - permission => <<"allow">>, - action => <<"all">> - } - ], - - ok = reset_authorizers(deny, false), - ok = SetupSamples(ClientInfo, Samples), - - ok = test_samples( - ClientInfo, - [ - %% Publish rules - - {deny, publish, <<"testpub1/username">>}, - {allow, publish, <<"testpub1/${username}">>}, - {allow, publish, <<"testpub2/clientid">>}, - {allow, publish, <<"testpub3/foobar">>}, - - {deny, publish, <<"testpub2/username">>}, - {deny, publish, <<"testpub1/clientid">>}, - - {deny, subscribe, <<"testpub1/username">>}, - {deny, subscribe, <<"testpub2/clientid">>}, - {deny, subscribe, <<"testpub3/foobar">>}, - - %% Subscribe rules - - {deny, subscribe, <<"testsub1/username">>}, - {allow, subscribe, <<"testsub1/${username}">>}, - {allow, subscribe, <<"testsub2/clientid">>}, - {allow, subscribe, <<"testsub3/foobar">>}, - {allow, subscribe, <<"testsub3/+/foobar">>}, - {allow, subscribe, <<"testsub3/#">>}, - - {deny, subscribe, <<"testsub2/username">>}, - {deny, subscribe, <<"testsub1/clientid">>}, - {deny, subscribe, <<"testsub4/foobar">>}, - {deny, publish, <<"testsub1/username">>}, - {deny, publish, <<"testsub2/clientid">>}, - {deny, publish, <<"testsub3/foobar">>}, - - %% All rules - - {deny, subscribe, <<"testall1/username">>}, - {allow, subscribe, <<"testall1/${username}">>}, - {allow, subscribe, <<"testall2/clientid">>}, - {allow, subscribe, <<"testall3/foobar">>}, - {allow, subscribe, <<"testall3/+/foobar">>}, - {allow, subscribe, <<"testall3/#">>}, - {deny, publish, <<"testall1/username">>}, - {allow, publish, <<"testall1/${username}">>}, - {allow, publish, <<"testall2/clientid">>}, - {allow, publish, <<"testall3/foobar">>}, - - {deny, subscribe, <<"testall2/username">>}, - {deny, subscribe, <<"testall1/clientid">>}, - {deny, subscribe, <<"testall4/foobar">>}, - {deny, publish, <<"testall2/username">>}, - {deny, publish, <<"testall1/clientid">>}, - {deny, publish, <<"testall4/foobar">>} - ] - ). - -test_deny_topic_rules(ClientInfo, SetupSamples) -> - Samples = [ - #{ - topics => [ - <<"eq testpub1/${username}">>, - <<"testpub2/${clientid}">>, - <<"testpub3/#">> - ], - permission => <<"deny">>, - action => <<"publish">> - }, - #{ - topics => [ - <<"eq testsub1/${username}">>, - <<"testsub2/${clientid}">>, - <<"testsub3/#">> - ], - permission => <<"deny">>, - action => <<"subscribe">> - }, - - #{ - topics => [ - <<"eq testall1/${username}">>, - <<"testall2/${clientid}">>, - <<"testall3/#">> - ], - permission => <<"deny">>, - action => <<"all">> - } - ], - - ok = reset_authorizers(allow, false), - ok = SetupSamples(ClientInfo, Samples), - - ok = test_samples( - ClientInfo, - [ - %% Publish rules - - {allow, publish, <<"testpub1/username">>}, - {deny, publish, <<"testpub1/${username}">>}, - {deny, publish, <<"testpub2/clientid">>}, - {deny, publish, <<"testpub3/foobar">>}, - - {allow, publish, <<"testpub2/username">>}, - {allow, publish, <<"testpub1/clientid">>}, - - {allow, subscribe, <<"testpub1/username">>}, - {allow, subscribe, <<"testpub2/clientid">>}, - {allow, subscribe, <<"testpub3/foobar">>}, - - %% Subscribe rules - - {allow, subscribe, <<"testsub1/username">>}, - {deny, subscribe, <<"testsub1/${username}">>}, - {deny, subscribe, <<"testsub2/clientid">>}, - {deny, subscribe, <<"testsub3/foobar">>}, - {deny, subscribe, <<"testsub3/+/foobar">>}, - {deny, subscribe, <<"testsub3/#">>}, - - {allow, subscribe, <<"testsub2/username">>}, - {allow, subscribe, <<"testsub1/clientid">>}, - {allow, subscribe, <<"testsub4/foobar">>}, - {allow, publish, <<"testsub1/username">>}, - {allow, publish, <<"testsub2/clientid">>}, - {allow, publish, <<"testsub3/foobar">>}, - - %% All rules - - {allow, subscribe, <<"testall1/username">>}, - {deny, subscribe, <<"testall1/${username}">>}, - {deny, subscribe, <<"testall2/clientid">>}, - {deny, subscribe, <<"testall3/foobar">>}, - {deny, subscribe, <<"testall3/+/foobar">>}, - {deny, subscribe, <<"testall3/#">>}, - {allow, publish, <<"testall1/username">>}, - {deny, publish, <<"testall1/${username}">>}, - {deny, publish, <<"testall2/clientid">>}, - {deny, publish, <<"testall3/foobar">>}, - - {allow, subscribe, <<"testall2/username">>}, - {allow, subscribe, <<"testall1/clientid">>}, - {allow, subscribe, <<"testall4/foobar">>}, - {allow, publish, <<"testall2/username">>}, - {allow, publish, <<"testall1/clientid">>}, - {allow, publish, <<"testall4/foobar">>} - ] +run_check(ClientInfo, Fun) when is_function(Fun, 0) -> + run_check(ClientInfo, Fun()); +run_check(ClientInfo, {ExpectedPermission, Action, Topic}) -> + ?assertEqual( + ExpectedPermission, + emqx_access_control:authorize( + ClientInfo, + Action, + Topic + ) ). diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index ac1a3443f..11d199c9d 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,13 +1,14 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ kernel, stdlib, emqx, + emqx_resource, emqx_connector ]}, {env, []}, diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 7a03b24ca..a71315a27 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -175,14 +175,14 @@ bridge_info_examples(Method) -> value => info_example(mqtt, Method) } }, - ee_bridge_examples(Method) + emqx_enterprise_bridge_examples(Method) ). -if(?EMQX_RELEASE_EDITION == ee). -ee_bridge_examples(Method) -> - emqx_ee_bridge:examples(Method). +emqx_enterprise_bridge_examples(Method) -> + emqx_bridge_enterprise:examples(Method). -else. -ee_bridge_examples(_Method) -> #{}. +emqx_enterprise_bridge_examples(_Method) -> #{}. -endif. info_example(Type, Method) -> @@ -985,9 +985,13 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> {error, timeout} -> ?SERVICE_UNAVAILABLE(<<"Request timeout">>); {error, {start_pool_failed, Name, Reason}} -> - ?SERVICE_UNAVAILABLE( - bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])) - ); + Msg = bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])), + case Reason of + nxdomain -> + ?BAD_REQUEST(Msg); + _ -> + ?SERVICE_UNAVAILABLE(Msg) + end; {error, not_found} -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), ?SLOG(warning, #{ diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 59c94cef7..3bae55090 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -31,7 +31,7 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_bridge_sup:start_link(), - ok = start_ee_apps(), + ok = ensure_enterprise_schema_loaded(), ok = emqx_bridge:load(), ok = emqx_bridge:load_hook(), ok = emqx_config_handler:add_handler(?LEAF_NODE_HDLR_PATH, ?MODULE), @@ -46,11 +46,11 @@ stop(_State) -> ok. -if(?EMQX_RELEASE_EDITION == ee). -start_ee_apps() -> - {ok, _} = application:ensure_all_started(emqx_ee_bridge), +ensure_enterprise_schema_loaded() -> + _ = emqx_bridge_enterprise:module_info(), ok. -else. -start_ee_apps() -> +ensure_enterprise_schema_loaded() -> ok. -endif. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index db8669f49..203a65072 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -64,7 +64,7 @@ bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; bridge_to_resource_type(webhook) -> emqx_connector_http; -bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType). +bridge_to_resource_type(BridgeType) -> emqx_bridge_enterprise:resource_type(BridgeType). -else. bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl rename to apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 66f0dc3b4..e76d1af37 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -1,7 +1,9 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge). +-module(emqx_bridge_enterprise). + +-if(?EMQX_RELEASE_EDITION == ee). -include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -30,7 +32,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), api_ref(emqx_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), - api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"), @@ -54,7 +56,7 @@ schema_modules() -> [ emqx_bridge_kafka, emqx_bridge_cassandra, - emqx_ee_bridge_hstreamdb, + emqx_bridge_hstreamdb, emqx_bridge_gcp_pubsub, emqx_bridge_influxdb, emqx_bridge_mongodb, @@ -93,7 +95,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; resource_type(cassandra) -> emqx_bridge_cassandra_connector; -resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; +resource_type(hstreamdb) -> emqx_bridge_hstreamdb_connector; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_impl_producer; resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer; resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector; @@ -123,7 +125,7 @@ fields(bridges) -> [ {hstreamdb, mk( - hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")), + hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config")), #{ desc => <<"HStreamDB Bridge Config">>, required => false @@ -365,3 +367,7 @@ rabbitmq_structs() -> api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. + +-else. + +-endif. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 03ae781ca..58be231e4 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -57,7 +57,7 @@ api_schema(Method) -> {<<"mqtt">>, emqx_bridge_mqtt_schema} ] ], - EE = ee_api_schemas(Method), + EE = enterprise_api_schemas(Method), hoconsc:union(bridge_api_union(Broker ++ EE)). bridge_api_union(Refs) -> @@ -86,36 +86,23 @@ bridge_api_union(Refs) -> end. -if(?EMQX_RELEASE_EDITION == ee). -ee_api_schemas(Method) -> - ensure_loaded(emqx_ee_bridge, emqx_ee_bridge), - case erlang:function_exported(emqx_ee_bridge, api_schemas, 1) of - true -> emqx_ee_bridge:api_schemas(Method); +enterprise_api_schemas(Method) -> + case erlang:function_exported(emqx_bridge_enterprise, api_schemas, 1) of + true -> emqx_bridge_enterprise:api_schemas(Method); false -> [] end. -ee_fields_bridges() -> - ensure_loaded(emqx_ee_bridge, emqx_ee_bridge), - case erlang:function_exported(emqx_ee_bridge, fields, 1) of - true -> emqx_ee_bridge:fields(bridges); +enterprise_fields_bridges() -> + case erlang:function_exported(emqx_bridge_enterprise, fields, 1) of + true -> emqx_bridge_enterprise:fields(bridges); false -> [] end. -%% must ensure the app is loaded before checking if fn is defined. -ensure_loaded(App, Mod) -> - try - _ = application:load(App), - _ = Mod:module_info(), - ok - catch - _:_ -> - ok - end. - -else. -ee_api_schemas(_) -> []. +enterprise_api_schemas(_) -> []. -ee_fields_bridges() -> []. +enterprise_fields_bridges() -> []. -endif. @@ -191,7 +178,7 @@ fields(bridges) -> end } )} - ] ++ ee_fields_bridges(); + ] ++ enterprise_fields_bridges(); fields("metrics") -> [ {"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})}, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index ea3495e0f..f449588cc 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, ecql]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + ecql + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index ad41329d2..2cbf0d6fe 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -396,7 +396,7 @@ conn_opts([Opt | Opts], Acc) -> %% prepare %% XXX: hardcode -%% note: the `cql` param is passed by emqx_ee_bridge_cassa +%% note: the `cql` param is passed by emqx_bridge_cassandra parse_prepare_cql(#{cql := SQL}) -> parse_prepare_cql([{send_message, SQL}], #{}, #{}); parse_prepare_cql(_) -> diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index fb16dd749..9df219296 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -170,9 +170,8 @@ common_init(Config0) -> ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to cassnadra directly and create the table catch connect_and_drop_table(Config0), diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 452db33a7..bceae1fd2 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -56,7 +56,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), %% keyspace `mqtt` must be created in advance {ok, Conn} = ecql:connect([ @@ -79,8 +78,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector). + _ = application:stop(emqx_connector). init_per_testcase(_, Config) -> Config. diff --git a/apps/emqx_bridge_clickhouse/rebar.config b/apps/emqx_bridge_clickhouse/rebar.config index a8da74b43..98d889f41 100644 --- a/apps/emqx_bridge_clickhouse/rebar.config +++ b/apps/emqx_bridge_clickhouse/rebar.config @@ -1,6 +1,6 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}} +{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3.1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index 58a92fde4..cfb08f47b 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, - {applications, [kernel, stdlib, clickhouse, emqx_resource]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + clickhouse + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index d0164b57c..98c524913 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -469,7 +469,7 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) -> reason => ClickhouseErrorResult }), case is_recoverable_error(ClickhouseErrorResult) of - %% TODO: The hackeny errors that the clickhouse library forwards are + %% TODO: The hackney errors that the clickhouse library forwards are %% very loosely defined. We should try to make sure that the following %% handles all error cases that we need to handle as recoverable_error true -> diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index 787fb81ff..b1a560442 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -12,7 +12,7 @@ -include_lib("emqx_connector/include/emqx_connector.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to +%% apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index 0e202b714..824f5ee7b 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, erlcloud]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + erlcloud + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl index ac2b59229..9490e6455 100644 --- a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl +++ b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl @@ -88,7 +88,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, erlcloud]), ok. init_per_testcase(TestCase, Config) -> @@ -128,10 +128,12 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge + ]), + _ = application:ensure_all_started(erlcloud), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % setup dynamo setup_dynamo(Config0), diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 85bbfdd8c..bf5510366 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, ehttpc ]}, {env, []}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 890a3faed..8ef369068 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -21,7 +21,7 @@ service_account_json_converter/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). -type service_account_json() :: map(). diff --git a/lib-ee/emqx_ee_connector/docker-ct b/apps/emqx_bridge_hstreamdb/docker-ct similarity index 50% rename from lib-ee/emqx_ee_connector/docker-ct rename to apps/emqx_bridge_hstreamdb/docker-ct index ef579c036..d25a92b6b 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/apps/emqx_bridge_hstreamdb/docker-ct @@ -1,2 +1,2 @@ toxiproxy -influxdb +hstreamdb diff --git a/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl b/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl new file mode 100644 index 000000000..6b99c507a --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(HSTREAMDB_DEFAULT_PORT, 6570). diff --git a/lib-ee/emqx_ee_connector/rebar.config b/apps/emqx_bridge_hstreamdb/rebar.config similarity index 75% rename from lib-ee/emqx_ee_connector/rebar.config rename to apps/emqx_bridge_hstreamdb/rebar.config index ee1d4e500..9a70b55f9 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/apps/emqx_bridge_hstreamdb/rebar.config @@ -1,11 +1,11 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.3.1+v0.12.0"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. {shell, [ - {apps, [emqx_ee_connector]} + {apps, [emqx_bridge_hstreamdb]} ]}. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 1cb3742b3..0549dd020 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + hstreamdb_erl + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl new file mode 100644 index 000000000..7052e0120 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_hstreamdb). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"hstreamdb">> => #{ + summary => <<"HStreamDB Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + values(post); +values(put) -> + values(post); +values(post) -> + #{ + type => <<"hstreamdb">>, + name => <<"demo">>, + direction => <<"egress">>, + url => <<"http://127.0.0.1:6570">>, + stream => <<"stream">>, + %% raw HRecord + record_template => + <<"{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }">>, + pool_size => 8, + %% grpc_timeout => <<"1m">> + resource_opts => #{ + query_mode => sync, + batch_size => 100, + batch_time => <<"20ms">> + }, + ssl => #{enable => false} + }; +values(_) -> + #{}. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_hstreamdb". + +roots() -> []. + +fields("config") -> + hstream_bridge_common_fields() ++ + connector_fields(); +fields("post") -> + hstream_bridge_common_fields() ++ + connector_fields() ++ + type_name_fields(); +fields("get") -> + hstream_bridge_common_fields() ++ + connector_fields() ++ + type_name_fields() ++ + emqx_bridge_schema:status_fields(); +fields("put") -> + hstream_bridge_common_fields() ++ + connector_fields(). + +hstream_bridge_common_fields() -> + emqx_bridge_schema:common_bridge_fields() ++ + [ + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, + {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, + {record_template, + mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("record_template")})} + ] ++ + emqx_resource_schema:fields("resource_opts"). + +connector_fields() -> + emqx_bridge_hstreamdb_connector:fields(config). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for HStreamDB bridge using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal +type_name_fields() -> + [ + {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl similarity index 57% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl rename to apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 70eca83d7..16092f262 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -1,11 +1,12 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_hstreamdb). +-module(emqx_bridge_hstreamdb_connector). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -import(hoconsc, [mk/2, enum/1]). @@ -17,6 +18,7 @@ on_start/2, on_stop/2, on_query/3, + on_batch_query/3, on_get_status/2 ]). @@ -28,10 +30,15 @@ namespace/0, roots/0, fields/1, - desc/1, - connector_examples/1 + desc/1 ]). +%% Allocatable resources +-define(hstreamdb_client, hstreamdb_client). + +-define(DEFAULT_GRPC_TIMEOUT, timer:seconds(30)). +-define(DEFAULT_GRPC_TIMEOUT_RAW, <<"30s">>). + %% ------------------------------------------------------------------------------------------------- %% resource callback callback_mode() -> always_sync. @@ -39,25 +46,52 @@ callback_mode() -> always_sync. on_start(InstId, Config) -> start_client(InstId, Config). -on_stop(InstId, #{client := Client, producer := Producer}) -> - StopClientRes = hstreamdb:stop_client(Client), - StopProducerRes = hstreamdb:stop_producer(Producer), - ?SLOG(info, #{ - msg => "stop hstreamdb connector", - connector => InstId, - client => Client, - producer => Producer, - stop_client => StopClientRes, - stop_producer => StopProducerRes - }). +on_stop(InstId, _State) -> + case emqx_resource:get_allocated_resources(InstId) of + #{client := Client, producer := Producer} -> + StopClientRes = hstreamdb:stop_client(Client), + StopProducerRes = hstreamdb:stop_producer(Producer), + ?SLOG(info, #{ + msg => "stop hstreamdb connector", + connector => InstId, + client => Client, + producer => Producer, + stop_client => StopClientRes, + stop_producer => StopProducerRes + }); + _ -> + ok + end. + +-define(FAILED_TO_APPLY_HRECORD_TEMPLATE, + {error, {unrecoverable_error, failed_to_apply_hrecord_template}} +). on_query( _InstId, {send_message, Data}, - #{producer := Producer, ordering_key := OrderingKey, payload := Payload} + _State = #{ + producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate + } ) -> - Record = to_record(OrderingKey, Payload, Data), - do_append(Producer, Record). + try to_record(PartitionKey, HRecordTemplate, Data) of + Record -> append_record(Producer, Record) + catch + _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE + end. + +on_batch_query( + _InstId, + BatchList, + _State = #{ + producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate + } +) -> + try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of + Records -> append_record(Producer, Records) + catch + _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE + end. on_get_status(_InstId, #{client := Client}) -> case is_alive(Client) of @@ -87,43 +121,16 @@ fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})}, - {ordering_key, mk(binary(), #{required => false, desc => ?DESC("ordering_key")})}, - {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} - ]; -fields("get") -> - fields("post"); -fields("put") -> - fields(config); -fields("post") -> - [ - {type, mk(hstreamdb, #{required => true, desc => ?DESC("type")})}, - {name, mk(binary(), #{required => true, desc => ?DESC("name")})} - ] ++ fields("put"). + {partition_key, mk(binary(), #{required => false, desc => ?DESC("partition_key")})}, + {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}, + {grpc_timeout, fun grpc_timeout/1} + ] ++ emqx_connector_schema_lib:ssl_fields(). -connector_examples(Method) -> - [ - #{ - <<"hstreamdb">> => #{ - summary => <<"HStreamDB Connector">>, - value => values(Method) - } - } - ]. - -values(post) -> - maps:merge(values(put), #{name => <<"connector">>}); -values(get) -> - values(post); -values(put) -> - #{ - type => hstreamdb, - url => <<"http://127.0.0.1:6570">>, - stream => <<"stream1">>, - ordering_key => <<"some_key">>, - pool_size => 8 - }; -values(_) -> - #{}. +grpc_timeout(type) -> emqx_schema:timeout_duration_ms(); +grpc_timeout(desc) -> ?DESC("grpc_timeout"); +grpc_timeout(default) -> ?DEFAULT_GRPC_TIMEOUT_RAW; +grpc_timeout(required) -> false; +grpc_timeout(_) -> undefined. desc(config) -> ?DESC("config"). @@ -168,6 +175,10 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> }), start_producer(InstId, Client, Config); _ -> + ?tp( + hstreamdb_connector_start_failed, + #{error => client_not_alive} + ), ?SLOG(error, #{ msg => "hstreamdb connector: client not alive", connector => InstId @@ -202,7 +213,7 @@ is_alive(Client) -> start_producer( InstId, Client, - Options = #{stream := Stream, pool_size := PoolSize, egress := #{payload := PayloadBin}} + Options = #{stream := Stream, pool_size := PoolSize} ) -> %% TODO: change these batch options after we have better disk cache. BatchSize = maps:get(batch_size, Options, 100), @@ -212,7 +223,8 @@ start_producer( {callback, {?MODULE, on_flush_result, []}}, {max_records, BatchSize}, {interval, Interval}, - {pool_size, PoolSize} + {pool_size, PoolSize}, + {grpc_timeout, maps:get(grpc_timeout, Options, ?DEFAULT_GRPC_TIMEOUT)} ], Name = produce_name(InstId), ?SLOG(info, #{ @@ -224,17 +236,18 @@ start_producer( ?SLOG(info, #{ msg => "hstreamdb connector: producer started" }), - EnableBatch = maps:get(enable_batch, Options, false), - Payload = emqx_placeholder:preproc_tmpl(PayloadBin), - OrderingKeyBin = maps:get(ordering_key, Options, <<"">>), - OrderingKey = emqx_placeholder:preproc_tmpl(OrderingKeyBin), State = #{ client => Client, producer => Producer, - enable_batch => EnableBatch, - ordering_key => OrderingKey, - payload => Payload + enable_batch => maps:get(enable_batch, Options, false), + partition_key => emqx_placeholder:preproc_tmpl( + maps:get(partition_key, Options, <<"">>) + ), + record_template => record_template(Options) }, + ok = emqx_resource:allocate_resource(InstId, ?hstreamdb_client, #{ + client => Client, producer => Producer + }), {ok, State}; {error, {already_started, Pid}} -> ?SLOG(info, #{ @@ -253,47 +266,53 @@ start_producer( {error, Reason} end. -to_record(OrderingKeyTmpl, PayloadTmpl, Data) -> - OrderingKey = emqx_placeholder:proc_tmpl(OrderingKeyTmpl, Data), - Payload = emqx_placeholder:proc_tmpl(PayloadTmpl, Data), - to_record(OrderingKey, Payload). +to_record(PartitionKeyTmpl, HRecordTmpl, Data) -> + PartitionKey = emqx_placeholder:proc_tmpl(PartitionKeyTmpl, Data), + RawRecord = emqx_placeholder:proc_tmpl(HRecordTmpl, Data), + to_record(PartitionKey, RawRecord). -to_record(OrderingKey, Payload) when is_binary(OrderingKey) -> - to_record(binary_to_list(OrderingKey), Payload); -to_record(OrderingKey, Payload) -> - hstreamdb:to_record(OrderingKey, raw, Payload). +to_record(PartitionKey, RawRecord) when is_binary(PartitionKey) -> + to_record(binary_to_list(PartitionKey), RawRecord); +to_record(PartitionKey, RawRecord) -> + hstreamdb:to_record(PartitionKey, raw, RawRecord). -do_append(Producer, Record) -> - do_append(false, Producer, Record). +to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) -> + Records0 = lists:map( + fun({send_message, Data}) -> + to_record(PartitionKeyTmpl, HRecordTmpl, Data) + end, + BatchList + ), + PartitionKeys = proplists:get_keys(Records0), + [ + {PartitionKey, proplists:get_all_values(PartitionKey, Records0)} + || PartitionKey <- PartitionKeys + ]. -%% TODO: this append is async, remove or change it after we have better disk cache. -% do_append(true, Producer, Record) -> -% case hstreamdb:append(Producer, Record) of -% ok -> -% ?SLOG(debug, #{ -% msg => "hstreamdb producer async append success", -% record => Record -% }); -% {error, Reason} = Err -> -% ?SLOG(error, #{ -% msg => "hstreamdb producer async append failed", -% reason => Reason, -% record => Record -% }), -% Err -% end; -do_append(false, Producer, Record) -> - %% TODO: this append is sync, but it does not support [Record], can only append one Record. - %% Change it after we have better dick cache. +append_record(Producer, MultiPartsRecords) when is_list(MultiPartsRecords) -> + lists:foreach(fun(Record) -> append_record(Producer, Record) end, MultiPartsRecords); +append_record(Producer, Record) when is_tuple(Record) -> + do_append_records(false, Producer, Record). + +%% TODO: only sync request supported. implement async request later. +do_append_records(false, Producer, Record) -> case hstreamdb:append_flush(Producer, Record) of - {ok, _} -> + {ok, _Result} -> + ?tp( + hstreamdb_connector_query_return, + #{result => _Result} + ), ?SLOG(debug, #{ - msg => "hstreamdb producer sync append success", + msg => "HStreamDB producer sync append success", record => Record }); {error, Reason} = Err -> + ?tp( + hstreamdb_connector_query_return, + #{error => Reason} + ), ?SLOG(error, #{ - msg => "hstreamdb producer sync append failed", + msg => "HStreamDB producer sync append failed", reason => Reason, record => Record }), @@ -306,6 +325,11 @@ client_name(InstId) -> produce_name(ActionId) -> list_to_atom("producer:" ++ to_string(ActionId)). +record_template(#{record_template := RawHRecordTemplate}) -> + emqx_placeholder:preproc_tmpl(RawHRecordTemplate); +record_template(_) -> + emqx_placeholder:preproc_tmpl(<<"${payload}">>). + to_string(List) when is_list(List) -> List; to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin); to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom). diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl new file mode 100644 index 000000000..430343274 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -0,0 +1,578 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_hstreamdb_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx_bridge_hstreamdb.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(STREAM, "stream"). +-define(REPLICATION_FACTOR, 1). +%% in seconds +-define(BACKLOG_RETENTION_SECOND, (24 * 60 * 60)). +-define(SHARD_COUNT, 1). + +-define(BRIDGE_NAME, <<"hstreamdb_demo_bridge">>). +-define(RECORD_TEMPLATE, + "{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }" +). + +-define(POOL_SIZE, 8). +-define(BATCH_SIZE, 10). +-define(GRPC_TIMEOUT, "1s"). + +-define(WORKER_POOL_SIZE, 4). + +-define(WITH_CLIENT(Process), + Client = connect_direct_hstream(_Name = test_c, Config), + Process, + ok = disconnect(Client) +). + +%% How to run it locally (all commands are run in $PROJ_ROOT dir): +%% A: run ct on host +%% 1. Start all deps services +%% ```bash +%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ +%% -f .ci/docker-compose-file/docker-compose-hstreamdb.yaml \ +%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ +%% up --build +%% ``` +%% +%% 2. Run use cases with special environment variables +%% 6570 is toxiproxy exported port. +%% Local: +%% ```bash +%% HSTREAMDB_HOST=$REAL_TOXIPROXY_IP HSTREAMDB_PORT=6570 \ +%% PROXY_HOST=$REAL_TOXIPROXY_IP PROXY_PORT=6570 \ +%% ./rebar3 as test ct -c -v --readable true --name ct@127.0.0.1 \ +%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +%% ``` +%% +%% B: run ct in docker container +%% run script: +%% ```bash +%% ./scripts/ct/run.sh --ci --app apps/emqx_bridge_hstreamdb/ -- \ +%% --name 'test@127.0.0.1' -c -v --readable true \ +%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +%% ```` + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, sync} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], + [ + {sync, BatchingGroups}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_delete_stream(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, hstreamdb_erl]), + ok. + +init_per_testcase(t_to_hrecord_failed, Config) -> + meck:new([hstreamdb], [passthrough, no_history, no_link]), + meck:expect(hstreamdb, to_record, fun(_, _, _) -> error(trans_to_hrecord_failed) end), + Config; +init_per_testcase(_Testcase, Config) -> + %% drop stream and will create a new one in common_init/1 + %% TODO: create a new stream for each test case + delete_bridge(Config), + snabbkaffe:start_trace(), + Config. + +end_per_testcase(t_to_hrecord_failed, _Config) -> + meck:unload([hstreamdb]); +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Data = rand_data(), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, Data)), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + lists:foreach( + fun(EachTrace) -> + ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + end, + Trace + ), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + HStreamDBConfig0 = ?config(hstreamdb_config, Config), + HStreamDBConfig = HStreamDBConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(HStreamDBConfig) + ), + Data = rand_data(), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, Data)), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace) -> + ?assertMatch( + [#{result := #{streamName := <>}}], + ?of_kind(hstreamdb_connector_query_return, Trace) + ) + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + health_check_resource_ok(Config), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + health_check_resource_down(Config) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := client_not_alive}], + ?of_kind(hstreamdb_connector_start_failed, Trace) + ), + ok + end + ), + %% TODO: Investigate why reconnection takes at least 5 seconds during ct. + %% While in practical applications, recovers to the 'connected' state + %% within 3 seconds after toxiproxy being enabled.'" + %% timer:sleep(10000), + restart_resource(Config), + health_check_resource_ok(Config), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + Data = rand_data(), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + health_check_resource_down(Config), + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{msg := "call resource timeout", reason := timeout}}}, + send_message(Config, Data) + ); + async -> + %% TODO: async mode is not supported yet, + %% but it will return ok if calling emqx_resource_buffer_worker:async_query/3, + ?assertMatch( + ok, + send_message(Config, Data) + ) + end + end), + ok. + +t_simple_query(Config) -> + BatchSize = batch_size(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Requests = gen_batch_req(BatchSize), + ?check_trace( + begin + ?wait_async_action( + lists:foreach( + fun(Request) -> + ?assertEqual(ok, query_resource(Config, Request)) + end, + Requests + ), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ) + end, + fun(Trace0) -> + Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + lists:foreach( + fun(EachTrace) -> + ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + end, + Trace + ), + ok + end + ), + ok. + +t_to_hrecord_failed(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, failed_to_apply_hrecord_template}}, + Result + ) + %% TODO: async mode is not supported yet + end, + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("HSTREAMDB_HOST", "toxiproxy"), + RawPort = os:getenv("HSTREAMDB_PORT", str(?HSTREAMDB_DEFAULT_PORT)), + Port = list_to_integer(RawPort), + URL = "http://" ++ Host ++ ":" ++ RawPort, + + Config0 = [ + {hstreamdb_host, Host}, + {hstreamdb_port, Port}, + {hstreamdb_url, URL}, + %% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json + {proxy_name, "hstreamdb"}, + {batch_size, batch_size(ConfigT)} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"hstreamdb">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]), + _ = application:ensure_all_started(hstreamdb_erl), + _ = emqx_bridge_enterprise:module_info(), + emqx_mgmt_api_test_util:init_suite(), + % Connect to hstreamdb directly + % drop old stream and then create new one + connect_and_delete_stream(Config0), + connect_and_create_stream(Config0), + {Name, HStreamDBConf} = hstreamdb_config(BridgeType, Config0), + Config = + [ + {hstreamdb_config, HStreamDBConf}, + {hstreamdb_bridge_type, BridgeType}, + {hstreamdb_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_hstreamdb); + _ -> + {skip, no_hstreamdb} + end + end. + +hstreamdb_config(BridgeType, Config) -> + Port = integer_to_list(?config(hstreamdb_port, Config)), + URL = "http://" ++ ?config(hstreamdb_host, Config) ++ ":" ++ Port, + Name = ?BRIDGE_NAME, + BatchSize = batch_size(Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " url = ~p\n" + " stream = ~p\n" + " record_template = ~p\n" + " pool_size = ~p\n" + " grpc_timeout = ~p\n" + " resource_opts = {\n" + %% always sync + " query_mode = sync\n" + " request_ttl = 500ms\n" + " batch_size = ~b\n" + " worker_pool_size = ~b\n" + " }\n" + "}", + [ + BridgeType, + Name, + URL, + ?STREAM, + ?RECORD_TEMPLATE, + ?POOL_SIZE, + ?GRPC_TIMEOUT, + BatchSize, + ?WORKER_POOL_SIZE + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +-define(RPC_OPTIONS, #{pool_size => 4}). + +-define(CONN_ATTEMPTS, 10). + +default_options(Config) -> + [ + {url, ?config(hstreamdb_url, Config)}, + {rpc_options, ?RPC_OPTIONS} + ]. + +connect_direct_hstream(Name, Config) -> + client(Name, Config, ?CONN_ATTEMPTS). + +client(_Name, _Config, N) when N =< 0 -> error(cannot_connect); +client(Name, Config, N) -> + try + _ = hstreamdb:stop_client(Name), + {ok, Client} = hstreamdb:start_client(Name, default_options(Config)), + {ok, echo} = hstreamdb:echo(Client), + Client + catch + Class:Error -> + ct:print("Error connecting: ~p", [{Class, Error}]), + ct:sleep(timer:seconds(1)), + client(Name, Config, N - 1) + end. + +disconnect(Client) -> + hstreamdb:stop_client(Client). + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + HSDBConfig0 = ?config(hstreamdb_config, Config), + HSDBConfig = emqx_utils_maps:deep_merge(HSDBConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, HSDBConfig). + +delete_bridge(Config) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Data) -> + Name = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Data). + +query_resource(Config, Request) -> + Name = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +restart_resource(Config) -> + BridgeName = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + emqx_bridge:disable_enable(disable, BridgeType, BridgeName), + timer:sleep(200), + emqx_bridge:disable_enable(enable, BridgeType, BridgeName). + +resource_id(Config) -> + BridgeName = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + _ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName). + +health_check_resource_ok(Config) -> + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))). + +health_check_resource_down(Config) -> + case emqx_resource_manager:health_check(resource_id(Config)) of + {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> + ok; + {error, timeout} -> + ok; + Other -> + ?assert( + false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) + ) + end. + +% These funs start and then stop the hstreamdb connection +connect_and_create_stream(Config) -> + ?WITH_CLIENT( + _ = hstreamdb:create_stream( + Client, ?STREAM, ?REPLICATION_FACTOR, ?BACKLOG_RETENTION_SECOND, ?SHARD_COUNT + ) + ), + %% force write to stream to make it created and ready to be written data for rest cases + ProducerOptions = [ + {pool_size, 4}, + {stream, ?STREAM}, + {callback, fun(_) -> ok end}, + {max_records, 10}, + {interval, 1000} + ], + ?WITH_CLIENT( + begin + {ok, Producer} = hstreamdb:start_producer(Client, test_producer, ProducerOptions), + _ = hstreamdb:append_flush(Producer, hstreamdb:to_record([], raw, rand_payload())), + _ = hstreamdb:stop_producer(Producer) + end + ). + +connect_and_delete_stream(Config) -> + ?WITH_CLIENT( + _ = hstreamdb:delete_stream(Client, ?STREAM) + ). + +%%-------------------------------------------------------------------- +%% help functions +%%-------------------------------------------------------------------- + +batch_size(Config) -> + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end. + +rand_data() -> + #{ + %% Raw MTTT Payload in binary + payload => rand_payload(), + id => <<"0005F8F84FFFAFB9F44200000D810002">>, + topic => <<"test/topic">>, + qos => 0 + }. + +rand_payload() -> + emqx_utils_json:encode(#{ + temperature => rand:uniform(40), humidity => rand:uniform(100) + }). + +gen_batch_req(Count) when + is_integer(Count) andalso Count > 0 +-> + [{send_message, rand_data()} || _Val <- lists:seq(1, Count)]; +gen_batch_req(Count) -> + ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]). + +str(List) when is_list(List) -> + unicode:characters_to_list(List, utf8); +str(Bin) when is_binary(Bin) -> + unicode:characters_to_list(Bin, utf8); +str(Num) when is_number(Num) -> + number_to_list(Num). + +number_to_list(Int) when is_integer(Int) -> + integer_to_list(Int); +number_to_list(Float) when is_float(Float) -> + float_to_list(Float, [{decimals, 10}, compact]). diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 80b708582..71b95a40d 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, influxdb]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + influxdb + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 1fe5b4f78..be5ed6b1c 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -638,8 +638,10 @@ value_type([UInt, <<"u">>]) when is_integer(UInt) -> {uint, UInt}; -value_type([Float]) when is_float(Float) -> - Float; +%% write `1`, `1.0`, `-1.0` all as float +%% see also: https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float +value_type([Number]) when is_number(Number) -> + Number; value_type([<<"t">>]) -> 't'; value_type([<<"T">>]) -> diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl index f97e5e977..3976d187a 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl @@ -454,24 +454,26 @@ query_by_clientid(ClientId, Config) -> {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}), DecodedCSV1 = [ [Field || Field <- Line, Field =/= <<>>] - || Line <- DecodedCSV0, - Line =/= [<<>>] + || Line <- DecodedCSV0, Line =/= [<<>>] ], - DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []), + DecodedCSV2 = csv_lines_to_maps(DecodedCSV1), index_by_field(DecodedCSV2). -decode_csv(RawBody) -> - Lines = - [ - binary:split(Line, [<<";">>], [global, trim_all]) - || Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all]) - ], - csv_lines_to_maps(Lines, []). +csv_lines_to_maps([Title | Rest]) -> + csv_lines_to_maps(Rest, Title, _Acc = []); +csv_lines_to_maps([]) -> + []. -csv_lines_to_maps([Fields, Data | Rest], Acc) -> - Map = maps:from_list(lists:zip(Fields, Data)), - csv_lines_to_maps(Rest, [Map | Acc]); -csv_lines_to_maps(_Data, Acc) -> +csv_lines_to_maps([[<<"_result">> | _] = Data | RestData], Title, Acc) -> + Map = maps:from_list(lists:zip(Title, Data)), + csv_lines_to_maps(RestData, Title, [Map | Acc]); +%% ignore the csv title line +%% it's always like this: +%% [<<"result">>,<<"table">>,<<"_start">>,<<"_stop">>, +%% <<"_time">>,<<"_value">>,<<"_field">>,<<"_measurement">>, Measurement], +csv_lines_to_maps([[<<"result">> | _] = _Title | RestData], Title, Acc) -> + csv_lines_to_maps(RestData, Title, Acc); +csv_lines_to_maps([], _Title, Acc) -> lists:reverse(Acc). index_by_field(DecodedCSV) -> @@ -768,6 +770,53 @@ t_boolean_variants(Config) -> ), ok. +t_any_num_as_float(Config) -> + QueryMode = ?config(query_mode, Config), + Const = erlang:system_time(nanosecond), + ConstBin = integer_to_binary(Const), + TsStr = iolist_to_binary( + calendar:system_time_to_rfc3339(Const, [{unit, nanosecond}, {offset, "Z"}]) + ), + ?assertMatch( + {ok, _}, + create_bridge( + Config, + #{ + <<"write_syntax">> => + <<"mqtt,clientid=${clientid}", " ", + "float_no_dp=${payload.float_no_dp},float_dp=${payload.float_dp},bar=5i ", + ConstBin/binary>> + } + ) + ), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + %% no decimal point + float_no_dp => 123, + %% with decimal point + float_dp => 123.0 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"payload">> => Payload, + <<"timestamp">> => erlang:system_time(millisecond) + }, + case QueryMode of + sync -> + ?assertMatch({ok, 204, _}, send_message(Config, SentData)), + ok; + async -> + ?assertMatch(ok, send_message(Config, SentData)), + ct:sleep(500) + end, + PersistedData = query_by_clientid(ClientId, Config), + Expected = #{float_no_dp => <<"123">>, float_dp => <<"123">>}, + assert_persisted_data(ClientId, Expected, PersistedData), + TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"float_no_dp">>, PersistedData)), + TimeReturned = pad_zero(TimeReturned0), + ?assertEqual(TsStr, TimeReturned). + t_bad_timestamp(Config) -> InfluxDBType = ?config(influxdb_type, Config), InfluxDBName = ?config(influxdb_name, Config), diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index a3e4f1eb3..869656dbd 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_impl @@ -10,6 +10,9 @@ {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, + %% for module emqx_connector_http emqx_connector ]}, {env, []}, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 724c3f43a..629ac0885 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -18,7 +18,7 @@ desc/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). %%------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index 59c26717e..87c1841e5 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -1,10 +1,13 @@ +%% -*- mode: erlang -*- {application, emqx_bridge_kafka, [ {description, "EMQX Enterprise Kafka Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, [emqx_bridge_kafka_consumer_sup]}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, telemetry, wolff, brod, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 4fa188c95..bcdeaf870 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -40,7 +40,7 @@ query_mode(_) -> callback_mode() -> async_if_possible. -%% @doc Config schema is defined in emqx_ee_bridge_kafka. +%% @doc Config schema is defined in emqx_bridge_kafka. on_start(InstId, Config) -> #{ authentication := Auth, diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 38d58c1e7..95dec2db0 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -73,11 +73,9 @@ wait_until_kafka_is_up(Attempts) -> end. init_per_suite(Config) -> - %% ensure loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - application:load(emqx_bridge), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), + %% Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src index 7dfe7eae6..42129bfc7 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -1,8 +1,13 @@ {application, emqx_bridge_matrix, [ {description, "EMQX Enterprise MatrixDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index b10c92aef..fa3ebd3c9 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,6 @@ emqx_connector, emqx_resource, emqx_bridge, - emqx_ee_bridge, emqx_mongodb ]}, {env, []}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index 72485815f..b108f654f 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -10,7 +10,7 @@ -behaviour(hocon_schema). -%% emqx_ee_bridge "callbacks" +%% emqx_bridge_enterprise "callbacks" -export([ conn_bridge_examples/1 ]). diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index eb0a22e9c..8c004d829 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -58,7 +58,7 @@ on_query(InstanceId, {send_message, Message0}, State) -> }, Message = render_message(PayloadTemplate, Message0), Res = emqx_mongodb:on_query(InstanceId, {send_message, Message}, NewConnectorState), - ?tp(mongo_ee_connector_on_query_return, #{result => Res}), + ?tp(mongo_bridge_connector_on_query_return, #{result => Res}), Res; on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_mongodb:on_query(InstanceId, Request, ConnectorState). diff --git a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl index 89243bf8e..758124713 100644 --- a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl +++ b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl @@ -116,7 +116,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf, emqx_rule_engine]), + ok = emqx_common_test_helpers:stop_apps([emqx_mongodb, emqx_bridge, emqx_rule_engine, emqx_conf]), ok. init_per_testcase(_Testcase, Config) -> @@ -146,9 +146,8 @@ start_apps() -> ]). ensure_loaded() -> - _ = application:load(emqx_ee_bridge), _ = application:load(emqtt), - _ = emqx_ee_bridge:module_info(), + _ = emqx_bridge_enterprise:module_info(), ok. mongo_type(Config) -> @@ -354,7 +353,7 @@ t_setup_via_config_and_publish(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -379,7 +378,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -395,7 +394,7 @@ t_payload_template(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val, clientid => ClientId}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -421,7 +420,7 @@ t_collection_template(Config) -> clientid => ClientId, mycollectionvar => <<"mycol">> }), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 2e6844712..2ecdd6a6a 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,8 +1,15 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_mysql]}, + {applications, [ + kernel, + stdlib, + emqx_connector, + emqx_resource, + emqx_bridge, + emqx_mysql + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 2ca0f410d..3ed40e903 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -142,10 +142,9 @@ common_init(Config0) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), + % Ensure enterprise bridge module is loaded ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, emqx_rule_engine]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to mysql directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 9037b8840..6ec938afd 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, opentsdb ]}, {env, []}, diff --git a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl index 93224d5ca..3632ce786 100644 --- a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl +++ b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl @@ -53,7 +53,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([opentsdb, emqx_bridge, emqx_resource, emqx_conf]), ok. init_per_testcase(_Testcase, Config) -> @@ -91,10 +91,12 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge + ]), + _ = application:ensure_all_started(opentsdb), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), {Name, OpenTSConf} = opents_config(BridgeType, Config0), Config = diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src index ad96b4744..a05533da3 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_oracle, [ {description, "EMQX Enterprise Oracle Database Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, emqx_oracle ]}, {env, []}, diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index d7c7cec74..5c6eddb39 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -83,8 +83,9 @@ common_init_per_group() -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - application:load(emqx_bridge), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), + %% Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index 5a72107a4..ade791a6d 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,8 +1,13 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 6806328d6..d16488bc6 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -145,10 +145,9 @@ common_init(Config0) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), + % Ensure enterprise bridge module is loaded ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to pgsql directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index 487e862bc..99fb25c33 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, pulsar ]}, {env, []}, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 038da3e61..2fa5d70cf 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -15,7 +15,7 @@ fields/1, desc/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). -export([producer_strategy_key_validator/1]). diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index 4530748de..15d4b63d4 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -14,7 +14,7 @@ -import(emqx_common_test_helpers, [on_exit/1]). -define(BRIDGE_TYPE_BIN, <<"pulsar_producer">>). --define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_pulsar]). +-define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine, emqx_bridge_pulsar]). -define(RULE_TOPIC, "mqtt/rule"). -define(RULE_TOPIC_BIN, <>). @@ -122,9 +122,11 @@ common_init_per_group() -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - application:load(emqx_bridge), + %% Ensure enterprise bridge module is loaded ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps(?APPS), + ok = emqx_common_test_helpers:start_apps(?APPS), + {ok, _} = application:ensure_all_started(pulsar), + _ = emqx_bridge_enterprise:module_info(), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), UniqueNum = integer_to_binary(erlang:unique_integer()), @@ -518,7 +520,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_pulsar]}, + {apps, [emqx_conf] ++ ?APPS ++ [pulsar]}, {listener_ports, []}, {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, @@ -1097,6 +1099,7 @@ do_t_cluster(Config) -> ), {ok, _} = erpc:call(N1, fun() -> create_bridge(Config) end), {ok, _} = snabbkaffe:receive_events(SRef1), + erpc:multicall(Nodes, fun wait_until_producer_connected/0), {ok, _} = snabbkaffe:block_until( ?match_n_events( NumNodes, @@ -1118,7 +1121,6 @@ do_t_cluster(Config) -> end, Nodes ), - erpc:multicall(Nodes, fun wait_until_producer_connected/0), Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), ?tp(publishing_message, #{}), erpc:call(N2, emqx, publish, [Message0]), diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index b8f7b3327..e9ef4d524 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,8 +1,16 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, ecql, rabbit_common, amqp_client]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + ecql, + rabbit_common, + amqp_client + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl index e6a6c03fb..d3f31f5fa 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl @@ -13,7 +13,7 @@ -include_lib("amqp_client/include/amqp_client.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_connector_rabbitmq_SUITE.erl for how to +%% apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture rabbit_mq_host() -> @@ -50,8 +50,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), - {ok, _} = application:ensure_all_started(emqx_ee_bridge), {ok, _} = application:ensure_all_started(amqp_client), emqx_mgmt_api_test_util:init_suite(), ChannelConnection = setup_rabbit_mq_exchange_and_queue(), @@ -112,7 +110,6 @@ end_per_suite(Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector), _ = application:stop(emqx_bridge), %% Close the channel ok = amqp_channel:close(Channel), diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index 6b6ad617f..106a4d67b 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -48,7 +48,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), {ok, _} = application:ensure_all_started(amqp_client), ChannelConnection = setup_rabbit_mq_exchange_and_queue(), [{channel_connection, ChannelConnection} | Config]; diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 0375b6cd2..bc21adcad 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,8 +1,15 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_redis]}, + {applications, [ + kernel, + stdlib, + emqx_connector, + emqx_resource, + emqx_bridge, + emqx_redis + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index 046c42180..38a80048e 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -28,7 +28,7 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) -> case emqx_redis:on_start(InstId, Config) of {ok, RedisConnSt} -> ?tp( - redis_ee_connector_start_success, + redis_bridge_connector_start_success, #{} ), {ok, #{ @@ -37,7 +37,7 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) -> }}; {error, _} = Error -> ?tp( - redis_ee_connector_start_error, + redis_bridge_connector_start_error, #{error => Error} ), Error @@ -60,12 +60,12 @@ on_query( ) -> Cmd = proc_command_template(CommandTemplate, Data), ?tp( - redis_ee_connector_cmd, + redis_bridge_connector_cmd, #{cmd => Cmd, batch => false, mode => sync} ), Result = query(InstId, {cmd, Cmd}, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{cmd => Cmd, batch => false, mode => sync, result => Result} ), Result; @@ -75,12 +75,12 @@ on_query( _State = #{conn_st := RedisConnSt} ) -> ?tp( - redis_ee_connector_query, + redis_bridge_connector_query, #{query => Query, batch => false, mode => sync} ), Result = query(InstId, Query, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{query => Query, batch => false, mode => sync, result => Result} ), Result. @@ -90,12 +90,12 @@ on_batch_query( ) -> Cmds = process_batch_data(BatchData, CommandTemplate), ?tp( - redis_ee_connector_send, + redis_bridge_connector_send, #{batch_data => BatchData, batch => true, mode => sync} ), Result = query(InstId, {cmds, Cmds}, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{ batch_data => BatchData, batch_size => length(BatchData), diff --git a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl index 242e74b3e..6a0248b67 100644 --- a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl +++ b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl @@ -117,11 +117,9 @@ wait_for_ci_redis(Checks, Config) -> ProxyHost = os:getenv("PROXY_HOST", ?PROXY_HOST), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", ?PROXY_PORT)), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([ - emqx_resource, emqx_bridge, emqx_rule_engine + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_connector, emqx_bridge, emqx_rule_engine ]), - {ok, _} = application:ensure_all_started(emqx_connector), [ {proxy_host, ProxyHost}, {proxy_port, ProxyPort} @@ -271,21 +269,21 @@ t_check_replay(Config) -> lists:seq(1, ?BATCH_SIZE) ), #{ - ?snk_kind := redis_ee_connector_send_done, + ?snk_kind := redis_bridge_connector_send_done, batch := true, result := {error, _} }, 10_000 ) end), - #{?snk_kind := redis_ee_connector_send_done, batch := true, result := {ok, _}}, + #{?snk_kind := redis_bridge_connector_send_done, batch := true, result := {ok, _}}, 10_000 ), fun(Trace) -> ?assert( ?strict_causality( - #{?snk_kind := redis_ee_connector_send_done, result := {error, _}}, - #{?snk_kind := redis_ee_connector_send_done, result := {ok, _}}, + #{?snk_kind := redis_bridge_connector_send_done, result := {error, _}}, + #{?snk_kind := redis_bridge_connector_send_done, result := {ok, _}}, Trace ) ) @@ -308,14 +306,14 @@ t_permanent_error(_Config) -> begin ?wait_async_action( publish_message(Topic, Payload), - #{?snk_kind := redis_ee_connector_send_done}, + #{?snk_kind := redis_bridge_connector_send_done}, 10_000 ) end, fun(Trace) -> ?assertMatch( [#{result := {error, _}} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ) end ), @@ -334,7 +332,7 @@ t_create_disconnected(Config) -> fun(Trace) -> ?assertMatch( [#{error := _} | _], - ?of_kind(redis_ee_connector_start_error, Trace) + ?of_kind(redis_bridge_connector_start_error, Trace) ), ok end @@ -365,7 +363,7 @@ check_resource_queries(ResourceId, BaseTopic, IsBatch) -> end, lists:seq(1, N) ), - #{?snk_kind := redis_ee_connector_send_done, batch := IsBatch}, + #{?snk_kind := redis_bridge_connector_send_done, batch := IsBatch}, 5000 ), fun(Trace) -> @@ -374,13 +372,13 @@ check_resource_queries(ResourceId, BaseTopic, IsBatch) -> true -> ?assertMatch( [#{result := {ok, _}, batch := true, batch_size := ?BATCH_SIZE} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ), ?assertEqual(?BATCH_SIZE, AddedMsgCount); false -> ?assertMatch( [#{result := {ok, _}, batch := false} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ), ?assertEqual(1, AddedMsgCount) end diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index 7da5430a9..e18b98e3a 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, rocketmq]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge, rocketmq]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl index 62e1a7b3f..1a5133b84 100644 --- a/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl +++ b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl @@ -109,10 +109,11 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge, rocketmq + ]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), {Name, RocketMQConf} = rocketmq_config(BridgeType, Config0), Config = diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index e5c5ae73d..35f4587b0 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, odbc]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge, odbc]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl index 0e60e9c97..101ead838 100644 --- a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl +++ b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl @@ -416,10 +416,9 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, odbc]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to sqlserver directly % drop old db and table, and then create new ones diff --git a/apps/emqx_bridge_tdengine/rebar.config b/apps/emqx_bridge_tdengine/rebar.config index 72ebca1db..97ccf918a 100644 --- a/apps/emqx_bridge_tdengine/rebar.config +++ b/apps/emqx_bridge_tdengine/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.7"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 97d8ff2e5..e4c946162 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, - {applications, [kernel, stdlib, tdengine]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + tdengine + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 9399f6029..54744d806 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -142,10 +142,9 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, tdengine]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to tdengine directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src index f533f3b04..7a4aeeb56 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_timescale, [ {description, "EMQX Enterprise TimescaleDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 599b8474b..bb154f8b5 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -29,7 +29,9 @@ status/0, skip_failed_commit/1, fast_forward_to_commit/2, - on_mria_stop/1 + on_mria_stop/1, + wait_for_cluster_rpc/0, + maybe_init_tnx_id/2 ]). -export([ commit/2, @@ -59,16 +61,17 @@ -export_type([tnx_id/0, succeed_num/0]). --ifdef(TEST). --compile(export_all). --compile(nowarn_export_all). --endif. - -boot_mnesia({mnesia, [boot]}). -include_lib("emqx/include/logger.hrl"). -include("emqx_conf.hrl"). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). + +-endif. + -define(INITIATE(MFA), {initiate, MFA}). -define(CATCH_UP, catch_up). -define(TIMEOUT, timer:minutes(1)). @@ -276,6 +279,20 @@ on_mria_stop(leave) -> on_mria_stop(_) -> ok. +wait_for_cluster_rpc() -> + %% Workaround for https://github.com/emqx/mria/issues/94: + Msg1 = #{msg => "wait_for_cluster_rpc_shard"}, + case mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1500) of + ok -> ?SLOG(info, Msg1#{result => ok}); + Error0 -> ?SLOG(error, Msg1#{result => Error0}) + end, + Msg2 = #{msg => "wait_for_cluster_rpc_tables"}, + case mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]) of + ok -> ?SLOG(info, Msg2#{result => ok}); + Error1 -> ?SLOG(error, Msg2#{result => Error1}) + end, + ok. + %%%=================================================================== %%% gen_server callbacks %%%=================================================================== @@ -285,20 +302,19 @@ init([Node, RetryMs]) -> register_mria_stop_cb(fun ?MODULE:on_mria_stop/1), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs, is_leaving => false}, - %% The init transaction ID is set in emqx_conf_app after - %% it has fetched the latest config from one of the core nodes - TnxId = emqx_app:get_init_tnx_id(), - ok = maybe_init_tnx_id(Node, TnxId), %% Now continue with the normal catch-up process %% That is: apply the missing transactions after the config %% was copied until now. - {ok, State, {continue, ?CATCH_UP}}. + {ok, State, {continue, {?CATCH_UP, init}}}. %% @private -handle_continue(?CATCH_UP, State) -> +handle_continue({?CATCH_UP, init}, State) -> %% emqx app must be started before %% trying to catch up the rpc commit logs ok = wait_for_emqx_ready(), + ok = wait_for_cluster_rpc(), + {noreply, State, catch_up(State)}; +handle_continue(?CATCH_UP, State) -> {noreply, State, catch_up(State)}. handle_call(reset, _From, State) -> @@ -388,7 +404,8 @@ read_next_mfa(Node) -> }), TnxId; [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> - LastAppliedID + 1 + OldestId = get_oldest_mfa_id(), + max(LastAppliedID + 1, OldestId) end, case mnesia:read(?CLUSTER_MFA, NextId) of [] -> caught_up; @@ -404,8 +421,7 @@ do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) -> true -> NodeId; false -> - {atomic, LatestId} = transaction(fun ?MODULE:get_cluster_tnx_id/0, []), - case LatestId =< NodeId of + case latest_tnx_id() =< NodeId of true -> NodeId; false -> @@ -420,6 +436,12 @@ get_cluster_tnx_id() -> Id -> Id end. +get_oldest_mfa_id() -> + case mnesia:first(?CLUSTER_MFA) of + '$end_of_table' -> 0; + Id -> Id + end. + %% The entry point of a config change transaction. init_mfa(Node, MFA) -> mnesia:write_lock_table(?CLUSTER_MFA), diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index d8ee672f3..3c1e5592f 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index c92c28971..0a486c829 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -21,9 +21,7 @@ -export([start/2, stop/1]). -export([get_override_config_file/0]). -export([sync_data_from_node/0]). - -%% Test purposes --export([init_load_done/0]). +-export([unset_config_loaded/0]). -include_lib("emqx/include/logger.hrl"). -include("emqx_conf.hrl"). @@ -45,11 +43,16 @@ start(_StartType, _StartArgs) -> stop(_State) -> ok. +%% @doc emqx_conf relies on this flag to synchronize configuration between nodes. +%% Therefore, we must clean up this flag when emqx application is restarted by mria. +unset_config_loaded() -> + emqx_app:unset_config_loaded(). + %% Read the cluster config from the local node. %% This function is named 'override' due to historical reasons. get_override_config_file() -> Node = node(), - case init_load_done() of + case emqx_app:init_load_done() of false -> {error, #{node => Node, msg => "init_conf_load_not_done"}}; true -> @@ -93,10 +96,12 @@ sync_data_from_node() -> %% Internal functions %% ------------------------------------------------------------------------------ -init_load() -> +init_load(TnxId) -> case emqx_app:get_config_loader() of Module when Module == emqx; Module == emqx_conf -> ok = emqx_config:init_load(emqx_conf:schema_module()), + %% Set load config done after update(init) tnx_id. + ok = emqx_cluster_rpc:maybe_init_tnx_id(node(), TnxId), ok = emqx_app:set_config_loader(emqx_conf), ok; Module -> @@ -107,17 +112,11 @@ init_load() -> }) end. -init_load_done() -> - % NOTE: Either us or some higher level (i.e. tests) code loaded config. - emqx_app:get_config_loader() =/= emqx. - init_conf() -> - %% Workaround for https://github.com/emqx/mria/issues/94: - _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), - _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), + emqx_cluster_rpc:wait_for_cluster_rpc(), {ok, TnxId} = sync_cluster_conf(), - _ = emqx_app:set_init_tnx_id(TnxId), - ok = init_load(). + ok = init_load(TnxId), + ok. cluster_nodes() -> mria:cluster_nodes(cores) -- [node()]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d5d92920d..816e2f454 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -684,10 +684,10 @@ fields("cluster_call") -> )}, {"max_history", sc( - range(1, 500), + range(100, 10240), #{ desc => ?DESC(cluster_call_max_history), - default => 100 + default => 1024 } )}, {"cleanup_interval", @@ -695,7 +695,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_cleanup_interval), - default => <<"5m">> + default => <<"24h">> } )} ]; diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 2e3b40b87..2e5da3d44 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -215,7 +215,7 @@ assert_no_cluster_conf_copied([Node | Nodes], File) -> assert_config_load_done(Nodes) -> lists:foreach( fun(Node) -> - Done = rpc:call(Node, emqx_conf_app, init_load_done, []), + Done = rpc:call(Node, emqx_app, init_load_done, []), ?assert(Done, #{node => Node}) end, Nodes diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 855b8ff12..d5c864fab 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -363,14 +363,14 @@ listeners_test() -> ?assertMatch( #{ <<"bind">> := {{0, 0, 0, 0}, 1883}, - <<"enabled">> := true + <<"enable">> := true }, Tcp ), ?assertMatch( #{ <<"bind">> := {{0, 0, 0, 0}, 8083}, - <<"enabled">> := true, + <<"enable">> := true, <<"websocket">> := #{<<"mqtt_path">> := "/mqtt"} }, Ws diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index d268a244a..9dcec9187 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.26"}, + {vsn, "0.1.27"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 71d18f4a8..c468aa8bd 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -69,7 +69,7 @@ roots() -> fields(config) -> [{server, server()}] ++ - emqx_connector_schema_lib:relational_db_fields() ++ + adjust_fields(emqx_connector_schema_lib:relational_db_fields()) ++ emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). @@ -77,6 +77,22 @@ server() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?PGSQL_HOST_OPTIONS). +adjust_fields(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, fun + (required) -> + true; + (Any) -> + OrigUsernameFn(Any) + end}; + (Field) -> + Field + end, + Fields + ). + %% =================================================================== callback_mode() -> always_sync. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index e2909eca6..9cceacf3a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.24"}, + {vsn, "5.0.25"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index de35c43b3..8c56d8014 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -49,7 +49,7 @@ start_listeners(Listeners) -> Authorization = {?MODULE, authorize}, GlobalSpec = #{ openapi => "3.0.0", - info => #{title => "EMQX API", version => ?EMQX_API_VERSION}, + info => #{title => emqx_api_name(), version => emqx_release_version()}, servers => [#{url => emqx_dashboard_swagger:base_path()}], components => #{ schemas => #{}, @@ -271,3 +271,9 @@ dynamic_dispatch() -> {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, {'_', emqx_dashboard_not_found, []} ]. + +emqx_api_name() -> + emqx_release:description() ++ " API". + +emqx_release_version() -> + emqx_release:version(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 47acee58b..b0c78f0fe 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -118,7 +118,7 @@ -type route_path() :: string() | binary(). -type route_methods() :: map(). -type route_handler() :: atom(). --type route_options() :: #{filter => filter() | undefined}. +-type route_options() :: #{filter => filter()}. -type api_spec_entry() :: {route_path(), route_methods(), route_handler(), route_options()}. -type api_spec_component() :: map(). @@ -137,10 +137,9 @@ spec(Module, Options) -> {ApiSpec, AllRefs} = lists:foldl( fun(Path, {AllAcc, AllRefsAcc}) -> - {OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options), - Opts = #{filter => filter(Options)}, + {OperationId, Specs, Refs, RouteOpts} = parse_spec_ref(Module, Path, Options), { - [{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc], + [{filename:join("/", Path), Specs, OperationId, RouteOpts} | AllAcc], Refs ++ AllRefsAcc } end, @@ -350,6 +349,7 @@ parse_spec_ref(Module, Path, Options) -> ), error({failed_to_generate_swagger_spec, Module, Path}) end, + OperationId = maps:get('operationId', Schema), {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> (not lists:member(Method, ?METHODS)) andalso @@ -358,9 +358,13 @@ parse_spec_ref(Module, Path, Options) -> {Acc#{Method => Spec}, SubRefs ++ RefsAcc} end, {#{}, []}, - maps:without(['operationId'], Schema) + maps:without(['operationId', 'filter'], Schema) ), - {maps:get('operationId', Schema), Specs, Refs}. + RouteOpts = generate_route_opts(Schema, Options), + {OperationId, Specs, Refs, RouteOpts}. + +generate_route_opts(Schema, Options) -> + #{filter => compose_filters(filter(Options), custom_filter(Schema))}. check_parameters(Request, Spec, Module) -> #{bindings := Bindings, query_string := QueryStr} = Request, @@ -852,6 +856,8 @@ typename_to_spec("timeout()", _Mod) -> }; typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; +typename_to_spec("mqtt_max_packet_size()", _Mod) -> + #{type => string, example => <<"32MB">>}; typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; typename_to_spec("map()", _Mod) -> @@ -896,16 +902,28 @@ typename_to_spec("json_binary()", _Mod) -> #{type => string, example => <<"{\"a\": [1,true]}">>}; typename_to_spec("port_number()", _Mod) -> range("1..65535"); +typename_to_spec("secret_access_key()", _Mod) -> + #{type => string, example => <<"TW8dPwmjpjJJuLW....">>}; typename_to_spec(Name, Mod) -> - Spec = range(Name), - Spec1 = remote_module_type(Spec, Name, Mod), - Spec2 = typerefl_array(Spec1, Name, Mod), - Spec3 = integer(Spec2, Name), - Spec3 =:= nomatch andalso - throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}), - Spec3. + try_convert_to_spec(Name, Mod, [ + fun try_remote_module_type/2, + fun try_typerefl_array/2, + fun try_range/2, + fun try_integer/2 + ]). range(Name) -> + #{} = try_range(Name, undefined). + +try_convert_to_spec(Name, Mod, []) -> + throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}); +try_convert_to_spec(Name, Mod, [Converter | Rest]) -> + case Converter(Name, Mod) of + nomatch -> try_convert_to_spec(Name, Mod, Rest); + Spec -> Spec + end. + +try_range(Name, _Mod) -> case string:split(Name, "..") of %% 1..10 1..inf -inf..10 [MinStr, MaxStr] -> @@ -917,39 +935,33 @@ range(Name) -> end. %% Module:Type -remote_module_type(nomatch, Name, Mod) -> +try_remote_module_type(Name, Mod) -> case string:split(Name, ":") of [_Module, Type] -> typename_to_spec(Type, Mod); _ -> nomatch - end; -remote_module_type(Spec, _Name, _Mod) -> - Spec. + end. -%% [string()] or [integer()] or [xxx]. -typerefl_array(nomatch, Name, Mod) -> +%% [string()] or [integer()] or [xxx] or [xxx,...] +try_typerefl_array(Name, Mod) -> case string:trim(Name, leading, "[") of Name -> nomatch; Name1 -> - case string:trim(Name1, trailing, "]") of + case string:trim(Name1, trailing, ",.]") of Name1 -> notmatch; Name2 -> Schema = typename_to_spec(Name2, Mod), #{type => array, items => Schema} end - end; -typerefl_array(Spec, _Name, _Mod) -> - Spec. + end. %% integer(1) -integer(nomatch, Name) -> +try_integer(Name, _Mod) -> case string:to_integer(Name) of {Int, []} -> #{type => integer, enum => [Int], default => Int}; _ -> nomatch - end; -integer(Spec, _Name) -> - Spec. + end. add_integer_prop(Schema, Key, Value) -> case string:to_integer(Value) of diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 1230316e0..f0b6db8ea 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -149,6 +149,15 @@ t_swagger_json(_Config) -> {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), ?assertEqual(Body1, Body2), + ?assertMatch( + #{ + <<"info">> := #{ + <<"title">> := _, + <<"version">> := _ + } + }, + emqx_utils_json:decode(Body1) + ), ok. t_cli(_Config) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 81b3f4402..14d6f48b7 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -108,8 +108,12 @@ t_ref(_Config) -> LocalPath = "/test/in/ref/local", Path = "/test/in/ref", Expect = [#{<<"$ref">> => <<"#/components/parameters/emqx_swagger_parameter_SUITE.page">>}], - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, LocalPath, #{}), + {OperationId, Spec, Refs, RouteOpts} = emqx_dashboard_swagger:parse_spec_ref( + ?MODULE, Path, #{} + ), + {OperationId, Spec, Refs, RouteOpts} = emqx_dashboard_swagger:parse_spec_ref( + ?MODULE, LocalPath, #{} + ), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(Expect, Params), @@ -122,7 +126,7 @@ t_public_ref(_Config) -> #{<<"$ref">> => <<"#/components/parameters/public.page">>}, #{<<"$ref">> => <<"#/components/parameters/public.limit">>} ], - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(Expect, Params), @@ -264,7 +268,7 @@ t_nullable(_Config) -> t_method(_Config) -> PathOk = "/method/ok", PathError = "/method/error", - {test, Spec, []} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathOk, #{}), + {test, Spec, [], #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathOk, #{}), ?assertEqual(lists:sort(?METHODS), lists:sort(maps:keys(Spec))), ?assertThrow( {error, #{module := ?MODULE, path := PathError, method := bar}}, @@ -393,7 +397,7 @@ assert_all_filters_equal(Spec, Filter) -> ). validate(Path, ExpectParams) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(ExpectParams, Params), diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index e8c79c57c..2457cd56a 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -105,6 +105,20 @@ t_deprecated(_Config) -> emqx_dashboard_swagger:components([{?MODULE, deprecated_ref}], #{}) ). +t_nonempty_list(_Config) -> + ?assertMatch( + [ + #{ + <<"emqx_swagger_requestBody_SUITE.nonempty_list_ref">> := + #{ + <<"properties">> := + [{<<"list">>, #{items := #{type := string}, type := array}}] + } + } + ], + emqx_dashboard_swagger:components([{?MODULE, nonempty_list_ref}], #{}) + ). + t_nest_object(_Config) -> GoodRef = <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>, Spec = #{ @@ -705,7 +719,7 @@ t_object_trans_error(_Config) -> ok. validate(Path, ExpectSpec, ExpectRefs) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), ?assertEqual(ExpectSpec, Spec), ?assertEqual(ExpectRefs, Refs), @@ -829,6 +843,10 @@ fields(deprecated_ref) -> {tag1, mk(binary(), #{desc => <<"tag1">>, deprecated => {since, "4.3.0"}})}, {tag2, mk(binary(), #{desc => <<"tag2">>, deprecated => true})}, {tag3, mk(binary(), #{desc => <<"tag3">>, deprecated => false})} + ]; +fields(nonempty_list_ref) -> + [ + {list, mk(nonempty_list(binary()), #{})} ]. enable(type) -> boolean(); diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index c0771f973..4488c7fc2 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -129,7 +129,7 @@ t_error(_Config) -> } } }, - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(get, Spec)), ?assertEqual(Error400, maps:get(<<"400">>, Response)), @@ -375,7 +375,7 @@ t_complicated_type(_Config) -> } } }, - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), ?assertEqual(Object, maps:get(<<"200">>, Response)), @@ -665,7 +665,7 @@ schema("/fields/sub") -> to_schema(hoconsc:ref(sub_fields)). validate(Path, ExpectObject, ExpectRefs) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), ?assertEqual(ExpectObject, maps:get(<<"200">>, Response)), diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl index 3953ec3e2..7425cb145 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -84,7 +84,7 @@ start_cluster(NamesWithPorts, Apps, Env) -> {env, [{emqx, boot_modules, [broker, listeners]}] ++ Env}, {apps, Apps}, {conf, - [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]] ++ + [{[listeners, Proto, default, enable], false} || Proto <- [ssl, ws, wss]] ++ [{[rpc, mode], async}]} ], Cluster = emqx_common_test_helpers:emqx_cluster( diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 92a70cf37..8a57249e9 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 8720f65ae..b4358969d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -16,9 +16,9 @@ -module(emqx_exhook_handler). --include("emqx_exhook.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -export([ on_client_connect/2, @@ -132,12 +132,13 @@ on_client_authenticate(ClientInfo, AuthResult) -> {ok, AuthResult} end. -on_client_authorize(ClientInfo, PubSub, Topic, Result) -> +on_client_authorize(ClientInfo, Action, Topic, Result) -> Bool = maps:get(result, Result, deny) == allow, + %% TODO: Support full action in major release Type = - case PubSub of - publish -> 'PUBLISH'; - subscribe -> 'SUBSCRIBE' + case Action of + ?authz_action(publish) -> 'PUBLISH'; + ?authz_action(subscribe) -> 'SUBSCRIBE' end, Req = #{ clientinfo => clientinfo(ClientInfo), diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index bd756620d..3da73c11a 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx_exhook.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -126,7 +126,7 @@ t_access_failed_if_no_server_running(Config) -> allow, emqx_access_control:authorize( ClientInfo#{username => <<"gooduser">>}, - publish, + ?AUTHZ_PUBLISH, <<"acl/1">> ) ), @@ -135,7 +135,7 @@ t_access_failed_if_no_server_running(Config) -> deny, emqx_access_control:authorize( ClientInfo#{username => <<"baduser">>}, - publish, + ?AUTHZ_PUBLISH, <<"acl/2">> ) ), @@ -148,7 +148,7 @@ t_access_failed_if_no_server_running(Config) -> ?assertMatch( {stop, #{result := deny, from := exhook}}, - emqx_exhook_handler:on_client_authorize(ClientInfo, publish, <<"t/1">>, #{ + emqx_exhook_handler:on_client_authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t/1">>, #{ result => allow, from => exhook }) ), diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 075cc736c..34d7a4342 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -18,6 +18,7 @@ -include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -import( emqx_proper_types, @@ -29,7 +30,8 @@ connack_return_code/0, topictab/0, topic/0, - subopts/0 + subopts/0, + pubsub/0 ] ). @@ -138,7 +140,7 @@ prop_client_authorize() -> {ClientInfo0, PubSub, Topic, Result, Meta}, { clientinfo(), - oneof([publish, subscribe]), + pubsub(), topic(), oneof([MkResult(allow), MkResult(deny)]), request_meta() @@ -554,8 +556,8 @@ authresult_to_bool(AuthResult) -> aclresult_to_bool(#{result := Result}) -> Result == allow. -pubsub_to_enum(publish) -> 'PUBLISH'; -pubsub_to_enum(subscribe) -> 'SUBSCRIBE'. +pubsub_to_enum(?authz_action(publish)) -> 'PUBLISH'; +pubsub_to_enum(?authz_action(subscribe)) -> 'SUBSCRIBE'. from_conninfo(ConnInfo) -> #{ diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 8c37c77a8..ac498d6c6 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 7bc3a1d90..c4877fc68 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -40,27 +40,30 @@ %% API callbacks -export([ '/file_transfer/files'/2, - '/file_transfer/files/:clientid/:fileid'/2 + '/file_transfer/files/:clientid/:fileid'/2, + '/file_transfer'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). +-define(SCHEMA_CONFIG, ref(emqx_ft_schema, file_transfer)). + namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{ - check_schema => true, filter => fun ?MODULE:check_ft_enabled/2 - }). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ "/file_transfer/files", - "/file_transfer/files/:clientid/:fileid" + "/file_transfer/files/:clientid/:fileid", + "/file_transfer" ]. schema("/file_transfer/files") -> #{ 'operationId' => '/file_transfer/files', + filter => fun ?MODULE:check_ft_enabled/2, get => #{ tags => ?TAGS, summary => <<"List all uploaded files">>, @@ -83,6 +86,7 @@ schema("/file_transfer/files") -> schema("/file_transfer/files/:clientid/:fileid") -> #{ 'operationId' => '/file_transfer/files/:clientid/:fileid', + filter => fun ?MODULE:check_ft_enabled/2, get => #{ tags => ?TAGS, summary => <<"List files uploaded in a specific transfer">>, @@ -101,6 +105,30 @@ schema("/file_transfer/files/:clientid/:fileid") -> ) } } + }; +schema("/file_transfer") -> + #{ + 'operationId' => '/file_transfer', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Get current File Transfer configuration">>, + description => ?DESC("file_transfer_get_config"), + responses => #{ + 200 => ?SCHEMA_CONFIG + } + }, + put => #{ + tags => [<<"file_transfer">>], + summary => <<"Update File Transfer configuration">>, + description => ?DESC("file_transfer_update_config"), + 'requestBody' => ?SCHEMA_CONFIG, + responses => #{ + 200 => ?SCHEMA_CONFIG, + 400 => emqx_dashboard_swagger:error_codes( + ['INVALID_CONFIG'], error_desc('INVALID_CONFIG') + ) + } + } }. check_ft_enabled(Params, _Meta) -> @@ -108,7 +136,7 @@ check_ft_enabled(Params, _Meta) -> true -> {ok, Params}; false -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + {503, error_msg('SERVICE_UNAVAILABLE')} end. '/file_transfer/files'(get, #{ @@ -147,6 +175,18 @@ check_ft_enabled(Params, _Meta) -> {503, error_msg('SERVICE_UNAVAILABLE')} end. +'/file_transfer'(get, _Meta) -> + {200, format_config(emqx_ft_conf:get())}; +'/file_transfer'(put, #{body := ConfigIn}) -> + case emqx_ft_conf:update(ConfigIn) of + {ok, #{config := Config}} -> + {200, format_config(Config)}; + {error, Error = #{kind := validation_error}} -> + {400, error_msg('INVALID_CONFIG', format_validation_error(Error))}; + {error, Error} -> + {400, error_msg('INVALID_CONFIG', emqx_utils:format(Error))} + end. + format_page(#{items := Files, cursor := Cursor}) -> #{ <<"files">> => lists:map(fun format_file_info/1, Files), @@ -157,14 +197,23 @@ format_page(#{items := Files}) -> <<"files">> => lists:map(fun format_file_info/1, Files) }. +format_config(Config) -> + Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)), + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}). + +format_validation_error(Error) -> + emqx_logger_jsonfmt:best_effort_json(Error). + error_msg(Code) -> #{code => Code, message => error_desc(Code)}. error_msg(Code, Msg) -> - #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. + #{code => Code, message => Msg}. error_desc('FILES_NOT_FOUND') -> <<"Files requested for this transfer could not be found">>; +error_desc('INVALID_CONFIG') -> + <<"Provided configuration is invalid">>; error_desc('SERVICE_UNAVAILABLE') -> <<"Service unavailable">>. diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 299683e43..43a4cc816 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -18,13 +18,16 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, prep_stop/1, stop/1]). start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), ok = emqx_ft_conf:load(), {ok, Sup}. -stop(_State) -> +prep_stop(State) -> ok = emqx_ft_conf:unload(), + State. + +stop(_State) -> ok. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 2e994925c..f936b3056 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -19,6 +19,7 @@ -module(emqx_ft_conf). -behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). -include_lib("emqx/include/logger.hrl"). @@ -34,7 +35,9 @@ %% Load/Unload -export([ load/0, - unload/0 + unload/0, + get/0, + update/1 ]). %% callbacks for emqx_config_handler @@ -43,6 +46,13 @@ post_config_update/5 ]). +%% callbacks for emqx_config_backup +-export([ + import_config/1 +]). + +-type update_request() :: emqx_config:config(). + -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). @@ -95,49 +105,118 @@ load() -> -spec unload() -> ok. unload() -> - ok = stop(), - emqx_conf:remove_handler([file_transfer]). + ok = emqx_conf:remove_handler([file_transfer]), + maybe_stop(). + +-spec get() -> emqx_config:config(). +get() -> + emqx_config:get([file_transfer]). + +-spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}. +update(Config) -> + emqx_conf:update([file_transfer], Config, #{override_to => cluster}). + +%%---------------------------------------------------------------------------------------- +%% Data backup +%%---------------------------------------------------------------------------------------- + +import_config(#{<<"file_transfer">> := FTConf}) -> + OldFTConf = emqx:get_raw_config([file_transfer], #{}), + NewFTConf = maps:merge(OldFTConf, FTConf), + case emqx_conf:update([file_transfer], NewFTConf, #{override_to => cluster}) of + {ok, #{raw_config := NewRawConf}} -> + Changed = maps:get(changed, emqx_utils_maps:diff_maps(NewRawConf, FTConf)), + ChangedPaths = [[file_transfer, K] || K <- maps:keys(Changed)], + {ok, #{root_key => file_transfer, changed => ChangedPaths}}; + Error -> + {error, #{root_key => file_transfer, reason => Error}} + end; +import_config(_) -> + {ok, #{root_key => file_transfer, changed => []}}. %%-------------------------------------------------------------------- %% emqx_config_handler callbacks %%-------------------------------------------------------------------- --spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> +-spec pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. -pre_config_update(_, Req, _Config) -> - {ok, Req}. +pre_config_update([file_transfer | _], NewConfig, OldConfig) -> + propagate_config_update( + fun emqx_ft_storage_exporter_s3:pre_config_update/3, + [<<"storage">>, <<"local">>, <<"exporter">>, <<"s3">>], + NewConfig, + OldConfig + ). -spec post_config_update( list(atom()), - emqx_config:update_request(), + update_request(), emqx_config:config(), emqx_config:config(), emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> - on_config_update(OldConfig, NewConfig). + PropResult = propagate_config_update( + fun emqx_ft_storage_exporter_s3:post_config_update/3, + [storage, local, exporter, s3], + NewConfig, + OldConfig + ), + case PropResult of + ok -> + on_config_update(OldConfig, NewConfig); + {error, Reason} -> + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +propagate_config_update(Fun, ConfKey, NewConfig, OldConfig) -> + NewSubConf = emqx_utils_maps:deep_get(ConfKey, NewConfig, undefined), + OldSubConf = emqx_utils_maps:deep_get(ConfKey, OldConfig, undefined), + case Fun(ConfKey, NewSubConf, OldSubConf) of + ok -> + ok; + {ok, undefined} -> + {ok, NewConfig}; + {ok, NewSubConfUpdate} -> + {ok, emqx_utils_maps:deep_put(ConfKey, NewConfig, NewSubConfUpdate)}; + {error, Reason} -> + {error, Reason} + end. on_config_update(#{enable := false}, #{enable := false}) -> ok; on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) -> - ok = emqx_ft_storage:on_config_update(OldStorage, undefined), - ok = emqx_ft:unhook(); + ok = stop(OldStorage); on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) -> - ok = emqx_ft_storage:on_config_update(undefined, NewStorage), - ok = emqx_ft:hook(); + ok = start(NewStorage); on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) -> - ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage). + ok = emqx_ft_storage:update_config(OldStorage, NewStorage). maybe_start() -> case emqx_config:get([file_transfer]) of #{enable := true, storage := Storage} -> - ok = emqx_ft_storage:on_config_update(undefined, Storage), - ok = emqx_ft:hook(); + start(Storage); _ -> ok end. -stop() -> +maybe_stop() -> + case emqx_config:get([file_transfer]) of + #{enable := true, storage := Storage} -> + stop(Storage); + _ -> + ok + end. + +start(Storage) -> + ok = emqx_ft_storage:update_config(undefined, Storage), + ok = emqx_ft:hook(). + +stop(Storage) -> ok = emqx_ft:unhook(), - ok = emqx_ft_storage:on_config_update(storage(), undefined). + ok = emqx_ft_storage:update_config(Storage, undefined). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index e2980c920..2d068466c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -16,6 +16,8 @@ -module(emqx_ft_storage). +-include_lib("emqx/include/types.hrl"). + -export( [ store_filemeta/2, @@ -29,7 +31,7 @@ with_storage_type/3, backend/0, - on_config_update/2 + update_config/2 ] ). @@ -94,10 +96,10 @@ -callback files(storage(), query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. --callback start(emqx_config:config()) -> any(). --callback stop(emqx_config:config()) -> any(). +-callback start(storage()) -> any(). +-callback stop(storage()) -> any(). --callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) -> +-callback update_config(_OldConfig :: maybe(storage()), _NewConfig :: maybe(storage())) -> any(). %%-------------------------------------------------------------------- @@ -157,9 +159,9 @@ with_storage_type(Type, Fun, Args) -> backend() -> backend(emqx_ft_conf:storage()). --spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> +-spec update_config(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> ok. -on_config_update(ConfigOld, ConfigNew) -> +update_config(ConfigOld, ConfigNew) -> on_backend_update( emqx_maybe:apply(fun backend/1, ConfigOld), emqx_maybe:apply(fun backend/1, ConfigNew) @@ -168,13 +170,13 @@ on_config_update(ConfigOld, ConfigNew) -> on_backend_update({Type, _} = Backend, {Type, _} = Backend) -> ok; on_backend_update({Type, StorageOld}, {Type, StorageNew}) -> - ok = (mod(Type)):on_config_update(StorageOld, StorageNew); + ok = (mod(Type)):update_config(StorageOld, StorageNew); on_backend_update(BackendOld, BackendNew) when (BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso (BackendNew =:= undefined orelse is_tuple(BackendNew)) -> - _ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld), - _ = emqx_maybe:apply(fun on_storage_start/1, BackendNew), + _ = emqx_maybe:apply(fun stop_backend/1, BackendOld), + _ = emqx_maybe:apply(fun start_backend/1, BackendNew), ok. %%-------------------------------------------------------------------- @@ -185,10 +187,10 @@ on_backend_update(BackendOld, BackendNew) when backend(Config) -> emqx_ft_schema:backend(Config). -on_storage_start({Type, Storage}) -> +start_backend({Type, Storage}) -> (mod(Type)):start(Storage). -on_storage_stop({Type, Storage}) -> +stop_backend({Type, Storage}) -> (mod(Type)):stop(Storage). mod(local) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 4c9cac67a..bc1b5fb4d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -31,7 +31,7 @@ -export([list/2]). %% Lifecycle API --export([on_config_update/2]). +-export([update_config/2]). %% Internal API -export([exporter/1]). @@ -81,7 +81,7 @@ -callback stop(exporter_conf()) -> ok. --callback update(exporter_conf(), exporter_conf()) -> +-callback update_config(exporter_conf(), exporter_conf()) -> ok | {error, _Reason}. %%------------------------------------------------------------------------------ @@ -148,8 +148,8 @@ list(Storage, Query) -> %% Lifecycle --spec on_config_update(storage(), storage()) -> ok | {error, term()}. -on_config_update(StorageOld, StorageNew) -> +-spec update_config(storage(), storage()) -> ok | {error, term()}. +update_config(StorageOld, StorageNew) -> on_exporter_update( emqx_maybe:apply(fun exporter/1, StorageOld), emqx_maybe:apply(fun exporter/1, StorageNew) @@ -158,7 +158,7 @@ on_config_update(StorageOld, StorageNew) -> on_exporter_update(Config, Config) -> ok; on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> - ExporterMod:update(ConfigOld, ConfigNew); + ExporterMod:update_config(ConfigOld, ConfigNew); on_exporter_update(ExporterOld, ExporterNew) -> _ = emqx_maybe:apply(fun stop/1, ExporterOld), _ = emqx_maybe:apply(fun start/1, ExporterNew), diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index e37ba25af..9f2e5fd58 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -31,7 +31,7 @@ -export([ start/1, stop/1, - update/2 + update_config/2 ]). %% Internal API for RPC @@ -161,8 +161,8 @@ start(_Options) -> ok. -spec stop(options()) -> ok. stop(_Options) -> ok. --spec update(options(), options()) -> ok. -update(_OldOptions, _NewOptions) -> ok. +-spec update_config(options(), options()) -> ok. +update_config(_OldOptions, _NewOptions) -> ok. %%-------------------------------------------------------------------- %% Internal API diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 4db2255f6..ac06ab957 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -28,7 +28,12 @@ -export([ start/1, stop/1, - update/2 + update_config/2 +]). + +-export([ + pre_config_update/3, + post_config_update/3 ]). -type options() :: emqx_s3:profile_config(). @@ -112,12 +117,22 @@ start(Options) -> -spec stop(options()) -> ok. stop(_Options) -> - ok = emqx_s3:stop_profile(?S3_PROFILE_ID). + emqx_s3:stop_profile(?S3_PROFILE_ID). --spec update(options(), options()) -> ok. -update(_OldOptions, NewOptions) -> +-spec update_config(options(), options()) -> ok. +update_config(_OldOptions, NewOptions) -> emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions). +%%-------------------------------------------------------------------- +%% Config update hooks +%%-------------------------------------------------------------------- + +pre_config_update(_ConfKey, NewOptions, OldOptions) -> + emqx_s3:pre_config_update(?S3_PROFILE_ID, NewOptions, OldOptions). + +post_config_update(_ConfKey, NewOptions, OldOptions) -> + emqx_s3:post_config_update(?S3_PROFILE_ID, NewOptions, OldOptions). + %%-------------------------------------------------------------------- %% Internal functions %% ------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 85aa08405..1fd4d3a5d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -48,9 +48,9 @@ -export([files/2]). --export([on_config_update/2]). -export([start/1]). -export([stop/1]). +-export([update_config/2]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -230,10 +230,10 @@ files(Storage, Query) -> %% -on_config_update(StorageOld, StorageNew) -> +update_config(StorageOld, StorageNew) -> % NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely ok = emqx_ft_storage_fs_gc:reset(StorageNew), - emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). + emqx_ft_storage_exporter:update_config(StorageOld, StorageNew). start(Storage) -> ok = lists:foreach( @@ -242,11 +242,11 @@ start(Storage) -> end, child_spec(Storage) ), - ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage), + ok = emqx_ft_storage_exporter:update_config(undefined, Storage), ok. stop(Storage) -> - ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined), + ok = emqx_ft_storage_exporter:update_config(Storage, undefined), ok = lists:foreach( fun(#{id := ChildId}) -> _ = supervisor:terminate_child(emqx_ft_sup, ChildId), diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 25ad42d75..ae8a5c01c 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,58 +24,24 @@ -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). -all() -> - [ - {group, single}, - {group, cluster} - ]. - -groups() -> - [ - {single, [], emqx_common_test_helpers:all(?MODULE)}, - {cluster, [], emqx_common_test_helpers:all(?MODULE) -- [t_ft_disabled]} - ]. +all() -> emqx_common_test_helpers:all(?MODULE). suite() -> [{timetrap, {seconds, 90}}]. init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(Group = single, Config) -> - WorkDir = ?config(priv_dir, Config), - Apps = emqx_cth_suite:start( - [ - {emqx, #{}}, - {emqx_ft, "file_transfer { enable = true }"}, - {emqx_management, #{}}, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} - ], - #{work_dir => WorkDir} - ), - {ok, App} = emqx_common_test_http:create_default_app(), - [{group, Group}, {group_apps, Apps}, {api, App} | Config]; -init_per_group(Group = cluster, Config) -> WorkDir = ?config(priv_dir, Config), Cluster = mk_cluster_specs(Config), Nodes = [Node1 | _] = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}), {ok, App} = erpc:call(Node1, emqx_common_test_http, create_default_app, []), - [{group, Group}, {cluster_nodes, Nodes}, {api, App} | Config]. + [{cluster_nodes, Nodes}, {api, App} | Config]. -end_per_group(single, Config) -> - {ok, _} = emqx_common_test_http:delete_default_app(), - ok = emqx_cth_suite:stop(?config(group_apps, Config)); -end_per_group(cluster, Config) -> - ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)); -end_per_group(_Group, _Config) -> - ok. +end_per_suite(Config) -> + ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)). mk_cluster_specs(_Config) -> Apps = [ - {emqx_conf, #{start => false}}, + emqx_conf, {emqx, #{override_env => [{boot_modules, [broker, listeners]}]}}, {emqx_ft, "file_transfer { enable = true }"}, {emqx_management, #{}} @@ -106,9 +72,8 @@ mk_cluster_specs(_Config) -> init_per_testcase(Case, Config) -> [{tc, Case} | Config]. -end_per_testcase(t_ft_disabled, _Config) -> - emqx_config:put([file_transfer, enable], true); -end_per_testcase(_Case, _Config) -> +end_per_testcase(_Case, Config) -> + ok = reset_ft_config(Config, true), ok. %%-------------------------------------------------------------------- @@ -294,7 +259,7 @@ t_ft_disabled(Config) -> ) ), - ok = emqx_config:put([file_transfer, enable], false), + ok = reset_ft_config(Config, false), ?assertMatch( {ok, 503, _}, @@ -310,17 +275,161 @@ t_ft_disabled(Config) -> ) ). +t_configure(Config) -> + ?assertMatch( + {ok, 200, #{<<"enable">> := true, <<"storage">> := #{}}}, + request_json(get, uri(["file_transfer"]), Config) + ), + ?assertMatch( + {ok, 200, #{<<"enable">> := false}}, + request_json(put, uri(["file_transfer"]), #{<<"enable">> => false}, Config) + ), + ?assertMatch( + {ok, 200, #{<<"enable">> := false}}, + request_json(get, uri(["file_transfer"]), Config) + ), + ?assertMatch( + {ok, 200, #{}}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => emqx_ft_test_helpers:local_storage(Config) + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{}, + <<"remote">> => #{} + } + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"gc">> => #{<<"interval">> => -42} + } + } + }, + Config + ) + ), + S3Exporter = #{ + <<"host">> => <<"localhost">>, + <<"port">> => 9000, + <<"bucket">> => <<"emqx">>, + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"enable">> => true, + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }, + ?assertMatch( + {ok, 200, #{ + <<"enable">> := true, + <<"storage">> := #{ + <<"local">> := #{ + <<"exporter">> := #{ + <<"s3">> := #{ + <<"transport_options">> := #{ + <<"ssl">> := #{ + <<"enable">> := true, + <<"certfile">> := <<"/", _CertFilepath/bytes>>, + <<"keyfile">> := <<"/", _KeyFilepath/bytes>> + } + } + } + } + } + } + }}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => S3Exporter + } + } + } + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => emqx_utils_maps:deep_put( + [<<"transport_options">>, <<"ssl">>, <<"keyfile">>], + S3Exporter, + <<>> + ) + } + } + } + }, + Config + ) + ), + ?assertMatch( + {ok, 200, #{}}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => emqx_utils_maps:deep_put( + [<<"transport_options">>, <<"ssl">>, <<"enable">>], + S3Exporter, + false + ) + } + } + } + }, + Config + ) + ), + ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- test_nodes(Config) -> - case proplists:get_value(cluster_nodes, Config, []) of - [] -> - [node()]; - Nodes -> - Nodes - end. + ?config(cluster_nodes, Config). client_id(Config) -> iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])). @@ -332,17 +441,26 @@ mk_file_name(N) -> "file." ++ integer_to_list(N). request(Method, Url, Config) -> - Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, - emqx_mgmt_api_test_util:request_api(Method, Url, [], auth_header(Config), [], Opts). + request(Method, Url, [], Config). -request_json(Method, Url, Config) -> - case request(Method, Url, Config) of - {ok, Code, Body} -> - {ok, Code, json(Body)}; +request(Method, Url, Body, Config) -> + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + request(Method, Url, Body, Opts, Config). + +request(Method, Url, Body, Opts, Config) -> + emqx_mgmt_api_test_util:request_api(Method, Url, [], auth_header(Config), Body, Opts). + +request_json(Method, Url, Body, Config) -> + case request(Method, Url, Body, Config) of + {ok, Code, RespBody} -> + {ok, Code, json(RespBody)}; Otherwise -> Otherwise end. +request_json(Method, Url, Config) -> + request_json(Method, Url, [], Config). + json(Body) when is_binary(Body) -> emqx_utils_json:decode(Body, [return_maps]). @@ -368,3 +486,17 @@ to_list(L) when is_list(L) -> pick(N, List) -> lists:nth(1 + (N rem length(List)), List). + +reset_ft_config(Config, Enable) -> + [Node | _] = test_nodes(Config), + LocalConfig = + #{ + <<"enable">> => Enable, + <<"storage">> => #{ + <<"local">> => #{ + <<"enable">> => true + } + } + }, + {ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]), + ok. diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index bc0adf416..3fdfdf65a 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -53,16 +53,13 @@ end_per_testcase(_Case, Config) -> t_update_config(_Config) -> ?assertMatch( {error, #{kind := validation_error}}, - emqx_conf:update( - [file_transfer], - #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}}, - #{} + emqx_ft_conf:update( + #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}} ) ), ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], + emqx_ft_conf:update( #{ <<"enable">> => true, <<"storage">> => #{ @@ -81,8 +78,7 @@ t_update_config(_Config) -> } } } - }, - #{} + } ) ), ?assertEqual( @@ -101,13 +97,8 @@ t_update_config(_Config) -> t_disable_restore_config(Config) -> ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{ - <<"enable">> => true, - <<"storage">> => #{<<"local">> => #{}} - }, - #{} + emqx_ft_conf:update( + #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}} ) ), ?assertEqual( @@ -119,11 +110,7 @@ t_disable_restore_config(Config) -> % Verify that clearing storage settings reverts config to defaults ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{<<"enable">> => false, <<"storage">> => undefined}, - #{} - ) + emqx_ft_conf:update(#{<<"enable">> => false, <<"storage">> => undefined}) ), ?assertEqual( false, @@ -155,8 +142,7 @@ t_disable_restore_config(Config) -> Root = emqx_ft_test_helpers:root(Config, node(), [segments]), ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], + emqx_ft_conf:update( #{ <<"enable">> => true, <<"storage">> => #{ @@ -167,8 +153,7 @@ t_disable_restore_config(Config) -> } } } - }, - #{} + } ) ), % Verify that GC is getting triggered eventually @@ -192,11 +177,7 @@ t_disable_restore_config(Config) -> t_switch_exporter(_Config) -> ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{<<"enable">> => true}, - #{} - ) + emqx_ft_conf:update(#{<<"enable">> => true}) ), ?assertMatch( #{local := #{exporter := #{local := _}}}, @@ -248,5 +229,129 @@ t_switch_exporter(_Config) -> % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). +t_persist_ssl_certfiles(Config) -> + ?assertMatch( + {ok, _}, + emqx_ft_conf:update(mk_storage(true)) + ), + ?assertEqual( + [], + list_ssl_certfiles(Config) + ), + ?assertMatch( + {error, {pre_config_update, _, {bad_ssl_config, #{}}}}, + emqx_ft_conf:update( + mk_storage(true, #{ + <<"s3">> => mk_s3_config(#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => <<"cert.pem">>, + <<"keyfile">> => <<"key.pem">> + } + } + }) + }) + ) + ), + ?assertMatch( + {ok, _}, + emqx_ft_conf:update( + mk_storage(false, #{ + <<"s3">> => mk_s3_config(#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }) + }) + ) + ), + ?assertMatch( + #{ + local := #{ + exporter := #{ + s3 := #{ + transport_options := #{ + ssl := #{ + certfile := <<"/", _CertFilepath/binary>>, + keyfile := <<"/", _KeyFilepath/binary>> + } + } + } + } + } + }, + emqx_ft_conf:storage() + ), + ?assertMatch( + [_Certfile, _Keyfile], + list_ssl_certfiles(Config) + ), + ?assertMatch( + {ok, _}, + emqx_ft_conf:update(mk_storage(true)) + ). + +t_import(_Config) -> + {ok, _} = + emqx_ft_conf:update( + mk_storage(true, #{ + <<"s3">> => mk_s3_config(#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }) + }) + ), + + BackupConfig = emqx_config:get_raw([]), + FTBackupConfig = maps:with([<<"file_transfer">>], BackupConfig), + + {ok, _} = emqx_ft_conf:update(mk_storage(true)), + + ?assertMatch( + {ok, _}, + emqx_ft_conf:import_config(FTBackupConfig) + ), + + ?assertMatch( + #{local := #{exporter := #{s3 := #{enable := true}}}}, + emqx_ft_conf:storage() + ). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +mk_storage(Enabled) -> + mk_storage(Enabled, #{<<"local">> => #{}}). + +mk_storage(Enabled, Exporter) -> + #{ + <<"enable">> => Enabled, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => Exporter + } + } + }. + +mk_s3_config(S3Config) -> + BaseS3Config = #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000 + }, + maps:merge(BaseS3Config, S3Config). + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). + +list_ssl_certfiles(_Config) -> + CertDir = emqx:mutable_certs_dir(), + filelib:fold_files(CertDir, ".*", true, fun(Filepath, Acc) -> [Filepath | Acc] end, []). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 50925cfb9..a57cdf621 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -35,10 +35,18 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), - Config. -end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + Storage = emqx_ft_test_helpers:local_storage(Config), + WorkDir = ?config(priv_dir, Config), + Apps = emqx_cth_suite:start( + [ + {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}} + ], + #{work_dir => WorkDir} + ), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), ok. init_per_testcase(Case, Config) -> @@ -46,14 +54,25 @@ init_per_testcase(Case, Config) -> end_per_testcase(_Case, _Config) -> ok. -init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1), - [{additional_node, Node} | Config]; +init_per_group(Group = cluster, Config) -> + WorkDir = filename:join(?config(priv_dir, Config), Group), + Apps = [ + {emqx_conf, #{start => false}}, + {emqx_ft, "file_transfer { enable = true, storage.local { enable = true } }"} + ], + Nodes = emqx_cth_cluster:start( + [ + {emqx_ft_storage_fs1, #{apps => Apps, join_to => node()}}, + {emqx_ft_storage_fs2, #{apps => Apps, join_to => node()}} + ], + #{work_dir => WorkDir} + ), + [{cluster, Nodes} | Config]; init_per_group(_Group, Config) -> Config. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config)); + ok = emqx_cth_suite:stop(?config(cluster, Config)); end_per_group(_Group, _Config) -> ok. @@ -62,12 +81,9 @@ end_per_group(_Group, _Config) -> %%-------------------------------------------------------------------- t_multinode_exports(Config) -> - Node1 = ?config(additional_node, Config), + [Node1, Node2 | _] = ?config(cluster, Config), ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), - - Node2 = node(), ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), - ?assertMatch( [ #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index fbb3e7d6f..9e69118c8 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -24,23 +24,6 @@ -define(S3_HOST, <<"minio">>). -define(S3_PORT, 9000). -start_additional_node(Config, Name) -> - emqx_common_test_helpers:start_slave( - Name, - [ - {apps, [emqx_ft]}, - {join_to, node()}, - {configure_gen_rpc, true}, - {env_handler, env_handler(Config)} - ] - ). - -stop_additional_node(Node) -> - _ = rpc:call(Node, ekka, leave, []), - ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), - ok = emqx_common_test_helpers:stop_slave(Node), - ok. - env_handler(Config) -> fun (emqx_ft) -> @@ -136,3 +119,13 @@ upload_file(ClientId, FileId, Name, Data, Node) -> aws_config() -> emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). + +pem_privkey() -> + << + "\n" + "-----BEGIN EC PRIVATE KEY-----\n" + "MHQCAQEEICKTbbathzvD8zvgjL7qRHhW4alS0+j0Loo7WeYX9AxaoAcGBSuBBAAK\n" + "oUQDQgAEJBdF7MIdam5T4YF3JkEyaPKdG64TVWCHwr/plC0QzNVJ67efXwxlVGTo\n" + "ju0VBj6tOX1y6C0U+85VOM0UU5xqvw==\n" + "-----END EC PRIVATE KEY-----\n" + >>. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 52f96bcd2..9f3344e95 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -715,13 +715,13 @@ parse_incoming( NState = State#state{parse_state = NParseState}, parse_incoming(Rest, [Packet | Packets], NState) catch - error:Reason:Stk -> + error:Reason:Stack -> ?SLOG(error, #{ msg => "parse_frame_failed", at_state => ParseState, input_bytes => Data, reason => Reason, - stacktrace => Stk + stacktrace => Stack }), {[{frame_error, Reason} | Packets], State} end. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index bfcf4f2f2..6db4b0674 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 8037f4197..8cfcb70e6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -57,7 +57,8 @@ qs2ms/2, run_fuzzy_filter/2, format_channel_info/1, - format_channel_info/2 + format_channel_info/2, + client_info_mountpoint/1 ]). -define(TAGS, [<<"Gateway Clients">>]). @@ -177,8 +178,12 @@ clients_insta(delete, #{ } }) -> with_gateway(Name0, fun(GwName, _) -> - _ = emqx_gateway_http:kickout_client(GwName, ClientId), - {204} + case emqx_gateway_http:kickout_client(GwName, ClientId) of + {error, not_found} -> + return_http_error(404, "Client not found"); + _ -> + {204} + end end). %% List the established subscriptions with mountpoint @@ -234,8 +239,13 @@ subscriptions(delete, #{ } }) -> with_gateway(Name0, fun(GwName, _) -> - _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), - {204} + case lookup_topic(GwName, ClientId, Topic) of + {ok, _} -> + _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), + {204}; + {error, not_found} -> + return_http_error(404, "Resource not found") + end end). %%-------------------------------------------------------------------- @@ -260,6 +270,34 @@ extra_sub_props(Props) -> #{subid => maps:get(<<"subid">>, Props, undefined)} ). +lookup_topic(GwName, ClientId, Topic) -> + Mountpoints = emqx_gateway_http:lookup_client( + GwName, + ClientId, + {?MODULE, client_info_mountpoint} + ), + case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of + {ok, Subscriptions} -> + case + [ + S + || S = #{topic := Topic0} <- Subscriptions, + Mountpoint <- Mountpoints, + Topic0 == emqx_mountpoint:mount(Mountpoint, Topic) + ] + of + [] -> + {error, not_found}; + Filtered -> + {ok, Filtered} + end; + Error -> + Error + end. + +client_info_mountpoint({_, #{clientinfo := #{mountpoint := Mountpoint}}, _}) -> + Mountpoint. + %%-------------------------------------------------------------------- %% QueryString to MatchSpec diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index e62923bc2..046e23300 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -112,8 +112,13 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> - ok = emqx_gateway_http:remove_listener(ListenerId), - {204} + case emqx_gateway_conf:listener(ListenerId) of + {ok, _Listener} -> + ok = emqx_gateway_http:remove_listener(ListenerId), + {204}; + {error, not_found} -> + return_http_error(404, "Listener not found") + end end); listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 32e5fcf96..11ad55d3e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -162,8 +162,8 @@ connection_closed(_Ctx = #{gwname := GwName}, ClientId) -> emqx_types:topic() ) -> allow | deny. -authorize(_Ctx, ClientInfo, PubSub, Topic) -> - emqx_access_control:authorize(ClientInfo, PubSub, Topic). +authorize(_Ctx, ClientInfo, Action, Topic) -> + emqx_access_control:authorize(ClientInfo, Action, Topic). metrics_inc(_Ctx = #{gwname := GwName}, Name) -> emqx_gateway_metrics:inc(GwName, Name). diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 58c201c75..2186ac3d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -550,7 +550,7 @@ with_gateway(GwName0, Fun) -> return_http_error(400, [K, " is required"]); %% Exceptions from emqx_gateway_utils:parse_listener_id/1 error:{invalid_listener_id, Id} -> - return_http_error(400, ["Invalid listener id: ", Id]); + return_http_error(404, ["Listener not found: ", Id]); %% Exceptions from emqx:get_config/1 error:{config_not_found, Path0} -> Path = lists:concat( diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 634a02f03..eb4ce9fdf 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -46,7 +46,8 @@ global_chain/1, listener_chain/3, find_gateway_definitions/0, - plus_max_connections/2 + plus_max_connections/2, + random_clientid/1 ]). -export([stringfy/1]). @@ -631,3 +632,6 @@ ensure_gateway_loaded() -> emqx_gateway_mqttsn ] ). + +random_clientid(GwName) when is_atom(GwName) -> + iolist_to_binary([atom_to_list(GwName), "-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index f1cfd26d0..a3fe39852 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -409,6 +409,7 @@ t_listeners_tcp(_) -> {204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"), {404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"), + {404, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"), ok. t_listeners_max_conns(_) -> @@ -480,9 +481,19 @@ t_listeners_authn(_) -> {200, ConfResp3} = request(get, Path), assert_confs(AuthConf2, ConfResp3), + {404, _} = request(get, Path ++ "/users/not_exists"), + {404, _} = request(delete, Path ++ "/users/not_exists"), + {204, _} = request(delete, Path), %% FIXME: 204? {204, _} = request(get, Path), + + BadPath = "/gateways/stomp/listeners/stomp:tcp:not_exists/authentication/users/foo", + {404, _} = request(get, BadPath), + {404, _} = request(delete, BadPath), + + {404, _} = request(get, "/gateways/stomp/listeners/not_exists"), + {404, _} = request(delete, "/gateways/stomp/listeners/not_exists"), ok. t_listeners_authn_data_mgmt(_) -> @@ -575,6 +586,47 @@ t_listeners_authn_data_mgmt(_) -> ok. +t_clients(_) -> + GwConf = #{ + name => <<"mqttsn">>, + gateway_id => 1, + broadcast => true, + predefined => [#{id => 1, topic => <<"t/a">>}], + enable_qos3 => true, + listeners => [ + #{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>} + ] + }, + init_gw("mqttsn", GwConf), + Path = "/gateways/mqttsn/clients", + MyClient = Path ++ "/my_client", + MyClientSubscriptions = MyClient ++ "/subscriptions", + {200, NoClients} = request(get, Path), + ?assertMatch(#{data := []}, NoClients), + + ClientSocket = emqx_gateway_test_utils:sn_client_connect(<<"my_client">>), + {200, _} = request(get, MyClient), + {200, Clients} = request(get, Path), + ?assertMatch(#{data := [#{clientid := <<"my_client">>}]}, Clients), + + {201, _} = request(post, MyClientSubscriptions, #{topic => <<"test/topic">>}), + {200, Subscriptions} = request(get, MyClientSubscriptions), + ?assertMatch([#{topic := <<"test/topic">>}], Subscriptions), + {204, _} = request(delete, MyClientSubscriptions ++ "/test%2Ftopic"), + {200, []} = request(get, MyClientSubscriptions), + {404, _} = request(delete, MyClientSubscriptions ++ "/test%2Ftopic"), + + {204, _} = request(delete, MyClient), + {404, _} = request(delete, MyClient), + {404, _} = request(get, MyClient), + {404, _} = request(get, MyClientSubscriptions), + {404, _} = request(post, MyClientSubscriptions, #{topic => <<"foo">>}), + {404, _} = request(delete, MyClientSubscriptions ++ "/topic"), + {200, NoClients2} = request(get, Path), + ?assertMatch(#{data := []}, NoClients2), + emqx_gateway_test_utils:sn_client_disconnect(ClientSocket), + ok. + t_authn_fuzzy_search(_) -> init_gw("stomp"), AuthConf = #{ diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index 641528eda..fdaa55ab8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -54,6 +54,8 @@ end). "}\n" ). +-import(emqx_gateway_test_utils, [sn_client_connect/1, sn_client_disconnect/1]). + %%-------------------------------------------------------------------- %% Setup %%-------------------------------------------------------------------- @@ -303,17 +305,3 @@ acc_print(Acc) -> after 200 -> Acc end. - -sn_client_connect(ClientId) -> - {ok, Socket} = gen_udp:open(0, [binary]), - _ = emqx_sn_protocol_SUITE:send_connect_msg(Socket, ClientId), - ?assertEqual( - <<3, 16#05, 0>>, - emqx_sn_protocol_SUITE:receive_response(Socket) - ), - Socket. - -sn_client_disconnect(Socket) -> - _ = emqx_sn_protocol_SUITE:send_disconnect_msg(Socket, undefined), - gen_udp:close(Socket), - ok. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 2e8a3a583..950ae1bcf 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -19,6 +19,8 @@ -compile(export_all). -compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). + assert_confs(Expected0, Effected) -> Expected = maybe_unconvert_listeners(Expected0), case do_assert_confs(root, Expected, Effected) of @@ -181,3 +183,17 @@ url(Path, Qs) -> auth(Headers) -> [emqx_mgmt_api_test_util:auth_header_() | Headers]. + +sn_client_connect(ClientId) -> + {ok, Socket} = gen_udp:open(0, [binary]), + _ = emqx_sn_protocol_SUITE:send_connect_msg(Socket, ClientId), + ?assertEqual( + <<3, 16#05, 0>>, + emqx_sn_protocol_SUITE:receive_response(Socket) + ), + Socket. + +sn_client_disconnect(Socket) -> + _ = emqx_sn_protocol_SUITE:send_disconnect_msg(Socket, undefined), + gen_udp:close(Socket), + ok. diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 21066655e..0c0b7310d 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -47,9 +47,6 @@ -include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). - --define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). -record(channel, { %% Context @@ -118,8 +115,8 @@ info(ctx, #channel{ctx = Ctx}) -> Ctx. -spec stats(channel()) -> emqx_types:stats(). -stats(_) -> - []. +stats(#channel{session = Session}) -> + emqx_coap_session:stats(Session). -spec init(map(), map()) -> channel(). init( @@ -166,8 +163,8 @@ init( conn_state = idle }. -validator(Type, Topic, Ctx, ClientInfo) -> - emqx_gateway_ctx:authorize(Ctx, ClientInfo, Type, Topic). +validator(Action, Topic, Ctx, ClientInfo) -> + emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic). -spec send_request(pid(), coap_message()) -> any(). send_request(Channel, Request) -> @@ -273,7 +270,7 @@ handle_call( SubReq, TempMsg, #{}, Session ), NSession = maps:get(session, Result), - {reply, {ok, {MountedTopic, NSubOpts}}, Channel#channel{session = NSession}}; + {reply, {ok, {MountedTopic, NSubOpts}}, [{event, updated}], Channel#channel{session = NSession}}; handle_call( {unsubscribe, Topic}, _From, @@ -300,7 +297,7 @@ handle_call( UnSubReq, TempMsg, #{}, Session ), NSession = maps:get(session, Result), - {reply, ok, Channel#channel{session = NSession}}; + {reply, ok, [{event, updated}], Channel#channel{session = NSession}}; handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> Subs = emqx_coap_session:info(subscriptions, Session), {reply, {ok, maps:to_list(Subs)}, Channel}; @@ -486,7 +483,6 @@ enrich_conninfo( conninfo = ConnInfo } ) -> - %% FIXME: generate a random clientid if absent case Queries of #{<<"clientid">> := ClientId} -> Interval = maps:get(interval, emqx_keepalive:info(KeepAlive)), @@ -500,32 +496,20 @@ enrich_conninfo( }, {ok, Channel#channel{conninfo = NConnInfo}}; _ -> - {error, "invalid queries", Channel} + {error, "clientid is required", Channel} end. enrich_clientinfo( {Queries, Msg}, - Channel = #channel{clientinfo = ClientInfo0} + Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo0} ) -> - %% FIXME: - %% 1. generate a random clientid if absent; - %% 2. assgin username, password to `undefined` if absent - case Queries of - #{ - <<"username">> := UserName, - <<"password">> := Password, - <<"clientid">> := ClientId - } -> - ClientInfo = ClientInfo0#{ - username => UserName, - password => Password, - clientid => ClientId - }, - {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), - {ok, Channel#channel{clientinfo = NClientInfo}}; - _ -> - {error, "invalid queries", Channel} - end. + ClientInfo = ClientInfo0#{ + clientid => maps:get(clientid, ConnInfo), + username => maps:get(<<"username">>, Queries, undefined), + password => maps:get(<<"password">>, Queries, undefined) + }, + {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), + {ok, Channel#channel{clientinfo = NClientInfo}}. set_log_meta(_Input, #channel{clientinfo = #{clientid := ClientId}}) -> emqx_logger:set_metadata_clientid(ClientId), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl index da1f5e0ef..3070ea891 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl @@ -18,6 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include("emqx_coap.hrl"). -export([handle_request/4]). @@ -50,14 +51,16 @@ handle_method(get, Topic, Msg, Ctx, CInfo) -> reply({error, bad_request}, <<"invalid observe value">>, Msg) end; handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx, CInfo) -> - case emqx_coap_channel:validator(publish, Topic, Ctx, CInfo) of + PublishOpts = get_publish_opts(Msg), + Qos = get_publish_qos(Msg, PublishOpts), + Action = ?AUTHZ_PUBLISH(Qos, get_publish_retain(PublishOpts)), + case emqx_coap_channel:validator(Action, Topic, Ctx, CInfo) of allow -> #{clientid := ClientId} = CInfo, MountTopic = mount(CInfo, Topic), - QOS = get_publish_qos(Msg), %% TODO: Append message metadata into headers - MQTTMsg = emqx_message:make(ClientId, QOS, MountTopic, Payload), - MQTTMsg2 = apply_publish_opts(Msg, MQTTMsg), + MQTTMsg = emqx_message:make(ClientId, Qos, MountTopic, Payload), + MQTTMsg2 = apply_publish_opts(PublishOpts, MQTTMsg), _ = emqx_broker:publish(MQTTMsg2), reply({ok, changed}, Msg); _ -> @@ -104,48 +107,70 @@ type_to_qos(coap, #coap_message{type = Type}) -> ?QOS_1 end. -get_publish_qos(Msg) -> - case emqx_coap_message:get_option(uri_query, Msg) of - #{<<"qos">> := QOS} -> - erlang:binary_to_integer(QOS); - _ -> - CfgType = emqx_conf:get([gateway, coap, publish_qos], ?QOS_0), - type_to_qos(CfgType, Msg) - end. - -apply_publish_opts(Msg, MQTTMsg) -> +get_publish_opts(Msg) -> case emqx_coap_message:get_option(uri_query, Msg) of undefined -> - MQTTMsg; + #{}; Qs -> maps:fold( fun (<<"retain">>, V, Acc) -> Val = V =:= <<"true">>, - emqx_message:set_flag(retain, Val, Acc); + Acc#{retain => Val}; (<<"expiry">>, V, Acc) -> Val = erlang:binary_to_integer(V), - Props = emqx_message:get_header(properties, Acc), - emqx_message:set_header( - properties, - Props#{'Message-Expiry-Interval' => Val}, - Acc - ); + Acc#{expiry_interval => Val}; + (<<"qos">>, V, Acc) -> + Val = erlang:binary_to_integer(V), + Acc#{qos => Val}; (_, _, Acc) -> Acc end, - MQTTMsg, + #{}, Qs ) end. +get_publish_qos(Msg, PublishOpts) -> + case PublishOpts of + #{qos := Qos} -> + Qos; + _ -> + CfgType = emqx_conf:get([gateway, coap, publish_qos], ?QOS_0), + type_to_qos(CfgType, Msg) + end. + +get_publish_retain(PublishOpts) -> + maps:get(retain, PublishOpts, false). + +apply_publish_opts(Opts, MQTTMsg) -> + maps:fold( + fun + (retain, Val, Acc) -> + emqx_message:set_flag(retain, Val, Acc); + (expiry, Val, Acc) -> + Props = emqx_message:get_header(properties, Acc), + emqx_message:set_header( + properties, + Props#{'Message-Expiry-Interval' => Val}, + Acc + ); + (_, _, Acc) -> + Acc + end, + MQTTMsg, + Opts + ). + subscribe(#coap_message{token = <<>>} = Msg, _, _, _) -> reply({error, bad_request}, <<"observe without token">>, Msg); subscribe(#coap_message{token = Token} = Msg, Topic, Ctx, CInfo) -> - case emqx_coap_channel:validator(subscribe, Topic, Ctx, CInfo) of + #{qos := Qos} = SubOpts = get_sub_opts(Msg), + Action = ?AUTHZ_SUBSCRIBE(Qos), + case emqx_coap_channel:validator(Action, Topic, Ctx, CInfo) of allow -> #{clientid := ClientId} = CInfo, - SubOpts = get_sub_opts(Msg), + MountTopic = mount(CInfo, Topic), emqx_broker:subscribe(MountTopic, ClientId, SubOpts), run_hooks(Ctx, 'session.subscribed', [CInfo, MountTopic, SubOpts]), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl index 5ae169675..562369e2f 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -117,15 +117,15 @@ info(inflight, _) -> info(inflight_cnt, _) -> 0; info(inflight_max, _) -> - 0; + infinity; info(retry_interval, _) -> infinity; info(mqueue, _) -> emqx_mqueue:init(#{max_len => 0, store_qos0 => false}); -info(mqueue_len, #session{transport_manager = TM}) -> - maps:size(TM); -info(mqueue_max, _) -> +info(mqueue_len, _) -> 0; +info(mqueue_max, _) -> + infinity; info(mqueue_dropped, _) -> 0; info(next_pkt_id, _) -> diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index e03066695..a0cbc3e18 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 999493a79..091978172 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -59,15 +59,14 @@ init_per_suite(Config) -> application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), Config. end_per_suite(_) -> - meck:unload(emqx_access_control), {ok, _} = emqx:remove_config([<<"gateway">>, <<"coap">>]), emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]). init_per_testcase(t_connection_with_authn_failed, Config) -> + ok = meck:new(emqx_access_control, [passthrough]), ok = meck:expect( emqx_access_control, authenticate, @@ -75,12 +74,11 @@ init_per_testcase(t_connection_with_authn_failed, Config) -> ), Config; init_per_testcase(_, Config) -> + ok = meck:new(emqx_access_control, [passthrough]), Config. -end_per_testcase(t_connection_with_authn_failed, Config) -> - ok = meck:delete(emqx_access_control, authenticate, 1), - Config; end_per_testcase(_, Config) -> + ok = meck:unload(emqx_access_control), Config. default_config() -> @@ -133,6 +131,42 @@ t_connection(_) -> end, do(Action). +t_connection_optional_params(_) -> + UsernamePasswordAreOptional = + fun(Channel) -> + URI = + ?MQTT_PREFIX ++ + "/connection?clientid=client1", + Req = make_req(post), + {ok, created, Data} = do_request(Channel, URI, Req), + #coap_content{payload = Token0} = Data, + Token = binary_to_list(Token0), + + timer:sleep(100), + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + disconnection(Channel, Token), + + timer:sleep(100), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ) + end, + ClientIdIsRequired = + fun(Channel) -> + URI = + ?MQTT_PREFIX ++ + "/connection", + Req = make_req(post), + {error, bad_request, _} = do_request(Channel, URI, Req) + end, + do(UsernamePasswordAreOptional), + do(ClientIdIsRequired). + t_connection_with_authn_failed(_) -> ChId = {{127, 0, 0, 1}, 5683}, {ok, Sock} = er_coap_udp_socket:start_link(), @@ -177,6 +211,38 @@ t_publish(_) -> end, with_connection(Topics, Action). +t_publish_with_retain_qos_expiry(_) -> + _ = meck:expect( + emqx_access_control, + authorize, + fun(_, #{action_type := publish, qos := 1, retain := true}, _) -> + allow + end + ), + + Topics = [<<"abc">>], + Action = fun(Topic, Channel, Token) -> + Payload = <<"123">>, + URI = pubsub_uri(binary_to_list(Topic), Token) ++ "&retain=true&qos=1&expiry=60", + + %% Sub topic first + emqx:subscribe(Topic), + + Req = make_req(post, Payload), + {ok, changed, _} = do_request(Channel, URI, Req), + + receive + {deliver, Topic, Msg} -> + ?assertEqual(Topic, Msg#message.topic), + ?assertEqual(Payload, Msg#message.payload) + after 500 -> + ?assert(false) + end + end, + with_connection(Topics, Action), + + _ = meck:validate(emqx_access_control). + t_subscribe(_) -> %% can subscribe to a normal topic Topics = [ @@ -327,6 +393,9 @@ t_clients_subscription_api(_) -> maps:get(topic, SubsResp2) ), + %% check subscription_cnt + {200, #{subscriptions_cnt := 1}} = request(get, "/gateways/coap/clients/client1"), + {204, _} = request(delete, Path ++ "/tx"), {200, []} = request(get, Path) diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 2a144ffeb..80d3282c5 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -19,6 +19,7 @@ -include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -428,7 +429,8 @@ handle_call( clientinfo = ClientInfo } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicFilter) of + Action = ?AUTHZ_SUBSCRIBE(Qos), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicFilter) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> @@ -464,7 +466,8 @@ handle_call( } } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + Action = ?AUTHZ_PUBLISH(Qos), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> @@ -782,7 +785,7 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) -> default_conninfo(ConnInfo) -> ConnInfo#{ clean_start => true, - clientid => anonymous_clientid(), + clientid => emqx_gateway_utils:random_clientid(exproto), username => undefined, conn_props => #{}, connected => true, @@ -822,6 +825,3 @@ proto_name_to_protocol(<<>>) -> exproto; proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). - -anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 66f9ddc89..5959eea3d 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index 3a1e3fc62..db7cd665f 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index 77652744a..e187b3fb7 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -17,7 +17,9 @@ -module(emqx_lwm2m_channel). -include("emqx_lwm2m.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API @@ -111,8 +113,8 @@ info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> info(ctx, #channel{ctx = Ctx}) -> Ctx. -stats(_) -> - []. +stats(#channel{session = Session}) -> + emqx_lwm2m_session:stats(Session). init( ConnInfo = #{ @@ -246,7 +248,7 @@ handle_call( Subs = emqx_lwm2m_session:info(subscriptions, Session), NSubs = maps:put(MountedTopic, NSubOpts, Subs), NSession = emqx_lwm2m_session:set_subscriptions(NSubs, Session), - {reply, {ok, {MountedTopic, NSubOpts}}, Channel#channel{session = NSession}}; + {reply, {ok, {MountedTopic, NSubOpts}}, [{event, updated}], Channel#channel{session = NSession}}; handle_call( {unsubscribe, Topic}, _From, @@ -269,7 +271,7 @@ handle_call( Subs = emqx_lwm2m_session:info(subscriptions, Session), NSubs = maps:remove(MountedTopic, Subs), NSession = emqx_lwm2m_session:set_subscriptions(NSubs, Session), - {reply, ok, Channel#channel{session = NSession}}; + {reply, ok, [{event, updated}], Channel#channel{session = NSession}}; handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> Subs = maps:to_list(emqx_lwm2m_session:info(subscriptions, Session)), {reply, {ok, Subs}, Channel}; @@ -644,7 +646,8 @@ with_context(Ctx, ClientInfo) -> end. with_context(publish, [Topic, Msg], Ctx, ClientInfo) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + Action = publish_action(Msg), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of allow -> _ = emqx_broker:publish(Msg), ok; @@ -660,7 +663,8 @@ with_context(subscribe, [Topic, Opts], Ctx, ClientInfo) -> clientid := ClientId, endpoint_name := EndpointName } = ClientInfo, - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic) of + Action = subscribe_action(Opts), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of allow -> run_hooks(Ctx, 'session.subscribed', [ClientInfo, Topic, Opts]), ?SLOG(debug, #{ @@ -681,6 +685,14 @@ with_context(subscribe, [Topic, Opts], Ctx, ClientInfo) -> with_context(metrics, Name, Ctx, _ClientInfo) -> emqx_gateway_ctx:metrics_inc(Ctx, Name). +publish_action(#message{qos = QoS, flags = Flags}) -> + Retain = maps:get(retain, Flags, false), + ?AUTHZ_PUBLISH(QoS, Retain). + +subscribe_action(Opts) -> + QoS = maps:get(qos, Opts, 0), + ?AUTHZ_SUBSCRIBE(QoS). + %%-------------------------------------------------------------------- %% Call Chain %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index e267692a6..8c37d48e2 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -248,11 +248,11 @@ stats(subscriptions_max, _) -> stats(inflight_cnt, _) -> 0; stats(inflight_max, _) -> - 0; + infinity; stats(mqueue_len, _) -> 0; stats(mqueue_max, _) -> - 0; + infinity; stats(mqueue_dropped, _) -> 0; stats(next_pkt_id, _) -> diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 1779bf842..df1a5d2b3 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -2486,6 +2486,12 @@ case100_subscription_api(Config) -> }, {201, _} = request(post, Path, SubReq), {200, _} = request(get, Path), + + %% check subscription_cnt + {200, #{subscriptions_cnt := 2}} = request( + get, "/gateways/lwm2m/clients/" ++ binary_to_list(ClientId) + ), + {204, _} = request(delete, Path ++ "/tx"), {200, [InitSub]} = request(get, Path). diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index b43201e1a..5e79d4d49 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 720c288d3..2443b149a 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -1099,10 +1100,11 @@ convert_topic_id_to_name( end. check_pub_authz( - {TopicName, _Flags, _Data}, + {TopicName, #mqtt_sn_flags{qos = QoS, retain = Retain}, _Data}, #channel{ctx = Ctx, clientinfo = ClientInfo} ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, TopicName) of + Action = ?AUTHZ_PUBLISH(QoS, Retain), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicName) of allow -> ok; deny -> {error, ?SN_RC2_NOT_AUTHORIZE} end. @@ -1251,10 +1253,11 @@ preproc_subs_type( {error, ?SN_RC_NOT_SUPPORTED}. check_subscribe_authz( - {_TopicId, TopicName, _QoS}, + {_TopicId, TopicName, QoS}, Channel = #channel{ctx = Ctx, clientinfo = ClientInfo} ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicName) of + Action = ?AUTHZ_SUBSCRIBE(QoS), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicName) of allow -> {ok, Channel}; _ -> diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src index 1fda99700..bcc018ad4 100644 --- a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_stomp, [ {description, "Stomp Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index eef30b3dd..3ae928ba3 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -20,6 +20,7 @@ -include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/logger.hrl"). -import(proplists, [get_value/2, get_value/3]). @@ -446,7 +447,10 @@ handle_in( } ) -> Topic = header(<<"destination">>, Headers), - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + %% Flags and QoS are not supported in STOMP anyway, + %% no need to look into the frame + Action = ?AUTHZ_PUBLISH, + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of deny -> ErrMsg = io_lib:format("Insufficient permissions for ~s", [Topic]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), @@ -508,9 +512,13 @@ handle_in( handle_out_and_update(receipt, receipt_id(Headers), NChannel1) end; {error, subscription_id_inused, NChannel} -> - ErrMsg = io_lib:format("Subscription id ~w is in used", [SubId]), + ErrMsg = io_lib:format("Subscription id ~s is in used", [SubId]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), shutdown(subscription_id_inused, ErrorFrame, NChannel); + {error, topic_already_subscribed, NChannel} -> + ErrMsg = io_lib:format("Topic ~s already in subscribed", [Topic]), + ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), + shutdown(topic_already_subscribed, ErrorFrame, NChannel); {error, acl_denied, NChannel} -> ErrMsg = io_lib:format("Insufficient permissions for ~s", [Topic]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), @@ -695,12 +703,15 @@ check_subscribed_status( ) -> MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), case lists:keyfind(SubId, 1, Subs) of - {SubId, MountedTopic, _Ack, _} -> - ok; - {SubId, _OtherTopic, _Ack, _} -> + {SubId, _MountedTopic, _Ack, _} -> {error, subscription_id_inused}; false -> - ok + case lists:keyfind(MountedTopic, 2, Subs) of + {_OtherSubId, MountedTopic, _Ack, _} -> + {error, topic_already_subscribed}; + false -> + ok + end end. check_sub_acl( @@ -710,7 +721,9 @@ check_sub_acl( clientinfo = ClientInfo } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, ParsedTopic) of + %% QoS is not supported in stomp + Action = ?AUTHZ_SUBSCRIBE, + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, ParsedTopic) of deny -> {error, acl_denied}; allow -> ok end. @@ -826,13 +839,20 @@ handle_call( NSubs = [{SubId, MountedTopic, <<"auto">>, NSubOpts} | Subs], NChannel1 = NChannel#channel{subscriptions = NSubs}, reply({ok, {MountedTopic, NSubOpts}}, [{event, updated}], NChannel1); - {error, ErrMsg, NChannel} -> + {error, ErrCode, NChannel} -> ?SLOG(error, #{ msg => "failed_to_subscribe_topic", topic => Topic, - reason => ErrMsg + reason => ErrCode }), - reply({error, ErrMsg}, NChannel) + ErrMsg = + case ErrCode of + subscription_id_inused -> + io_lib:format("Subscription id ~s is in used", [SubId]); + topic_already_subscribed -> + io_lib:format("Topic ~s already in subscribed", [Topic]) + end, + reply({error, lists:flatten(ErrMsg)}, NChannel) end end; handle_call( diff --git a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index 47b191855..d9f0f4ce2 100644 --- a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -162,62 +162,28 @@ t_heartbeat(_) -> t_subscribe(_) -> with_connection(fun(Sock) -> - gen_tcp:send( - Sock, - serialize( - <<"CONNECT">>, - [ - {<<"accept-version">>, ?STOMP_VER}, - {<<"host">>, <<"127.0.0.1:61613">>}, - {<<"login">>, <<"guest">>}, - {<<"passcode">>, <<"guest">>}, - {<<"heart-beat">>, <<"0,0">>} - ] - ) - ), - {ok, Data} = gen_tcp:recv(Sock, 0), - {ok, - #stomp_frame{ - command = <<"CONNECTED">>, - headers = _, - body = _ - }, - _, _} = parse(Data), + ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>), + ?assertMatch({ok, #stomp_frame{command = <<"CONNECTED">>}}, recv_a_frame(Sock)), - %% Subscribe - gen_tcp:send( - Sock, - serialize( - <<"SUBSCRIBE">>, - [ - {<<"id">>, 0}, - {<<"destination">>, <<"/queue/foo">>}, - {<<"ack">>, <<"auto">>} - ] - ) - ), + ok = send_subscribe_frame(Sock, 0, <<"/queue/foo">>), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), %% 'user-defined' header will be retain - gen_tcp:send( - Sock, - serialize( - <<"SEND">>, - [ - {<<"destination">>, <<"/queue/foo">>}, - {<<"user-defined">>, <<"emq">>} - ], - <<"hello">> - ) - ), + ok = send_message_frame(Sock, <<"/queue/foo">>, <<"hello">>, [ + {<<"user-defined">>, <<"emq">>} + ]), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), - {ok, Data1} = gen_tcp:recv(Sock, 0, 1000), - {ok, - Frame = #stomp_frame{ + {ok, Frame} = recv_a_frame(Sock), + + ?assertMatch( + #stomp_frame{ command = <<"MESSAGE">>, headers = _, body = <<"hello">> }, - _, _} = parse(Data1), + Frame + ), lists:foreach( fun({Key, Val}) -> Val = proplists:get_value(Key, Frame#stomp_frame.headers) @@ -234,43 +200,80 @@ t_subscribe(_) -> ?assertMatch(#{subscriptions_cnt := 1}, ClientInfo1), %% Unsubscribe - gen_tcp:send( - Sock, - serialize( - <<"UNSUBSCRIBE">>, - [ - {<<"id">>, 0}, - {<<"receipt">>, <<"12345">>} - ] - ) - ), - - {ok, Data2} = gen_tcp:recv(Sock, 0, 1000), - - {ok, - #stomp_frame{ - command = <<"RECEIPT">>, - headers = [{<<"receipt-id">>, <<"12345">>}], - body = _ - }, - _, _} = parse(Data2), + ok = send_unsubscribe_frame(Sock, 0), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), %% assert subscription stats [ClientInfo2] = clients(), ?assertMatch(#{subscriptions_cnt := 0}, ClientInfo2), - gen_tcp:send( - Sock, - serialize( - <<"SEND">>, - [{<<"destination">>, <<"/queue/foo">>}], - <<"You will not receive this msg">> - ) - ), + ok = send_message_frame(Sock, <<"/queue/foo">>, <<"You will not receive this msg">>), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), {error, timeout} = gen_tcp:recv(Sock, 0, 500) end). +t_subscribe_inuse(_) -> + UsedTopic = <<"/queue/foo">>, + UsedSubId = <<"0">>, + Setup = + fun(Sock) -> + ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>), + ?assertMatch({ok, #stomp_frame{command = <<"CONNECTED">>}}, recv_a_frame(Sock)), + ok = send_subscribe_frame(Sock, UsedSubId, UsedTopic), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)) + end, + TopicIdInuse = + fun(Sock) -> + Setup(Sock), + %% topic-id is in use + ok = send_subscribe_frame(Sock, UsedSubId, <<"/queue/bar">>), + + {ok, ErrorFrame} = recv_a_frame(Sock), + ?assertMatch(#stomp_frame{command = <<"ERROR">>}, ErrorFrame), + ?assertEqual(<<"Subscription id 0 is in used">>, ErrorFrame#stomp_frame.body), + ?assertMatch({error, closed}, gen_tcp:recv(Sock, 0)) + end, + + SubscriptionInuse = + fun(Sock) -> + Setup(Sock), + %% topic is in use + ok = send_subscribe_frame(Sock, 1, UsedTopic), + + {ok, ErrorFrame} = recv_a_frame(Sock), + ?assertMatch(#stomp_frame{command = <<"ERROR">>}, ErrorFrame), + ?assertEqual(<<"Topic /queue/foo already in subscribed">>, ErrorFrame#stomp_frame.body), + ?assertMatch({error, closed}, gen_tcp:recv(Sock, 0)) + end, + + TopicIdInuseViaHttp = + fun(Sock) -> + Setup(Sock), + %% assert subscription stats + [#{clientid := ClientId}] = clients(), + {error, ErrMsg} = create_subscription(ClientId, <<"/queue/bar">>, UsedSubId), + ?assertEqual(<<"Subscription id 0 is in used">>, ErrMsg), + + ok = send_disconnect_frame(Sock) + end, + + SubscriptionInuseViaHttp = + fun(Sock) -> + Setup(Sock), + %% assert subscription stats + [#{clientid := ClientId}] = clients(), + {error, ErrMsg} = create_subscription(ClientId, UsedTopic, <<"1">>), + ?assertEqual(<<"Topic /queue/foo already in subscribed">>, ErrMsg), + + ok = send_disconnect_frame(Sock) + end, + + with_connection(TopicIdInuse), + with_connection(SubscriptionInuse), + with_connection(TopicIdInuseViaHttp), + with_connection(SubscriptionInuseViaHttp). + t_transaction(_) -> with_connection(fun(Sock) -> gen_tcp:send( @@ -1072,6 +1075,7 @@ recv_a_frame(Sock) -> {ok, Frame, Rest, NParser} -> put(parser, NParser), put(rest, Rest), + ct:pal("recv_a_frame: ~p~n", [Frame]), {ok, Frame}; {error, _} = Err -> erase(parser), @@ -1124,11 +1128,23 @@ send_subscribe_frame(Sock, Id, Topic) -> ], ok = gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, Headers)). +send_unsubscribe_frame(Sock, Id) when is_integer(Id) -> + Headers = + [ + {<<"id">>, Id}, + {<<"receipt">>, <<"rp-", (integer_to_binary(Id))/binary>>} + ], + gen_tcp:send(Sock, serialize(<<"UNSUBSCRIBE">>, Headers)). + send_message_frame(Sock, Topic, Payload) -> + send_message_frame(Sock, Topic, Payload, []). + +send_message_frame(Sock, Topic, Payload, Headers0) -> Headers = [ {<<"destination">>, Topic}, {<<"receipt">>, <<"rp-", Topic/binary>>} + | Headers0 ], ok = gen_tcp:send(Sock, serialize(<<"SEND">>, Headers, Payload)). @@ -1142,3 +1158,17 @@ send_disconnect_frame(Sock, ReceiptId) -> clients() -> {200, Clients} = request(get, "/gateways/stomp/clients"), maps:get(data, Clients). + +create_subscription(ClientId, Topic, SubId) -> + Path = io_lib:format("/gateways/stomp/clients/~s/subscriptions", [ClientId]), + Body = #{ + topic => Topic, + qos => 1, + sub_props => #{subid => SubId} + }, + case request(post, Path, Body) of + {201, _} -> + ok; + {400, #{message := Message}} -> + {error, Message} + end. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index b34b577e3..e81d4b53f 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.7"}, + {vsn, "0.2.8"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 1adec9c01..b929f0d72 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -61,6 +61,7 @@ start_autocluster() -> stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), + ok = emqx_conf_app:unset_config_loaded(), lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). %% Those port apps are terminated after the main apps diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 0e2c2646e..d6286f454 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 6991bb11c..f2e336d0f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -122,6 +122,7 @@ schema("/configs") -> }} ] }, + 400 => emqx_dashboard_swagger:error_codes(['INVALID_ACCEPT']), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']), 500 => emqx_dashboard_swagger:error_codes(['BAD_NODE']) } @@ -337,9 +338,10 @@ config_reset(post, _Params, Req) -> configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> %% Should deprecated json v1 since 5.2.0 - case maps:get(<<"accept">>, Headers, <<"text/plain">>) of - <<"application/json">> -> get_configs_v1(QueryStr); - <<"text/plain">> -> get_configs_v2(QueryStr) + case find_suitable_accept(Headers, [<<"text/plain">>, <<"application/json">>]) of + {ok, <<"application/json">>} -> get_configs_v1(QueryStr); + {ok, <<"text/plain">>} -> get_configs_v2(QueryStr); + {error, _} = Error -> {400, #{code => 'INVALID_ACCEPT', message => ?ERR_MSG(Error)}} end; configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> case emqx_conf_cli:load_config(Conf, Mode) of @@ -348,6 +350,28 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} end. +find_suitable_accept(Headers, Perferences) when is_list(Perferences), length(Perferences) > 0 -> + AcceptVal = maps:get(<<"accept">>, Headers, <<"*/*">>), + %% Multiple types, weighted with the quality value syntax: + %% Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8 + Accepts = lists:map( + fun(S) -> + [T | _] = binary:split(string:trim(S), <<";">>), + T + end, + re:split(AcceptVal, ",") + ), + case lists:member(<<"*/*">>, Accepts) of + true -> + {ok, lists:nth(1, Perferences)}; + false -> + Found = lists:filter(fun(Accept) -> lists:member(Accept, Accepts) end, Perferences), + case Found of + [] -> {error, no_suitalbe_accept}; + _ -> {ok, lists:nth(1, Found)} + end + end. + get_configs_v1(QueryStr) -> Node = maps:get(<<"node">>, QueryStr, node()), case diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index dd9013b16..719d0913d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -45,14 +45,10 @@ update/3 ]). --include_lib("emqx/include/emqx.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --define(NODE_LISTENER_NOT_FOUND, <<"Node name or listener id not found">>). --define(NODE_NOT_FOUND_OR_DOWN, <<"Node not found or Down">>). -define(LISTENER_NOT_FOUND, <<"Listener id not found">>). -define(LISTENER_ID_INCONSISTENT, <<"Path and body's listener id not match">>). --define(ADDR_PORT_INUSE, <<"Addr port in use">>). namespace() -> "listeners". @@ -156,7 +152,7 @@ schema("/listeners/:id") -> parameters => [?R_REF(listener_id)], responses => #{ 204 => <<"Listener deleted">>, - 400 => error_codes(['BAD_REQUEST']) + 404 => error_codes(['BAD_LISTENER_ID']) } } }; @@ -405,20 +401,8 @@ list_listeners(get, #{query_string := Query}) -> list_listeners(post, #{body := Body}) -> create_listener(Body). -crud_listeners_by_id(get, #{bindings := #{id := Id0}}) -> - Listeners = - [ - Conf#{ - <<"id">> => Id, - <<"type">> => Type, - <<"bind">> := iolist_to_binary( - emqx_listeners:format_bind(maps:get(<<"bind">>, Conf)) - ) - } - || {Id, Type, Conf} <- emqx_listeners:list_raw(), - Id =:= Id0 - ], - case Listeners of +crud_listeners_by_id(get, #{bindings := #{id := Id}}) -> + case find_listeners_by_id(Id) of [] -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; [L] -> {200, L} end; @@ -449,9 +433,12 @@ crud_listeners_by_id(post, #{body := Body}) -> create_listener(Body); crud_listeners_by_id(delete, #{bindings := #{id := Id}}) -> {ok, #{type := Type, name := Name}} = emqx_listeners:parse_listener_id(Id), - case ensure_remove(Type, Name) of - {ok, _} -> {204}; - {error, Reason} -> {400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}} + case find_listeners_by_id(Id) of + [_L] -> + {ok, _Res} = ensure_remove(Type, Name), + {204}; + [] -> + {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}} end. parse_listener_conf(Conf0) -> @@ -510,9 +497,9 @@ action_listeners_by_id(post, #{bindings := #{id := Id, action := Action}}) -> %%%============================================================================================== -enabled(start) -> #{<<"enabled">> => true}; -enabled(stop) -> #{<<"enabled">> => false}; -enabled(restart) -> #{<<"enabled">> => true}. +enabled(start) -> #{<<"enable">> => true}; +enabled(stop) -> #{<<"enable">> => false}; +enabled(restart) -> #{<<"enable">> => true}. err_msg(Atom) when is_atom(Atom) -> atom_to_binary(Atom); err_msg(Reason) -> list_to_binary(err_msg_str(Reason)). @@ -585,6 +572,19 @@ do_list_listeners() -> <<"listeners">> => Listeners }. +find_listeners_by_id(Id) -> + [ + Conf#{ + <<"id">> => Id0, + <<"type">> => Type, + <<"bind">> := iolist_to_binary( + emqx_listeners:format_bind(maps:get(<<"bind">>, Conf)) + ) + } + || {Id0, Type, Conf} <- emqx_listeners:list_raw(), + Id0 =:= Id + ]. + wrap_rpc({badrpc, Reason}) -> {error, Reason}; wrap_rpc(Res) -> @@ -594,7 +594,7 @@ format_status(Key, Node, Listener, Acc) -> #{ <<"id">> := Id, <<"type">> := Type, - <<"enabled">> := Enabled, + <<"enable">> := Enable, <<"running">> := Running, <<"max_connections">> := MaxConnections, <<"current_connections">> := CurrentConnections, @@ -609,7 +609,7 @@ format_status(Key, Node, Listener, Acc) -> GroupKey => #{ name => Name, type => Type, - enable => Enabled, + enable => Enable, ids => [Id], acceptors => Acceptors, bind => iolist_to_binary(emqx_listeners:format_bind(Bind)), diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index f50e44771..3db0c42fb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -132,6 +132,7 @@ schema("/plugins/:name") -> parameters => [hoconsc:ref(name)], responses => #{ 204 => <<"Uninstall successfully">>, + 400 => emqx_dashboard_swagger:error_codes(['PARAM_ERROR'], <<"Bad parameter">>), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } } @@ -484,6 +485,8 @@ ensure_action(Name, restart) -> return(Code, ok) -> {Code}; +return(_, {error, #{error := "bad_info_file", return := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}; return(_, {error, Reason}) -> {400, #{code => 'PARAM_ERROR', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a53ffc9c4..0e63b38ab 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -206,9 +206,9 @@ cluster_specs() -> {env, [{emqx, boot_modules, all}]}, {apps, []}, {conf, [ - {[listeners, ssl, default, enabled], false}, - {[listeners, ws, default, enabled], false}, - {[listeners, wss, default, enabled], false} + {[listeners, ssl, default, enable], false}, + {[listeners, ws, default, enable], false}, + {[listeners, wss, default, enable], false} ]} ], emqx_common_test_helpers:emqx_cluster( diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 6e3768431..47756cc4c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -89,15 +89,15 @@ t_clients(_) -> AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2), - %% get /clients/:clientid/authorization/cache should has no authz cache + %% get /clients/:clientid/authorization/cache should have no authz cache Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path([ "clients", binary_to_list(ClientId1), "authorization", "cache" ]), - {ok, Client1AuthzCache} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath), - ?assertEqual("[]", Client1AuthzCache), + {ok, Client1AuthzCache0} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath), + ?assertEqual("[]", Client1AuthzCache0), %% post /clients/:clientid/subscribe SubscribeBody = #{topic => Topic, qos => Qos, nl => 1, rh => 1}, @@ -167,6 +167,35 @@ t_clients(_) -> AfterKickoutResponse1 = emqx_mgmt_api_test_util:request_api(get, Client1Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse1). +t_authz_cache(_) -> + ClientId = <<"client_authz">>, + + {ok, C} = emqtt:start_link(#{clientid => ClientId}), + {ok, _} = emqtt:connect(C), + {ok, _, _} = emqtt:subscribe(C, <<"topic/1">>, 0), + + ClientAuthzCachePath = emqx_mgmt_api_test_util:api_path([ + "clients", + binary_to_list(ClientId), + "authorization", + "cache" + ]), + {ok, ClientAuthzCache} = emqx_mgmt_api_test_util:request_api(get, ClientAuthzCachePath), + ?assertMatch( + [ + #{ + <<"access">> := + #{<<"action_type">> := <<"subscribe">>, <<"qos">> := 1}, + <<"result">> := <<"allow">>, + <<"topic">> := <<"topic/1">>, + <<"updated_time">> := _ + } + ], + emqx_utils_json:decode(ClientAuthzCache, [return_maps]) + ), + + ok = emqtt:stop(C). + t_kickout_clients(_) -> process_flag(trap_exit, true), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 0e54a3e22..43554c9ff 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -323,6 +323,36 @@ t_configs_key(_Config) -> ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), ok. +t_get_configs_in_different_accept(_Config) -> + [Key | _] = lists:sort(emqx_conf_cli:keys()), + URI = emqx_mgmt_api_test_util:api_path(["configs?key=" ++ Key]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Request = fun(Accept) -> + Headers = [{"accept", Accept}, Auth], + case + emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) + of + {ok, {{_, Code, _}, RespHeaders, Body}} -> + Type = proplists:get_value("content-type", RespHeaders), + {Code, Type, Body}; + {error, {{_, Code, _}, RespHeaders, Body}} -> + Type = proplists:get_value("content-type", RespHeaders), + {Code, Type, Body} + end + end, + + %% returns text/palin if text/plain is acceptable + ?assertMatch({200, "text/plain", _}, Request(<<"text/plain">>)), + ?assertMatch({200, "text/plain", _}, Request(<<"*/*">>)), + ?assertMatch( + {200, "text/plain", _}, + Request(<<"application/json, application/xml;q=0.9, image/webp, */*;q=0.8">>) + ), + %% returns application/json if it only support it + ?assertMatch({200, "application/json", _}, Request(<<"application/json">>)), + %% returns error if it set to other type + ?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)). + %% Helpers get_config(Name) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 862d81ab8..96ec8f2de 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -399,12 +399,15 @@ crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, Port ?assertEqual([], delete(MinPath)), ?assertEqual({error, not_found}, is_running(NewListenerId)), ?assertMatch({error, {"HTTP/1.1", 404, _}}, request(get, NewPath, [], [])), - ?assertEqual([], delete(NewPath)), + ?assertMatch({error, {"HTTP/1.1", 404, _}}, request(delete, NewPath, [], [])), ok. t_delete_nonexistent_listener(Config) when is_list(Config) -> NonExist = emqx_mgmt_api_test_util:api_path(["listeners", "tcp:nonexistent"]), - ?assertEqual([], delete(NonExist)), + ?assertMatch( + {error, {_, 404, _}}, + request(delete, NonExist, [], []) + ), ok. t_action_listeners(Config) when is_list(Config) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 62fed8211..ba613abc4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -133,6 +133,14 @@ t_bad_plugin(Config) -> ) ). +t_delete_non_existing(_Config) -> + Path = emqx_mgmt_api_test_util:api_path(["plugins", "non_exists"]), + ?assertMatch( + {error, {_, 404, _}}, + emqx_mgmt_api_test_util:request_api(delete, Path) + ), + ok. + list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz index b7da76bbb..a22ff40a5 100644 Binary files a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz and b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz differ diff --git a/apps/emqx_mongodb/rebar.config b/apps/emqx_mongodb/rebar.config index e8a7e281d..cfd7dc9be 100644 --- a/apps/emqx_mongodb/rebar.config +++ b/apps/emqx_mongodb/rebar.config @@ -3,5 +3,5 @@ {erl_opts, [debug_info]}. {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}} + , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.20"}}} ]}. diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 56419e37b..00dcb0cfb 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index 4236517e2..dfa732a7b 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -424,7 +424,7 @@ init_worker_options([{auth_source, V} | R], Acc) -> init_worker_options([{username, V} | R], Acc) -> init_worker_options(R, [{login, V} | Acc]); init_worker_options([{password, V} | R], Acc) -> - init_worker_options(R, [{password, V} | Acc]); + init_worker_options(R, [{password, emqx_secret:wrap(V)} | Acc]); init_worker_options([{w_mode, V} | R], Acc) -> init_worker_options(R, [{w_mode, V} | Acc]); init_worker_options([{r_mode, V} | R], Acc) -> diff --git a/apps/emqx_mysql/rebar.config b/apps/emqx_mysql/rebar.config index 58b6665ad..fc7f4df7a 100644 --- a/apps/emqx_mysql/rebar.config +++ b/apps/emqx_mysql/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ %% NOTE: mind ecpool version when updating eredis_cluster version - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.4"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index c0f8ec7e7..df4846356 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 2a4db3147..c9273f3f1 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -361,7 +361,7 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, undefined_table}; {error, Reason} -> - % FIXME: we should try to differ on transient failers and + % FIXME: we should try to differ on transient failures and % syntax failures. Retrying syntax failures is not very productive. ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, Reason} diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src index c175097e5..edfa6574e 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -1,6 +1,6 @@ {application, emqx_node_rebalance, [ {description, "EMQX Node Rebalance"}, - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {registered, [ emqx_node_rebalance_sup, emqx_node_rebalance, diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index 1d25bfb33..fb27c0a30 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -709,25 +709,30 @@ fields(global_status) -> rebalance_example() -> #{ - wait_health_check => <<"10s">>, - conn_evict_rate => 10, - sess_evict_rate => 20, - abs_conn_threshold => 10, - rel_conn_threshold => 1.5, - abs_sess_threshold => 10, - rel_sess_threshold => 1.5, - wait_takeover => <<"10s">>, - nodes => [<<"othernode@127.0.0.1">>] + rebalance => + #{ + wait_health_check => <<"10s">>, + conn_evict_rate => 10, + sess_evict_rate => 20, + abs_conn_threshold => 10, + rel_conn_threshold => 1.5, + abs_sess_threshold => 10, + rel_sess_threshold => 1.5, + wait_takeover => <<"10s">>, + nodes => [<<"othernode@127.0.0.1">>] + } }. rebalance_evacuation_example() -> #{ - wait_health_check => <<"10s">>, - conn_evict_rate => 100, - sess_evict_rate => 100, - redirect_to => <<"othernode:1883">>, - wait_takeover => <<"10s">>, - migrate_to => [<<"othernode@127.0.0.1">>] + evacuation => #{ + wait_health_check => <<"10s">>, + conn_evict_rate => 100, + sess_evict_rate => 100, + redirect_to => <<"othernode:1883">>, + wait_takeover => <<"10s">>, + migrate_to => [<<"othernode@127.0.0.1">>] + } }. local_status_response_schema() -> diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl index 1d45d64e8..a0102c4f4 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -51,7 +51,7 @@ format_local_status(Status) -> -spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. global_status() -> - Nodes = mria_mnesia:running_nodes(), + Nodes = emqx:running_nodes(), {RebalanceResults, _} = emqx_node_rebalance_status_proto_v1:rebalance_status(Nodes), Rebalances = [ {Node, coordinator_rebalance(Status)} diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl index bb691a754..188e6bf71 100644 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -192,7 +192,7 @@ t_start_stop_evacuation(Config) -> [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), StartOpts = maps:merge( - emqx_node_rebalance_api:rebalance_evacuation_example(), + maps:get(evacuation, emqx_node_rebalance_api:rebalance_evacuation_example()), #{migrate_to => [atom_to_binary(RecipientNode)]} ), @@ -295,7 +295,7 @@ t_start_stop_rebalance(Config) -> StartOpts = maps:without( [nodes], - emqx_node_rebalance_api:rebalance_example() + maps:get(rebalance, emqx_node_rebalance_api:rebalance_example()) ), ?assertMatch( diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl new file mode 100644 index 000000000..167c37d8c --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +suite() -> + [{timetrap, {seconds, 90}}]. + +init_per_suite(Config) -> + WorkDir = ?config(priv_dir, Config), + Apps = [ + emqx_conf, + emqx, + emqx_node_rebalance + ], + Cluster = [ + {emqx_node_rebalance_status_SUITE1, #{ + role => core, + apps => Apps + }}, + {emqx_node_rebalance_status_SUITE2, #{ + role => replicant, + apps => Apps + }} + ], + Nodes = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}), + [{cluster_nodes, Nodes} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)), + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_cluster_status(Config) -> + [CoreNode, ReplicantNode] = ?config(cluster_nodes, Config), + ok = emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_start(CoreNode, #{}), + + ?assertMatch( + #{evacuations := [_], rebalances := []}, + rpc:call(ReplicantNode, emqx_node_rebalance_status, global_status, []) + ). diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index d5c16ea59..368a1ad46 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 04faa44e9..5181000de 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -51,6 +51,11 @@ get_tar/1 ]). +%% `emqx_config_handler' API +-export([ + post_config_update/5 +]). + %% internal -export([do_ensure_started/1]). -export([ @@ -857,3 +862,27 @@ running_apps() -> end, application:which_applications(infinity) ). + +%%-------------------------------------------------------------------- +%% `emqx_config_handler' API +%%-------------------------------------------------------------------- + +post_config_update([?CONF_ROOT], _Req, #{states := NewStates}, #{states := OldStates}, _Envs) -> + NewStatesIndex = maps:from_list([{NV, S} || S = #{name_vsn := NV} <- NewStates]), + OldStatesIndex = maps:from_list([{NV, S} || S = #{name_vsn := NV} <- OldStates]), + #{changed := Changed} = emqx_utils_maps:diff_maps(NewStatesIndex, OldStatesIndex), + maps:foreach(fun enable_disable_plugin/2, Changed), + ok; +post_config_update(_Path, _Req, _NewConf, _OldConf, _Envs) -> + ok. + +enable_disable_plugin(NameVsn, {#{enable := true}, #{enable := false}}) -> + %% errors are already logged in this fn + _ = ensure_stopped(NameVsn), + ok; +enable_disable_plugin(NameVsn, {#{enable := false}, #{enable := true}}) -> + %% errors are already logged in this fn + _ = ensure_started(NameVsn), + ok; +enable_disable_plugin(_NameVsn, _Diff) -> + ok. diff --git a/apps/emqx_plugins/src/emqx_plugins_app.erl b/apps/emqx_plugins/src/emqx_plugins_app.erl index c42936d56..f75089144 100644 --- a/apps/emqx_plugins/src/emqx_plugins_app.erl +++ b/apps/emqx_plugins/src/emqx_plugins_app.erl @@ -18,6 +18,8 @@ -behaviour(application). +-include("emqx_plugins.hrl"). + -export([ start/2, stop/1 @@ -27,7 +29,9 @@ start(_Type, _Args) -> %% load all pre-configured ok = emqx_plugins:ensure_started(), {ok, Sup} = emqx_plugins_sup:start_link(), + ok = emqx_config_handler:add_handler([?CONF_ROOT], emqx_plugins), {ok, Sup}. stop(_State) -> + ok = emqx_config_handler:remove_handler([?CONF_ROOT]), ok. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index d6dee2c1e..9bb3f5e72 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -65,7 +65,7 @@ init_per_suite(Config) -> WorkDir = proplists:get_value(data_dir, Config), filelib:ensure_path(WorkDir), OrigInstallDir = emqx_plugins:get_config(install_dir, undefined), - emqx_common_test_helpers:start_apps([emqx_conf]), + emqx_common_test_helpers:start_apps([emqx_conf, emqx_plugins]), emqx_plugins:put_config(install_dir, WorkDir), [{orig_install_dir, OrigInstallDir} | Config]. @@ -77,7 +77,7 @@ end_per_suite(Config) -> undefined -> ok; OrigInstallDir -> emqx_plugins:put_config(install_dir, OrigInstallDir) end, - emqx_common_test_helpers:stop_apps([emqx_conf]), + emqx_common_test_helpers:stop_apps([emqx_plugins, emqx_conf]), ok. init_per_testcase(TestCase, Config) -> @@ -505,6 +505,65 @@ t_elixir_plugin(Config) -> ?assertEqual([], emqx_plugins:list()), ok. +t_load_config_from_cli({init, Config}) -> + #{package := Package} = get_demo_plugin_package(), + NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), + [{name_vsn, NameVsn} | Config]; +t_load_config_from_cli({'end', Config}) -> + NameVsn = ?config(name_vsn, Config), + ok = emqx_plugins:ensure_stopped(NameVsn), + ok = emqx_plugins:ensure_uninstalled(NameVsn), + ok; +t_load_config_from_cli(Config) when is_list(Config) -> + NameVsn = ?config(name_vsn, Config), + ok = emqx_plugins:ensure_installed(NameVsn), + ?assertEqual([], emqx_plugins:configured()), + ok = emqx_plugins:ensure_enabled(NameVsn), + ok = emqx_plugins:ensure_started(NameVsn), + Params0 = unused, + ?assertMatch( + {200, [#{running_status := [#{status := running}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + %% Now we disable it via CLI loading + Conf0 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := true}]}, + Conf0 + ), + #{states := [Plugin0]} = Conf0, + Conf1 = Conf0#{states := [Plugin0#{enable := false}]}, + Filename = filename:join(["/tmp", [?FUNCTION_NAME, ".hocon"]]), + ok = file:write_file(Filename, hocon_pp:do(#{plugins => Conf1}, #{})), + ok = emqx_conf_cli:conf(["load", Filename]), + + Conf2 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := false}]}, + Conf2 + ), + ?assertMatch( + {200, [#{running_status := [#{status := stopped}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + %% Re-enable it via CLI loading + ok = file:write_file(Filename, hocon_pp:do(#{plugins => Conf0}, #{})), + ok = emqx_conf_cli:conf(["load", Filename]), + + Conf3 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := true}]}, + Conf3 + ), + ?assertMatch( + {200, [#{running_status := [#{status := running}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + ok. + group_t_copy_plugin_to_a_new_node({init, Config}) -> WorkDir = proplists:get_value(data_dir, Config), FromInstallDir = filename:join(WorkDir, atom_to_list(plugins_copy_from)), diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 7252e4436..e6ee145ff 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, @@ -11,7 +11,6 @@ {licenses, ["Apache-2.0"]}, {maintainers, ["EMQX Team "]}, {links, [ - {"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-prometheus"} + {"Homepage", "https://emqx.io/"} ]} ]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index c2bfaefc8..987386b61 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -130,10 +130,8 @@ prometheus_data_schema() -> description => <<"Get Prometheus Data. Note that support for JSON output is deprecated and will be removed in v5.2.">>, content => - #{ - 'application/json' => - #{schema => #{type => object}}, - 'text/plain' => - #{schema => #{type => string}} - } + [ + {'text/plain', #{schema => #{type => string}}}, + {'application/json', #{schema => #{type => object}}} + ] }. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 3300e8b28..3884f7065 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -59,7 +59,7 @@ fields("prometheus") -> ?HOCON( list({string(), string()}), #{ - default => [], + default => #{}, required => false, converter => fun ?MODULE:convert_headers/1, desc => ?DESC(headers) diff --git a/apps/emqx_psk/include/emqx_psk.hrl b/apps/emqx_psk/include/emqx_psk.hrl index 700160a5f..ece0d500f 100644 --- a/apps/emqx_psk/include/emqx_psk.hrl +++ b/apps/emqx_psk/include/emqx_psk.hrl @@ -17,3 +17,7 @@ -define(TAB, emqx_psk). -define(PSK_SHARD, emqx_psk_shard). + +-define(PSK_KEY, psk_authentication). + +-define(DEFAULT_DELIMITER, <<":">>). diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index 26885673c..be24112e4 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl index 6b2199832..7a0986fe7 100644 --- a/apps/emqx_psk/src/emqx_psk.erl +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -27,7 +27,8 @@ load/0, unload/0, on_psk_lookup/2, - import/1 + import/1, + post_config_update/5 ]). -export([ @@ -68,13 +69,11 @@ -include("emqx_psk.hrl"). --define(DEFAULT_DELIMITER, <<":">>). - -define(CR, 13). -define(LF, 10). -ifdef(TEST). --export([call/1, trim_crlf/1]). +-export([call/1, trim_crlf/1, import_psks/3]). -endif. %%------------------------------------------------------------------------------ @@ -135,10 +134,6 @@ stop() -> import_config(#{<<"psk_authentication">> := PskConf}) -> case emqx_conf:update([psk_authentication], PskConf, #{override_to => cluster}) of {ok, _} -> - case get_config(enable) of - true -> load(); - false -> ok - end, {ok, #{root_key => psk_authentication, changed => []}}; Error -> {error, #{root_key => psk_authentication, reason => Error}} @@ -146,6 +141,16 @@ import_config(#{<<"psk_authentication">> := PskConf}) -> import_config(_RawConf) -> {ok, #{root_key => psk_authentication, changed => []}}. +post_config_update([?PSK_KEY], _Req, #{enable := Enable} = NewConf, _OldConf, _AppEnvs) -> + case Enable of + true -> + load(), + _ = gen_server:cast(?MODULE, {import_from_conf, NewConf}); + false -> + unload() + end, + ok. + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ @@ -169,6 +174,15 @@ handle_call(Req, _From, State) -> ?SLOG(info, #{msg => "unexpected_call_discarded", req => Req}), {reply, {error, unexpected}, State}. +handle_cast({import_from_conf, Conf}, State) -> + Separator = maps:get(separator, Conf, ?DEFAULT_DELIMITER), + ChunkSize = maps:get(chunk_size, Conf), + _ = + case maps:get(init_file, Conf, undefined) of + undefined -> ok; + InitFile -> import_psks(InitFile, Separator, ChunkSize) + end, + {noreply, State}; handle_cast(Req, State) -> ?SLOG(info, #{msg => "unexpected_cast_discarded", req => Req}), {noreply, State}. @@ -198,6 +212,11 @@ get_config(chunk_size) -> emqx_conf:get([psk_authentication, chunk_size]). import_psks(SrcFile) -> + Separator = get_config(separator), + ChunkSize = get_config(chunk_size), + import_psks(SrcFile, Separator, ChunkSize). + +import_psks(SrcFile, Separator, ChunkSize) -> case file:open(SrcFile, [read, raw, binary, read_ahead]) of {error, Reason} -> ?SLOG(error, #{ @@ -207,7 +226,7 @@ import_psks(SrcFile) -> }), {error, Reason}; {ok, Io} -> - try import_psks(Io, get_config(separator), get_config(chunk_size), 0) of + try import_psks(Io, Separator, ChunkSize, 0) of ok -> ok; {error, Reason} -> diff --git a/apps/emqx_psk/src/emqx_psk_app.erl b/apps/emqx_psk/src/emqx_psk_app.erl index f1a7cf18c..d4735f4c9 100644 --- a/apps/emqx_psk/src/emqx_psk_app.erl +++ b/apps/emqx_psk/src/emqx_psk_app.erl @@ -27,6 +27,7 @@ start(_Type, _Args) -> ok = mria:wait_for_tables([?TAB]), + emqx_conf:add_handler([?PSK_KEY], emqx_psk), {ok, Sup} = emqx_psk_sup:start_link(), {ok, Sup}. diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 45a1a077e..e6c922c1e 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_psk.hrl"). -export([ namespace/0, @@ -52,7 +53,7 @@ fields() -> })}, {separator, ?HOCON(binary(), #{ - default => <<":">>, + default => ?DEFAULT_DELIMITER, desc => ?DESC(separator) })}, {chunk_size, diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index 00702efa0..2a28ceb2c 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -20,6 +20,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include("emqx_psk.hrl"). -define(CR, 13). -define(LF, 10). @@ -124,7 +125,15 @@ t_load_unload(_) -> t_import(_) -> Init = emqx_conf:get([psk_authentication, init_file], undefined), + Separator = emqx_conf:get([psk_authentication, separator], ?DEFAULT_DELIMITER), + ChunkSize = emqx_conf:get([psk_authentication, chunk_size], 50), ?assertEqual(ok, emqx_psk:import(Init)), + Keys0 = lists:sort(mnesia:dirty_all_keys(emqx_psk)), + ?assert(length(Keys0) > 0), + {atomic, ok} = mnesia:clear_table(emqx_psk), + ok = emqx_psk:import_psks(Init, Separator, ChunkSize), + Keys1 = lists:sort(mnesia:dirty_all_keys(emqx_psk)), + ?assertEqual(Keys0, Keys1), ?assertMatch({error, _}, emqx_psk:import("~/_none_")), ok. diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 41e5673d9..10dc001c2 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -114,7 +114,6 @@ %% boolean -define(START_AFTER_CREATED, true). --define(START_AFTER_CREATED_RAW, <<"true">>). -define(TEST_ID_PREFIX, "_probe_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 57ab8129a..19b6ca5e2 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 59687eb8d..b98f50a98 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -129,7 +129,7 @@ health_check_interval_range(HealthCheckInterval) -> start_after_created(type) -> boolean(); start_after_created(desc) -> ?DESC("start_after_created"); -start_after_created(default) -> ?START_AFTER_CREATED_RAW; +start_after_created(default) -> ?START_AFTER_CREATED; start_after_created(required) -> false; start_after_created(_) -> undefined. diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index d13359509..f117fda05 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 7b1337140..3274f0e4c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -102,6 +102,7 @@ schema(?PREFIX ++ "/message/:topic") -> parameters => parameters(), responses => #{ 204 => <<>>, + 404 => error_codes(['NOT_FOUND'], ?DESC(message_not_exist)), 400 => error_codes( ['BAD_REQUEST'], ?DESC(unsupported_backend) @@ -187,8 +188,16 @@ with_topic(get, #{bindings := Bindings}) -> end; with_topic(delete, #{bindings := Bindings}) -> Topic = maps:get(topic, Bindings), - emqx_retainer_mnesia:delete_message(undefined, Topic), - {204}. + case emqx_retainer_mnesia:page_read(undefined, Topic, 1, 1) of + {ok, []} -> + {404, #{ + code => <<"NOT_FOUND">>, + message => <<"Viewed message doesn't exist">> + }}; + {ok, _} -> + emqx_retainer_mnesia:delete_message(undefined, Topic), + {204} + end. format_message(#message{ id = ID, diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 61eee0510..d00ade556 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -218,6 +218,7 @@ t_lookup_and_delete(_) -> {ok, []} = request_api(delete, API), {error, {"HTTP/1.1", 404, "Not Found"}} = request_api(get, API), + {error, {"HTTP/1.1", 404, "Not Found"}} = request_api(delete, API), ok = emqtt:disconnect(C1). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index ccd436d86..f0388631f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 7f14f6d8b..3ff588f48 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx_bridge/include/emqx_bridge_resource.hrl"). -export([ @@ -160,7 +161,10 @@ on_client_connack(ConnInfo, Reason, _, Conf) -> Conf ). -on_client_check_authz_complete(ClientInfo, PubSub, Topic, Result, AuthzSource, Conf) -> +%% TODO: support full action in major release +on_client_check_authz_complete( + ClientInfo, ?authz_action(PubSub), Topic, Result, AuthzSource, Conf +) -> apply_event( 'client.check_authz_complete', fun() -> diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 0599d7923..6dee7ed0a 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index cc48cdb93..be91a19d2 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -14,6 +14,11 @@ with_client/2 ]). +-export([ + pre_config_update/3, + post_config_update/3 +]). + -export_type([ profile_id/0, profile_config/0, @@ -94,3 +99,31 @@ with_client(ProfileId, Fun) when is_function(Fun, 1) andalso ?IS_PROFILE_ID(Prof {error, _} = Error -> Error end. + +%% + +-spec pre_config_update( + profile_id(), maybe(emqx_config:raw_config()), maybe(emqx_config:raw_config()) +) -> + {ok, maybe(profile_config())} | {error, term()}. +pre_config_update(ProfileId, NewConfig = #{<<"transport_options">> := TransportOpts}, _OldConfig) -> + case emqx_connector_ssl:convert_certs(mk_certs_dir(ProfileId), TransportOpts) of + {ok, TransportOptsConv} -> + {ok, NewConfig#{<<"transport_options">> := TransportOptsConv}}; + {error, Reason} -> + {error, Reason} + end; +pre_config_update(_ProfileId, NewConfig, _OldConfig) -> + {ok, NewConfig}. + +-spec post_config_update( + profile_id(), + maybe(emqx_config:config()), + maybe(emqx_config:config()) +) -> + ok. +post_config_update(_ProfileId, _NewConfig, _OldConfig) -> + ok. + +mk_certs_dir(ProfileId) -> + filename:join([s3, profiles, ProfileId]). diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index c2460e20d..5fa57c230 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -14,6 +14,9 @@ -export([translate/1]). -export([translate/2]). +-type secret_access_key() :: string() | function(). +-reflect_type([secret_access_key/0]). + roots() -> [s3]. @@ -34,7 +37,7 @@ fields(s3) -> )}, {secret_access_key, mk( - hoconsc:union([string(), function()]), + secret_access_key(), #{ desc => ?DESC("secret_access_key"), required => false, diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index df7d67321..5900514dc 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 86667063c..80a9f8754 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -60,7 +60,8 @@ safe_filename/1, diff_lists/3, merge_lists/3, - tcp_keepalive_opts/4 + tcp_keepalive_opts/4, + format/1 ]). -export([ @@ -525,6 +526,9 @@ tcp_keepalive_opts({unix, darwin}, Idle, Interval, Probes) -> tcp_keepalive_opts(OS, _Idle, _Interval, _Probes) -> {error, {unsupported_os, OS}}. +format(Term) -> + iolist_to_binary(io_lib:format("~0p", [Term])). + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ @@ -606,7 +610,7 @@ to_hr_error({not_authorized, _}) -> to_hr_error({malformed_username_or_password, _}) -> <<"Bad username or password">>; to_hr_error(Error) -> - iolist_to_binary(io_lib:format("~0p", [Error])). + format(Error). try_to_existing_atom(Convert, Data, Encoding) -> try Convert(Data, Encoding) of diff --git a/changes/ce/feat-11226.en.md b/changes/ce/feat-11226.en.md new file mode 100644 index 000000000..0c43886bc --- /dev/null +++ b/changes/ce/feat-11226.en.md @@ -0,0 +1 @@ +Unify the listener switch to `enable`, while being compatible with the previous `enabled`. diff --git a/changes/ce/fix-11077.en.md b/changes/ce/fix-11077.en.md new file mode 100644 index 000000000..64911df4a --- /dev/null +++ b/changes/ce/fix-11077.en.md @@ -0,0 +1 @@ +Fixes crash when updating binding with a non-integer port. diff --git a/changes/ce/fix-11184.en.md b/changes/ce/fix-11184.en.md new file mode 100644 index 000000000..46a790a72 --- /dev/null +++ b/changes/ce/fix-11184.en.md @@ -0,0 +1 @@ +Config value for `max_packet_size` has a max value of 256MB defined by protocol. This is now enforced and any configuration with a value greater than that will break. diff --git a/changes/ce/fix-11192.en.md b/changes/ce/fix-11192.en.md new file mode 100644 index 000000000..6bbc0b12d --- /dev/null +++ b/changes/ce/fix-11192.en.md @@ -0,0 +1,2 @@ +Fix produces valid HOCON file when atom type is used. +Remove unnecessary `"` from HOCON file. diff --git a/changes/ce/fix-11195.en.md b/changes/ce/fix-11195.en.md new file mode 100644 index 000000000..4c2d8b6b7 --- /dev/null +++ b/changes/ce/fix-11195.en.md @@ -0,0 +1 @@ +Avoid to create duplicated subscription by HTTP API or client in Stomp gateway diff --git a/changes/ce/fix-11206.en.md b/changes/ce/fix-11206.en.md new file mode 100644 index 000000000..e16b1e3f8 --- /dev/null +++ b/changes/ce/fix-11206.en.md @@ -0,0 +1 @@ +Make the username and password params of CoAP client to optional in connection mode. diff --git a/changes/ce/fix-11208.en.md b/changes/ce/fix-11208.en.md new file mode 100644 index 000000000..56d5a398a --- /dev/null +++ b/changes/ce/fix-11208.en.md @@ -0,0 +1 @@ +Fix the issue of abnormal data statistics for LwM2M client. diff --git a/changes/ce/fix-11211.en.md b/changes/ce/fix-11211.en.md new file mode 100644 index 000000000..0b69fefca --- /dev/null +++ b/changes/ce/fix-11211.en.md @@ -0,0 +1 @@ +Consistently return `404` for `DELETE` operations on non-existent resources. diff --git a/changes/ce/fix-11214.en.md b/changes/ce/fix-11214.en.md new file mode 100644 index 000000000..35a33970a --- /dev/null +++ b/changes/ce/fix-11214.en.md @@ -0,0 +1 @@ +Fix a bug where node configuration may fail to synchronize correctly when joining the cluster. diff --git a/changes/ce/fix-11229.en.md b/changes/ce/fix-11229.en.md new file mode 100644 index 000000000..864f545fe --- /dev/null +++ b/changes/ce/fix-11229.en.md @@ -0,0 +1 @@ +Fixed an issue preventing plugins from starting/stopping after changing configuration via `emqx ctl conf load`. diff --git a/changes/ce/fix-11237.en.md b/changes/ce/fix-11237.en.md new file mode 100644 index 000000000..d6220040e --- /dev/null +++ b/changes/ce/fix-11237.en.md @@ -0,0 +1 @@ +The `headers` default value in /prometheus API should be a map instead of a list. diff --git a/changes/ee/feat-10203.en.md b/changes/ee/feat-10203.en.md new file mode 100644 index 000000000..a2ff3b3bb --- /dev/null +++ b/changes/ee/feat-10203.en.md @@ -0,0 +1 @@ +Add HStreamDB bridge support, adapted to the HStreamDB `v0.15.0`. diff --git a/changes/ee/feat-11132.en.md b/changes/ee/feat-11132.en.md new file mode 100644 index 000000000..6ebc7efe2 --- /dev/null +++ b/changes/ee/feat-11132.en.md @@ -0,0 +1,2 @@ +Add support for MQTT action authorization based on QoS level and Retain flag values. +Now, EMQX can check by ACL whether a client has permission to publish/subscribe using a specified QoS level and to use retained messages. diff --git a/changes/ee/feat-11207.en.md b/changes/ee/feat-11207.en.md new file mode 100644 index 000000000..d1d4c1812 --- /dev/null +++ b/changes/ee/feat-11207.en.md @@ -0,0 +1,6 @@ +Update the dependent versions of multiple data bridges to enhance security and ensure that sensitive data will not be leaked. +Including: + - TDEngine + - MongoDB + - MySQL + - Clickhouse diff --git a/changes/ee/fix-11175.en.md b/changes/ee/fix-11175.en.md new file mode 100644 index 000000000..24a9def70 --- /dev/null +++ b/changes/ee/fix-11175.en.md @@ -0,0 +1 @@ +Now when using a nonexistent hostname for connecting to MySQL will result in a 400 error rather than 503 in the HTTP API. diff --git a/changes/ee/fix-11198.en.md b/changes/ee/fix-11198.en.md new file mode 100644 index 000000000..60173dc73 --- /dev/null +++ b/changes/ee/fix-11198.en.md @@ -0,0 +1,2 @@ +Fix global rebalance status evaluation on replicant nodes. +Previously, `/api/v5/load_rebalance/global_status` API method could return incomplete results if handled by a replicant node. diff --git a/changes/ee/fix-11223.en.md b/changes/ee/fix-11223.en.md new file mode 100644 index 000000000..6d97746be --- /dev/null +++ b/changes/ee/fix-11223.en.md @@ -0,0 +1,5 @@ +In InfluxDB bridging, if intend to write using the float data type but the placeholder represents the original value +as an integer without a decimal point during serialization, it will result in the failure of Influx Line Protocol serialization +and the inability to write to the InfluxDB bridge. + +See also: [InfluxDB v2.7 Line-Protocol](https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float) diff --git a/changes/ee/fix-11225.en.md b/changes/ee/fix-11225.en.md new file mode 100644 index 000000000..8d7ad554f --- /dev/null +++ b/changes/ee/fix-11225.en.md @@ -0,0 +1 @@ +Fix the `username` of PostgreSQL/Timescale/MatrixDB bridges could be empty diff --git a/changes/v5.1.1.en.md b/changes/v5.1.1.en.md new file mode 100644 index 000000000..521a96af8 --- /dev/null +++ b/changes/v5.1.1.en.md @@ -0,0 +1,104 @@ +# v5.1.1 + +## Enhancements + +- [#10667](https://github.com/emqx/emqx/pull/10667) The MongoDB connector and bridge have been refactored to a separate app to improve code structure. + +- [#11115](https://github.com/emqx/emqx/pull/11115) Added info logs to indicate when buffered messages are dropped due to time-to-live (TTL) expiration. + +- [#11133](https://github.com/emqx/emqx/pull/11133) Rename `deliver_rate` to `delivery_rate` in the configuration of `retainer`. + +- [#11137](https://github.com/emqx/emqx/pull/11137) Refactors the dashboard listener configuration to use a nested `ssl_options` field for ssl settings. + +- [#11138](https://github.com/emqx/emqx/pull/11138) - Change k8s `api_server` default value from `http://127.0.0.1:9091` to `https://kubernetes.default.svc:443` + - `emqx_ctl conf show cluster` no longer displays irrelevant configuration items, such as when `discovery_strategy=static`, + it will not display configuration information related to `etcd/k8s/dns`. + - Remove `zones`(deprecated config key) from `emqx_ctl conf show_keys` + +- [#11165](https://github.com/emqx/emqx/pull/11165) Remove `/configs/limiter` api from `swagger.json`, only the api documentation was removed, + and the `/configs/limiter` api functionalities have not been changed. + +- [#11166](https://github.com/emqx/emqx/pull/11166) Added 3 random SQL functions to the rule engine. + - random(): Generates a random number between 0 and 1 (0.0 =< X < 1.0). + - uuid_v4(): Generates a random UUID (version 4) string. + - uuid_v4_no_hyphen(): Generates a random UUID (version 4) string without hyphens. + +- [#11180](https://github.com/emqx/emqx/pull/11180) Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file. + +- [#11020](https://github.com/emqx/emqx/pull/11020) Upgraded emqtt dependency to avoid sensitive data leakage in the debug log. + +- [#11135](https://github.com/emqx/emqx/pull/11135) Improve time offset parser in rules engine and return uniform error codes. + +## Bug Fixes + +- [#11004](https://github.com/emqx/emqx/pull/11004) Do not allow wildcards for destination topic in rewrite rules. + +- [#11026](https://github.com/emqx/emqx/pull/11026) Addressed an inconsistency in the usage of 'div' and 'mod' operations within the rule engine. Previously, the 'div' operation was only usable as an infix operation and 'mod' could only be applied through a function call. With this change, both 'div' and 'mod' can be used via function call syntax and infix syntax. + +- [#11037](https://github.com/emqx/emqx/pull/11037) When starting an HTTP connector EMQX now returns a descriptive error in case the system is unable to connect to the remote target system. + +- [#11039](https://github.com/emqx/emqx/pull/11039) Fixed database number validation for Redis connector. Previously negative numbers were accepted as valid database numbers. + +- [#11074](https://github.com/emqx/emqx/pull/11074) Fix to adhere to Protocol spec MQTT-5.0 [MQTT-3.8.3-4]. + +- [#11094](https://github.com/emqx/emqx/pull/11094) Fixed an issue where connection errors in Kafka Producer would not be reported when reconnecting the bridge. + +- [#11103](https://github.com/emqx/emqx/pull/11103) Updated `erlcloud` dependency. + +- [#11106](https://github.com/emqx/emqx/pull/11106) Added a validation for the maximum number of pool workers of a bridge. + + Now the maximum amount is 1024 to avoid large memory consumption from an unreasonable number of workers. + +- [#11118](https://github.com/emqx/emqx/pull/11118) Ensure that validation errors in REST API responses are slightly less confusing. Now, if there are out-of-range errors, they will be presented as `{"value": 42, "reason": {"expected": "1..10"}, ...}`, replacing the previous usage of `expected_type` with `expected`. + +- [#11126](https://github.com/emqx/emqx/pull/11126) Rule metrics for async mode bridges will set failure counters correctly now. + +- [#11134](https://github.com/emqx/emqx/pull/11134) Fix the value of the uppercase `authorization` header is not obfuscated. + +- [#11139](https://github.com/emqx/emqx/pull/11139) The Redis connector has been refactored to its own Erlang application to improve the code structure. + +- [#11145](https://github.com/emqx/emqx/pull/11145) Add several fixes and improvements in Ekka and Mria. + + Ekka: + - improve cluster discovery log messages to consistently describe actual events + [Ekka PR](https://github.com/emqx/ekka/pull/204) + - remove deprecated cluster auto-clean configuration parameter (it has been moved to Mria) + [Ekka PR](https://github.com/emqx/ekka/pull/203) + + Mria: + - ping only running replicant nodes. Previously, `mria_lb` was trying to ping both stopped and running + replicant nodes, which might result in timeout errors. + [Mria PR](https://github.com/emqx/mria/pull/146) + - use `null_copies` storage when copying `$mria_rlog_sync` table. + This fix has no effect on EMQX for now, as `$mria_rlog_sync` is only used in `mria:sync_transaction/2,3,4`, + which is not utilized by EMQX. + [Mria PR](https://github.com/emqx/mria/pull/144) + +- [#11148](https://github.com/emqx/emqx/pull/11148) Fix when a node has left the cluster, other nodes still try to synchronize configuration update operations to it. + +- [#11150](https://github.com/emqx/emqx/pull/11150) Wait for Mria table when emqx_psk app is being started to ensure that + PSK data is synced to replicant nodes even if they don't have init PSK file. + +- [#11151](https://github.com/emqx/emqx/pull/11151) The MySQL connector has been refactored to its own Erlang application to improve the code structure. + +- [#11158](https://github.com/emqx/emqx/pull/11158) Wait for Mria table when the mnesia backend of retainer starts to avoid a possible error of the retainer when joining a cluster. + +- [#11162](https://github.com/emqx/emqx/pull/11162) Fixed an issue in webhook bridge where, in async query mode, HTTP status codes like 4XX and 5XX would be treated as successes in the bridge metrics. + +- [#11164](https://github.com/emqx/emqx/pull/11164) Reintroduced support for nested (i.e.: `${payload.a.b.c}`) placeholders for extracting data from rule action messages without the need for calling `json_decode(payload)` first. + +- [#11172](https://github.com/emqx/emqx/pull/11172) Fix the `payload` will be duplicated in the below situations: + - Use a `foreach` sentence without the `as` sub-expression and select all fields(use the `*` or omitted the `do` sub-expression) + + For example: + + `FOREACH payload.sensors FROM "t/#"` + - Select the `payload` field and all fields + + For example: + + `SELECT payload.sensors, * FROM "t/#"` + +- [#11174](https://github.com/emqx/emqx/pull/11174) Fixed the encoding of the `server` key coming from an ingress MQTT bridge. + + Before the fix, it was being encoded as a list of integers corresponding to the ASCII characters of the server string. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 821c19e8c..626436517 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.1.0 +version: 5.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.1.0 +appVersion: 5.1.1 diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index df3be6766..b11159c84 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -100,6 +100,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | | `ssl.existingName` | Name of existing certificate | emqx-tls | | `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | +| `ssl.commonName` | Common name for or certificate to be generated | | | `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | | `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | diff --git a/deploy/charts/emqx-enterprise/templates/certificate.yaml b/deploy/charts/emqx-enterprise/templates/certificate.yaml index 9a2ed969a..528f989a1 100644 --- a/deploy/charts/emqx-enterprise/templates/certificate.yaml +++ b/deploy/charts/emqx-enterprise/templates/certificate.yaml @@ -9,6 +9,9 @@ spec: issuerRef: name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + {{- if .Values.ssl.commonName }} + commonName: {{ .Values.ssl.commonName }} + {{- end }} dnsNames: {{- range .Values.ssl.dnsnames }} - {{ . }} diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 71569b9a3..412462854 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -237,6 +237,7 @@ ssl: useExisting: false existingName: emqx-tls dnsnames: [] + commonName: issuer: name: letsencrypt-dns kind: ClusterIssuer diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 045001b79..a2262da8b 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.1.0 +version: 5.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.1.0 +appVersion: 5.1.1 diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 47ae89245..7c6ef122f 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -99,6 +99,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `ssl.enabled` | Enable SSL support | false | | `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | | `ssl.existingName` | Name of existing certificate | emqx-tls | +| `ssl.commonName` | Common name for or certificate to be generated | | | `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | | `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | | `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 3e9e39f2c..33efe70c7 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -107,14 +107,14 @@ spec: - name: wss containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 @@ -147,14 +147,14 @@ spec: readinessProbe: httpGet: path: /status - port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 30 livenessProbe: httpGet: path: /status - port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} initialDelaySeconds: 60 periodSeconds: 30 failureThreshold: 10 diff --git a/deploy/charts/emqx/templates/certificate.yaml b/deploy/charts/emqx/templates/certificate.yaml index 9a2ed969a..528f989a1 100644 --- a/deploy/charts/emqx/templates/certificate.yaml +++ b/deploy/charts/emqx/templates/certificate.yaml @@ -9,6 +9,9 @@ spec: issuerRef: name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + {{- if .Values.ssl.commonName }} + commonName: {{ .Values.ssl.commonName }} + {{- end }} dnsNames: {{- range .Values.ssl.dnsnames }} - {{ . }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index f7c6483fe..86fa44880 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -240,6 +240,7 @@ ssl: useExisting: false existingName: emqx-tls dnsnames: [] + commonName: issuer: name: letsencrypt-dns kind: ClusterIssuer diff --git a/lib-ee/emqx_ee_bridge/.gitignore b/lib-ee/emqx_ee_bridge/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/lib-ee/emqx_ee_bridge/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/lib-ee/emqx_ee_bridge/README.md b/lib-ee/emqx_ee_bridge/README.md deleted file mode 100644 index 5cb4d8694..000000000 --- a/lib-ee/emqx_ee_bridge/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_ee_bridge -===== - -An OTP application - -Build ------ - - $ rebar3 compile diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct deleted file mode 100644 index 80f0d394b..000000000 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ /dev/null @@ -1 +0,0 @@ -toxiproxy diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config deleted file mode 100644 index 3b3be6ccf..000000000 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ /dev/null @@ -1,11 +0,0 @@ -%% -*- mode: erlang; -*- -{erl_opts, [debug_info]}. -{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {emqx_bridge, {path, "../../apps/emqx_bridge"}} - , {emqx_utils, {path, "../emqx_utils"}} - ]}. - -{shell, [ - {apps, [emqx_ee_bridge]} -]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src deleted file mode 100644 index e03cc9423..000000000 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ /dev/null @@ -1,27 +0,0 @@ -{application, emqx_ee_bridge, [ - {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.16"}, - {registered, []}, - {applications, [ - kernel, - stdlib, - emqx_ee_connector, - telemetry, - emqx_bridge_kafka, - emqx_bridge_gcp_pubsub, - emqx_bridge_cassandra, - emqx_bridge_opents, - emqx_bridge_pulsar, - emqx_bridge_dynamo, - emqx_bridge_sqlserver, - emqx_bridge_rocketmq, - emqx_bridge_rabbitmq, - emqx_bridge_tdengine, - emqx_bridge_influxdb, - emqx_bridge_clickhouse - ]}, - {env, []}, - {modules, []}, - - {links, []} -]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl deleted file mode 100644 index 13a70e7c7..000000000 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl +++ /dev/null @@ -1,90 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_ee_bridge_hstreamdb). - --include_lib("typerefl/include/types.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - --import(hoconsc, [mk/2, enum/1, ref/2]). - --export([ - conn_bridge_examples/1 -]). - --export([ - namespace/0, - roots/0, - fields/1, - desc/1 -]). - -%% ------------------------------------------------------------------------------------------------- -%% api - -conn_bridge_examples(Method) -> - [ - #{ - <<"hstreamdb">> => #{ - summary => <<"HStreamDB Bridge">>, - value => values(Method) - } - } - ]. - -values(_Method) -> - #{ - type => hstreamdb, - name => <<"demo">>, - connector => <<"hstreamdb:connector">>, - enable => true, - direction => egress, - local_topic => <<"local/topic/#">>, - payload => <<"${payload}">> - }. - -%% ------------------------------------------------------------------------------------------------- -%% Hocon Schema Definitions -namespace() -> "bridge_hstreamdb". - -roots() -> []. - -fields("config") -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, - {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, - {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}, - {connector, field(connector)} - ]; -fields("post") -> - [type_field(), name_field() | fields("config")]; -fields("put") -> - fields("config"); -fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). - -field(connector) -> - mk( - hoconsc:union([binary(), ref(emqx_ee_connector_hstreamdb, config)]), - #{ - required => true, - example => <<"hstreamdb:demo">>, - desc => ?DESC("desc_connector") - } - ). - -desc("config") -> - ?DESC("desc_config"); -desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> - ["Configuration for HStream using `", string:to_upper(Method), "` method."]; -desc(_) -> - undefined. - -%% ------------------------------------------------------------------------------------------------- -%% internal -type_field() -> - {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}. - -name_field() -> - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl deleted file mode 100644 index 867b09f32..000000000 --- a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(ee_bridge_hstreamdb_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -%% TODO: diff --git a/lib-ee/emqx_ee_connector/.gitignore b/lib-ee/emqx_ee_connector/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/lib-ee/emqx_ee_connector/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/lib-ee/emqx_ee_connector/README.md b/lib-ee/emqx_ee_connector/README.md deleted file mode 100644 index e665af458..000000000 --- a/lib-ee/emqx_ee_connector/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_ee_connector -===== - -An OTP application - -Build ------ - - $ rebar3 compile diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src deleted file mode 100644 index c3187f807..000000000 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ /dev/null @@ -1,16 +0,0 @@ -{application, emqx_ee_connector, [ - {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.15"}, - {registered, []}, - {applications, [ - kernel, - stdlib, - ecpool, - hstreamdb_erl, - emqx_redis - ]}, - {env, []}, - {modules, []}, - - {links, []} -]}. diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl deleted file mode 100644 index ad49d9f62..000000000 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_ee_connector_hstreamdb_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -%% TODO: diff --git a/lib-ee/emqx_license/include/emqx_license.hrl b/lib-ee/emqx_license/include/emqx_license.hrl index b2a0bb40b..2bd5ee9ec 100644 --- a/lib-ee/emqx_license/include/emqx_license.hrl +++ b/lib-ee/emqx_license/include/emqx_license.hrl @@ -12,7 +12,7 @@ "========================================================================\n" "Using an evaluation license limited to ~p concurrent connections.\n" "Apply for a license at https://emqx.com/apply-licenses/emqx.\n" - "Or contact EMQ customer services.\n" + "Or contact EMQ customer services via email at contact@emqx.io\n" "========================================================================\n" ). @@ -21,7 +21,7 @@ "========================================================================\n" "License has been expired for ~p days.\n" "Apply for a new license at https://emqx.com/apply-licenses/emqx.\n" - "Or contact EMQ customer services.\n" + "Or contact EMQ customer services via email at contact@emqx.io\n" "========================================================================\n" ). diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 2f21b8a52..f3a614a70 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/mix.exs b/mix.exs index 1e18ef8ef..65c942394 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.11", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.13", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -91,8 +91,7 @@ defmodule EMQXUmbrella.MixProject do {:cowlib, github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true}, # in conflict by cowboy_swagger and cowboy - {:ranch, - github: "emqx/ranch", ref: "de8ba2a00817c0a6eb1b8f20d6fb3e44e2c9a5aa", override: true}, + {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.7", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, @@ -196,7 +195,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.3.1+v0.12.0"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.7.6"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, @@ -396,8 +395,6 @@ defmodule EMQXUmbrella.MixProject do do: [ emqx_license: :permanent, emqx_enterprise: :load, - emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, emqx_bridge_pulsar: :permanent, emqx_bridge_gcp_pubsub: :permanent, diff --git a/rebar.config b/rebar.config index ff55d2e70..0f6864c5e 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.11"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 312fc1173..5f86afaa2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -463,8 +463,6 @@ relx_apps_per_edition(ee) -> [ emqx_license, {emqx_enterprise, load}, - emqx_ee_connector, - emqx_ee_bridge, emqx_bridge_kafka, emqx_bridge_pulsar, emqx_bridge_gcp_pubsub, diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index d0021c6a5..0a3b4aba4 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -1,10 +1,20 @@ emqx_authz_api_mnesia { action.desc: -"""Authorized action (pub/sub/all)""" +"""Authorized action (publish/subscribe/all)""" action.label: """action""" +qos.desc: +"""QoS of authorized action""" +qos.label: +"""QoS""" + +retain.desc: +"""Retain flag of authorized action""" +retain.label: +"""retain""" + clientid.desc: """ClientID""" clientid.label: diff --git a/rel/i18n/emqx_bridge_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo.hocon index 46ae9d1bb..899a47c75 100644 --- a/rel/i18n/emqx_bridge_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo.hocon @@ -35,7 +35,9 @@ local_topic.label: """Local Topic""" template.desc: -"""Template, the default value is empty. When this value is empty the whole message will be stored in the database""" +"""Template, the default value is empty. When this value is empty the whole message will be stored in the database.
+The template can be any valid json with placeholders and make sure all keys for table are here, example:
+ {"id" : ${id}, "clientid" : ${clientid}, "data" : ${payload}}""" template.label: """Template""" diff --git a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb.hocon similarity index 79% rename from rel/i18n/emqx_ee_bridge_hstreamdb.hocon rename to rel/i18n/emqx_bridge_hstreamdb.hocon index cb43d483a..10700d4eb 100644 --- a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_hstreamdb { +emqx_bridge_hstreamdb { config_direction.desc: """The direction of this bridge, MUST be 'egress'""" @@ -6,12 +6,6 @@ config_direction.desc: config_direction.label: """Bridge Direction""" -config_enable.desc: -"""Enable or disable this bridge""" - -config_enable.label: -"""Enable Or Disable Bridge""" - desc_config.desc: """Configuration for an HStreamDB bridge.""" @@ -46,10 +40,10 @@ will be forwarded.""" local_topic.label: """Local Topic""" -payload.desc: -"""The payload to be forwarded to the HStreamDB. Placeholders supported.""" +record_template.desc: +"""The HStream Record template to be forwarded to the HStreamDB. Placeholders supported.""" -payload.label: -"""Payload""" +record_template.label: +"""HStream Record""" } diff --git a/rel/i18n/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon similarity index 77% rename from rel/i18n/emqx_ee_connector_hstreamdb.hocon rename to rel/i18n/emqx_bridge_hstreamdb_connector.hocon index f6838297f..c0faa794c 100644 --- a/rel/i18n/emqx_ee_connector_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_hstreamdb { +emqx_bridge_hstreamdb_connector { config.desc: """HStreamDB connection config""" @@ -6,16 +6,34 @@ config.desc: config.label: """Connection config""" +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" + name.desc: """Connector name, used as a human-readable description of the connector.""" name.label: """Connector Name""" -ordering_key.desc: +url.desc: +"""HStreamDB Server URL""" + +url.label: +"""HStreamDB Server URL""" + +stream_name.desc: +"""HStreamDB Stream Name""" + +stream_name.label: +"""HStreamDB Stream Name""" + +partition_key.desc: """HStreamDB Ordering Key""" -ordering_key.label: +partition_key.label: """HStreamDB Ordering Key""" pool_size.desc: @@ -24,22 +42,10 @@ pool_size.desc: pool_size.label: """HStreamDB Pool Size""" -stream_name.desc: -"""HStreamDB Stream Name""" +grpc_timeout.desc: +"""HStreamDB gRPC Timeout""" -stream_name.label: -"""HStreamDB Stream Name""" - -type.desc: -"""The Connector Type.""" - -type.label: -"""Connector Type""" - -url.desc: -"""HStreamDB Server URL""" - -url.label: -"""HStreamDB Server URL""" +grpc_timeout.label: +"""HStreamDB gRPC Timeout""" } diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 9d88fcddd..81f908867 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -6,6 +6,12 @@ file_list.desc: file_list_transfer.desc: """List a file uploaded during specified transfer, identified by client id and file id.""" +file_transfer_get_config.desc: +"""Show current File Transfer configuration.""" + +file_transfer_update_config.desc: +"""Replace File Transfer configuration.""" + } emqx_ft_storage_exporter_fs_api { diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index ea49f6e21..e4061f7cb 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -216,6 +216,9 @@ for dep in ${CT_DEPS}; do gcp_emulator) FILES+=( '.ci/docker-compose-file/docker-compose-gcp-emulator.yaml' ) ;; + hstreamdb) + FILES+=( '.ci/docker-compose-file/docker-compose-hstreamdb.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 diff --git a/scripts/dev-cluster-host.sh b/scripts/dev-cluster-host.sh new file mode 100755 index 000000000..330bf1896 --- /dev/null +++ b/scripts/dev-cluster-host.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +set -euo pipefail + +## This starts configurable number of core and replicant nodes on the same host (not in docker). +## The nodes are named as core1, core2, replicant3, replicant4, ... where the number monotically increases. +## The number in node name is used as an offset for ekka to avoid clashing (see ekka_dist:offset/1). +## Nodes are started on loopback addresses starting from 127.0.0.1. +## The script uses sudo to add loopback aliases. +## The boot script is ./_build/emqx/rel/emqx/bin/emqx. +## The data and log directories are configured to use ./tmp/ + +# ensure dir +cd -P -- "$(dirname -- "$0")/../" + +help() { + echo + echo "start | stop" + echo "-h|--help: To display this usage info" + echo "-n|--nodes: total number of nodes to start (default: 2)" + echo "-c|--core_nodes: number of core nodes to start (default: 1)" + echo "-b|--boot: boot script (default: ./_build/emqx/rel/emqx/bin/emqx)" +} + +CMD="$1" +shift || true + +export EMQX_NODE__COOKIE=test +BOOT_SCRIPT='./_build/emqx/rel/emqx/bin/emqx' +NODES=2 +CORE_NODES=1 + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + -n|--nodes) + NODES="$2" + shift 2 + ;; + -c|--core-nodes) + CORE_NODES="$2" + shift 2 + ;; + -b|--boot) + BOOT_SCRIPT="$2" + shift 2 + ;; + *) + echo "unknown option $1" + exit 1 + ;; + esac +done + +REPLICANT_NODES=$((NODES - CORE_NODES)) + +# cannot use the same node name even IPs are different because Erlang distribution listens on 0.0.0.0 +CORE_IDS=() +REPLICANT_IDS=() +SEEDS_ARRAY=() +for i in $(seq 1 "$CORE_NODES"); do + SEEDS_ARRAY+=("core${i}@127.0.0.$i") + CORE_IDS+=("$i") +done +for i in $(seq "$((CORE_NODES+1))" "$((CORE_NODES+REPLICANT_NODES))"); do + REPLICANT_IDS+=("$i") +done + +SEEDS="$(IFS=,; echo "${SEEDS_ARRAY[*]}")" + +if [ "$CMD" = "stop" ]; then + for id in "${REPLICANT_IDS[@]}"; do + env EMQX_NODE_NAME="replicant${id}@127.0.0.$id" "$BOOT_SCRIPT" stop || true + done + for id in "${CORE_IDS[@]}"; do + env EMQX_NODE_NAME="core${id}@127.0.0.$id" "$BOOT_SCRIPT" stop || true + done + exit 0 +fi + +start_cmd() { + local role="$1" + local id="$2" + local ip="127.0.0.$id" + local nodename="$role$id" + local nodehome + nodehome="$(pwd)/tmp/$nodename" + mkdir -p "${nodehome}/data" "${nodehome}/log" + cat <<-EOF +env DEBUG="${DEBUG:-0}" \ +EMQX_NODE_NAME="$nodename@$ip" \ +EMQX_CLUSTER__STATIC__SEEDS="$SEEDS" \ +EMQX_CLUSTER__DISCOVERY_STRATEGY=static \ +EMQX_NODE__ROLE="$role" \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="${nodehome}/log/emqx.log" \ +EMQX_LOG_DIR="${nodehome}/log" \ +EMQX_NODE__DATA_DIR="${nodehome}/data" \ +EMQX_LISTENERS__TCP__DEFAULT__BIND="$ip:1883" \ +EMQX_LISTENERS__SSL__DEFAULT__BIND="$ip:8883" \ +EMQX_LISTENERS__WS__DEFAULT__BIND="$ip:8083" \ +EMQX_LISTENERS__WSS__DEFAULT__BIND="$ip:8084" \ +EMQX_DASHBOARD__LISTENERS__HTTP__BIND="$ip:18083" \ +"$BOOT_SCRIPT" start +EOF +} + +start_node() { + local cmd + cmd="$(start_cmd "$1" "$2" | envsubst)" + echo "$cmd" + eval "$cmd" +} + +for id in "${CORE_IDS[@]}"; do + sudo ifconfig lo0 alias "127.0.0.$id" up + start_node core "$id" & +done + +for id in "${REPLICANT_IDS[@]}"; do + sudo ifconfig lo0 alias "127.0.0.$id" up + start_node replicant "$id" & +done diff --git a/scripts/rerun-apps-version-check.py b/scripts/rerun-apps-version-check.py new file mode 100644 index 000000000..3b9fa1d3d --- /dev/null +++ b/scripts/rerun-apps-version-check.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# Usage: python3 rerun-apps-version-check.py -t -r +# +# Default repo is emqx/emqx +# +import requests +import http.client +import json +import os +import sys +import time +import math +import inspect +from optparse import OptionParser +from urllib.parse import urlparse, parse_qs +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +user_agent = sys.argv[0].split('/')[-1] + +def query(owner, repo): + return """ +query { + repository(owner: "%s", name: "%s") { + pullRequests(last: 25, states: OPEN) { + nodes { + url + commits(last: 1) { + nodes { + commit { + checkSuites(first: 17) { + nodes { + url + checkRuns(first: 1, filterBy: {checkName: "check_apps_version"}) { + nodes { + url + } + } + } + } + } + } + } + } + } + } +} + """ % (owner, repo) + + +def get_headers(token: str): + return {'Accept': 'application/vnd.github+json', + 'Authorization': f'Bearer {token}', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': f'{user_agent}' + } + +def get_session(): + session = requests.Session() + + retries = Retry(total=10, + backoff_factor=1, # 1s + allowed_methods=None, + status_forcelist=[ 429, 500, 502, 503, 504 ]) # Retry on these status codes + + session.mount('https://', HTTPAdapter(max_retries=retries)) + + return session + +def get_check_suite_ids(token: str, repo: str): + session = get_session() + url = f'https://api.github.com/graphql' + [repo_owner, repo_repo] = repo.split('/') + r = session.post(url, headers=get_headers(token), json={'query': query(repo_owner, repo_repo)}) + if r.status_code == 200: + resp = r.json() + if not 'data' in resp: + print(f'Failed to fetch check runs: {r.status_code}\n{r.json()}') + sys.exit(1) + ids = [] + for pr in resp['data']['repository']['pullRequests']['nodes']: + if not pr['commits']['nodes']: + continue + if not pr['commits']['nodes'][0]['commit']['checkSuites']['nodes']: + continue + for node in pr['commits']['nodes'][0]['commit']['checkSuites']['nodes']: + if node['checkRuns']['nodes']: + id = node['checkRuns']['nodes'][0]['url'].rsplit('/', 1)[-1] + url_parsed = urlparse(node['url']) + params = parse_qs(url_parsed.query) + check_suite_id = params['check_suite_id'][0] + ids.extend([check_suite_id]) + return ids + else: + print(f'Failed to fetch check runs: {r.status_code}\n{r.text}') + sys.exit(1) + +def rerequest_check_suite(token: str, repo: str, check_suite_id: str): + session = get_session() + url = f'https://api.github.com/repos/{repo}/check-suites/{check_suite_id}/rerequest' + r = session.post(url, headers=get_headers(token)) + if r.status_code == 201: + print(f'Successfully triggered rerequest for check suite {check_suite_id}') + else: + print(f'Failed to trigger rerequest for check suite {check_suite_id}: {r.status_code}\n{r.text}') + +def main(): + parser = OptionParser() + parser.add_option("-r", "--repo", dest="repo", + help="github repo", default="emqx/emqx") + parser.add_option("-t", "--token", dest="gh_token", + help="github API token") + (options, args) = parser.parse_args() + + # Get github token from env var if provided, else use the one from command line. + # The token must be exported in the env from ${{ secrets.GITHUB_TOKEN }} in the workflow. + token = os.environ['GITHUB_TOKEN'] if 'GITHUB_TOKEN' in os.environ else options.gh_token + for id in get_check_suite_ids(token, options.repo): + rerequest_check_suite(token, options.repo, id) + +if __name__ == '__main__': + main() diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 03587aa54..953b0b762 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -270,6 +270,10 @@ hstream hstreamDB hstream hstreamdb +hrecord +hRecord +Hrecord +HRecord SASL GSSAPI keytab