From fa6343cc80f971f74b499b299ef6267bade049b4 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 31 Jul 2023 19:30:24 +0800 Subject: [PATCH 1/5] feat(ldap): add LDAP connector --- apps/emqx_ldap/BSL.txt | 94 +++++++ apps/emqx_ldap/README.md | 14 ++ apps/emqx_ldap/docker-ct | 1 + apps/emqx_ldap/rebar.config | 7 + apps/emqx_ldap/src/emqx_ldap.app.src | 13 + apps/emqx_ldap/src/emqx_ldap.erl | 230 ++++++++++++++++++ apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl | 24 ++ .../emqx_ldap/src/emqx_ldap_filter_parser.yrl | 133 ++++++++++ apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 165 +++++++++++++ rel/i18n/emqx_ldap.hocon | 25 ++ 10 files changed, 706 insertions(+) create mode 100644 apps/emqx_ldap/BSL.txt create mode 100644 apps/emqx_ldap/README.md create mode 100644 apps/emqx_ldap/docker-ct create mode 100644 apps/emqx_ldap/rebar.config create mode 100644 apps/emqx_ldap/src/emqx_ldap.app.src create mode 100644 apps/emqx_ldap/src/emqx_ldap.erl create mode 100644 apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl create mode 100644 apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl create mode 100644 apps/emqx_ldap/test/emqx_ldap_SUITE.erl create mode 100644 rel/i18n/emqx_ldap.hocon diff --git a/apps/emqx_ldap/BSL.txt b/apps/emqx_ldap/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_ldap/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_ldap/README.md b/apps/emqx_ldap/README.md new file mode 100644 index 000000000..5923a10d7 --- /dev/null +++ b/apps/emqx_ldap/README.md @@ -0,0 +1,14 @@ +# LDAP Connector + +This application houses the LDAP connector. +It provides the APIs to connect to the LDAP service. + +It is used by the emqx_authz and emqx_authn applications to check user permissions. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [APL](../../APL.txt). diff --git a/apps/emqx_ldap/docker-ct b/apps/emqx_ldap/docker-ct new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/apps/emqx_ldap/docker-ct @@ -0,0 +1 @@ + diff --git a/apps/emqx_ldap/rebar.config b/apps/emqx_ldap/rebar.config new file mode 100644 index 000000000..e6e2db243 --- /dev/null +++ b/apps/emqx_ldap/rebar.config @@ -0,0 +1,7 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} +]}. diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src new file mode 100644 index 000000000..976d85371 --- /dev/null +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -0,0 +1,13 @@ +{application, emqx_ldap, [ + {description, "EMQX LDAP Connector"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl new file mode 100644 index 000000000..4b43bb6f6 --- /dev/null +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -0,0 +1,230 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ldap). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("eldap/include/eldap.hrl"). + +-behaviour(emqx_resource). + +%% callbacks of behaviour emqx_resource +-export([ + callback_mode/0, + on_start/2, + on_stop/2, + on_query/3, + on_get_status/2 +]). + +%% ecpool connect & reconnect +-export([connect/1]). + +-export([roots/0, fields/1]). + +-export([do_get_status/1]). + +-define(LDAP_HOST_OPTIONS, #{ + default_port => 389 +}). + +-type params_tokens() :: #{atom() => list()}. +-type state() :: + #{ + pool_name := binary(), + base_tokens := params_tokens(), + filter_tokens := params_tokens() + }. + +-define(ECS, emqx_connector_schema_lib). + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, server()}, + {pool_size, fun ?ECS:pool_size/1}, + {username, fun ensure_username/1}, + {password, fun ?ECS:password/1}, + {base_object, + ?HOCON(binary(), #{ + desc => ?DESC(base_object), + required => true + })}, + {filter, ?HOCON(binary(), #{desc => ?DESC(filter), default => ""})}, + {auto_reconnect, fun ?ECS:auto_reconnect/1} + ] ++ emqx_connector_schema_lib:ssl_fields(). + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?LDAP_HOST_OPTIONS). + +ensure_username(required) -> + true; +ensure_username(Field) -> + ?ECS:username(Field). + +%% =================================================================== +callback_mode() -> always_sync. + +-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +on_start( + InstId, + #{ + server := Server, + pool_size := PoolSize, + ssl := SSL + } = Config +) -> + HostPort = emqx_schema:parse_server(Server, ?LDAP_HOST_OPTIONS), + ?SLOG(info, #{ + msg => "starting_ldap_connector", + connector => InstId, + config => emqx_utils:redact(Config) + }), + + Config2 = maps:merge(Config, HostPort), + Config3 = + case maps:get(enable, SSL) of + true -> + Config2#{sslopts => emqx_tls_lib:to_client_opts(SSL)}; + false -> + Config2 + end, + Options = [ + {pool_size, PoolSize}, + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, + {options, Config3} + ], + + case emqx_resource_pool:start(InstId, ?MODULE, Options) of + ok -> + {ok, prepare_template(Config, #{pool_name => InstId})}; + {error, Reason} -> + ?tp( + ldap_connector_start_failed, + #{error => Reason} + ), + {error, Reason} + end. + +on_stop(InstId, _State) -> + ?SLOG(info, #{ + msg => "stopping_ldap_connector", + connector => InstId + }), + emqx_resource_pool:stop(InstId). + +on_query(InstId, {query, Data}, State) -> + on_query(InstId, {query, Data}, [], State); +on_query(InstId, {query, Data, Attrs}, State) -> + on_query(InstId, {query, Data}, [{attributes, Attrs}], State); +on_query(InstId, {query, Data, Attrs, Timeout}, State) -> + on_query(InstId, {query, Data}, [{attributes, Attrs}, {timeout, Timeout}], State). + +on_get_status(_InstId, #{pool_name := PoolName} = _State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of + true -> + connected; + false -> + connecting + end. + +do_get_status(Conn) -> + erlang:is_process_alive(Conn). + +%% =================================================================== + +connect(Options) -> + #{host := Host, username := Username, password := Password} = + Conf = proplists:get_value(options, Options), + OpenOpts = maps:to_list(maps:with([port, sslopts], Conf)), + case eldap:open([Host], [{log, fun log/3}, OpenOpts]) of + {ok, Handle} = Ret -> + case eldap:simple_bind(Handle, Username, Password) of + ok -> Ret; + Error -> Error + end; + Error -> + Error + end. + +on_query( + InstId, + {query, Data}, + SearchOptions, + #{base_tokens := BaseTks, filter_tokens := FilterTks} = State +) -> + Base = emqx_placeholder:proc_tmpl(BaseTks, Data), + case FilterTks of + [] -> + do_ldap_query(InstId, [{base, Base} | SearchOptions], State); + _ -> + FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data), + %% TODO + Filter = FilterBin, + do_ldap_query( + InstId, + [{base, Base}, {filter, Filter} | SearchOptions], + State + ) + end. + +do_ldap_query( + InstId, + SearchOptions, + #{pool_name := PoolName} = State +) -> + LogMeta = #{connector => InstId, search => SearchOptions, state => State}, + ?TRACE("QUERY", "ldap_connector_received", LogMeta), + case + ecpool:pick_and_do( + PoolName, + {eldap, search, SearchOptions}, + handover + ) + of + {ok, Result} -> + ?tp( + ldap_connector_query_return, + #{result => Result} + ), + {ok, + case Result#eldap_search_result.entries of + [First | _] -> + %% TODO Support multi entries? + First; + _ -> + undefined + end}; + {error, Reason} -> + ?SLOG( + error, + LogMeta#{msg => "ldap_connector_do_sql_query_failed", reason => Reason} + ), + {error, {unrecoverable_error, Reason}} + end. + +log(Level, Format, Args) -> + ?SLOG( + Level, + #{ + msg => "ldap_log", + log => io_lib:format(Format, Args) + } + ). + +prepare_template(Config, State) -> + do_prepare_template(maps:to_list(maps:with([base_object, filter], Config)), State). + +do_prepare_template([{base_object, V} | T], State) -> + do_prepare_template(T, State#{base_tokens => emqx_placeholder:preproc_tmpl(V)}); +do_prepare_template([{filter, V} | T], State) -> + do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)}). diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl new file mode 100644 index 000000000..ae5c0cdbf --- /dev/null +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -0,0 +1,24 @@ +Definitions. + +NonControl = [^()&|!=~><:*] +String = {NonControl}* +White = [\s\t\n\r]+ + +Rules. + +\( : {token, {lparen, TokenLine}}. +\) : {token, {rparen, TokenLine}}. +\& : {token, {'and', TokenLine}}. +\| : {token, {'or', TokenLine}}. +\! : {token, {'not', TokenLine}}. += : {token, {equal, TokenLine}}. +~= : {token, {approx, TokenLine}}. +>= : {token, {greaterOrEqual, TokenLine}}. +<= : {token, {lessOrEqual, TokenLine}}. +\* : {token, {asterisk, TokenLine}}. +\: : {token, {colon, TokenLine}}. +dn : {token, {dn, TokenLine}}. +{White} : skip_token. +{String} : {token, {string, TokenLine, TokenChars}}. + +Erlang code. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl new file mode 100644 index 000000000..2e9cf193d --- /dev/null +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -0,0 +1,133 @@ +Nonterminals +filter filtercomp filterlist item simple present substring initial any final extensible attr value type dnattrs matchingrule. + +Terminals +lparen rparen 'and' 'or' 'not' equal approx greaterOrEqual lessOrEqual asterisk colon dn string. + +Rootsymbol filter. +Left 100 present. +Left 500 substring. + +filter -> + lparen filtercomp rparen : '$2'. + +filtercomp -> + 'and' filterlist: 'and'('$2'). +filtercomp -> + 'or' filterlist: 'or'('$2'). +filtercomp -> + 'not' filterlist: 'not'('$2'). +filtercomp -> + item: '$1'. + +filterlist -> + filter: '$1'. +filterlist -> + filter filterlist: ['$1' | '$2']. + +item -> + simple: '$1'. +item -> + present: '$1'. +item -> + substring: '$1'. +item-> + extensible: '$1'. + +simple -> + attr equal value: equal('$1', '$3'). +simple -> + attr approx value: approx('$1', '$3'). +simple -> + attr greaterOrEqual value: greaterOrEqual('$1', '$3'). +simple -> + attr lessOrEqual value: lessOrEqual('$1', '$3'). + +present -> + attr equal asterisk: present('$1'). + +substring -> + attr equal initial asterisk any final: substrings('$1', ['$3', '$5', '$6']). +substring -> + attr equal asterisk any final: substrings('$1', ['$4', '$5']). +substring -> + attr equal initial asterisk any: substrings('$1', ['$3', '$5']). +substring -> + attr equal asterisk any: substrings('$1', ['$4']). + +initial -> + value: {initial, '$1'}. + +final -> + value: {final, '$1'}. + +any -> any value asterisk: 'any'('$1', '$2'). +any -> '$empty': []. + +extensible -> + type dnattrs matchingrule colon equal value : extensible('$6', ['$1', '$2', '$3']). +extensible -> + type dnattrs colon equal value: extensible('$5', ['$1', '$2']). +extensible -> + type matchingrule colon equal value: extensible('$5', ['$1', '$2']). +extensible -> + type colon equal value: extensible('$4', []). + +extensible -> + dnattrs matchingrule colon equal value: extensible('$5', ['$1', '$2']). +extensible -> + matchingrule colon equal value: extensible('$4', ['$1']). + +attr -> + string: get_value('$1'). + +value -> + string: get_value('$1'). + +type -> + value: {type, '$1'}. + +dnattrs -> + colon dn: {dnAttributes, true}. + +matchingrule -> + colon value: {matchingRule, '$2'}. + +Erlang code. + +'and'(Value) -> + eldap:'and'(Value). + +'or'(Value) -> + eldap:'or'(Value). + +'not'(Value) -> + eldap:'not'(Value). + +equal(Attr, Value) -> + eldap:equalityMatch(Attr, Value). + +approx(Attr, Value) -> + eldap:approxMatch(Attr, Value). + +greaterOrEqual(Attr, Value) -> + eldap:greaterOrEqual(Attr, Value). + +lessOrEqual(Attr, Value) -> + eldap:lessOrEqual(Attr, Value). + +present(Value) -> + eldap:present(Value). + +substrings(Attr, List) -> + eldap:substrings(Attr, flatten(List)). + +'any'(List, Item) -> + [{any, Item} | List]. + +extensible(Value, Opts) -> eldap:extensibleMatch(Value, Opts). + +flatten(List) -> lists:flatten(List). + +get_value({_Token, _Line, Value}) -> + Value. diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl new file mode 100644 index 000000000..515f8b250 --- /dev/null +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -0,0 +1,165 @@ +% %%-------------------------------------------------------------------- +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% +% %% Licensed under the Apache License, Version 2.0 (the "License"); +% %% you may not use this file except in compliance with the License. +% %% You may obtain a copy of the License at +% %% http://www.apache.org/licenses/LICENSE-2.0 +% %% +% %% Unless required by applicable law or agreed to in writing, software +% %% distributed under the License is distributed on an "AS IS" BASIS, +% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% %% See the License for the specific language governing permissions and +% %% limitations under the License. +% %%-------------------------------------------------------------------- + +-module(emqx_ldap_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(MYSQL_HOST, "ldap"). +-define(MYSQL_RESOURCE_MOD, emqx_ldap). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + Config; + false -> + {skip, no_ldap} + end. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_lifecycle(_Config) -> + perform_lifecycle_check( + <<"emqx_ldap_SUITE">>, + ldap_config() + ). + +perform_lifecycle_check(ResourceId, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), + {ok, #{ + state := #{pool_name := PoolName} = State, + status := InitialStatus + }} = emqx_resource:create_local( + ResourceId, + ?CONNECTOR_RESOURCE_GROUP, + ?MYSQL_RESOURCE_MOD, + CheckedConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + % Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + % % Perform query as further check that the resource is working as expected + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), + ?assertMatch( + {ok, _, [[1]]}, + emqx_resource:query( + ResourceId, + test_query_with_params_and_timeout() + ) + ), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), + % Resource will be listed still, but state will be changed and healthcheck will fail + % as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = + emqx_resource:get_instance(ResourceId), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(ResourceId)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(ResourceId)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), + ?assertMatch( + {ok, _, [[1]]}, + emqx_resource:query( + ResourceId, + test_query_with_params_and_timeout() + ) + ), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ + +ldap_config() -> + RawConfig = list_to_binary( + io_lib:format( + "" + "\n" + " auto_reconnect = true\n" + " database = mqtt\n" + " username= root\n" + " password = public\n" + " pool_size = 8\n" + " server = \"~s:~b\"\n" + " " + "", + [?MYSQL_HOST, ?MYSQL_DEFAULT_PORT] + ) + ), + + {ok, Config} = hocon:binary(RawConfig), + #{<<"config">> => Config}. + +test_query_no_params() -> + {sql, <<"SELECT 1">>}. + +test_query_with_params() -> + {sql, <<"SELECT ?">>, [1]}. + +test_query_with_params_and_timeout() -> + {sql, <<"SELECT ?">>, [1], 1000}. diff --git a/rel/i18n/emqx_ldap.hocon b/rel/i18n/emqx_ldap.hocon new file mode 100644 index 000000000..9f3d9216b --- /dev/null +++ b/rel/i18n/emqx_ldap.hocon @@ -0,0 +1,25 @@ +emqx_ldap { + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The MySQL default port 3306 is used if `[:Port]` is not specified.""" + +server.label: +"""Server Host""" + +base_object.desc: +"""The name of the base object entry (or possibly the root) relative to +which the Search is to be performed.""" + +base_object.label: +"""Base Object""" + +filter.desc: +"""The filter that defines the conditions that must be fulfilled in order +for the Search to match a given entry.""" + +filter.label: +"""Filter""" + +} From 8c9b136d159321150b6a4fb947469e93f5c756e8 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Aug 2023 11:40:37 +0800 Subject: [PATCH 2/5] fix(ldap): integrate parser and fix lexer errors --- apps/emqx_ldap/.gitignore | 2 ++ apps/emqx_ldap/src/emqx_ldap.erl | 24 ++++++++++++------- apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl | 3 +++ .../emqx_ldap/src/emqx_ldap_filter_parser.yrl | 12 ++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 apps/emqx_ldap/.gitignore diff --git a/apps/emqx_ldap/.gitignore b/apps/emqx_ldap/.gitignore new file mode 100644 index 000000000..b912731cf --- /dev/null +++ b/apps/emqx_ldap/.gitignore @@ -0,0 +1,2 @@ +src/emqx_ldap_filter_lexer.erl +src/emqx_ldap_filter_parser.erl \ No newline at end of file diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 4b43bb6f6..390c34501 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -167,14 +167,22 @@ on_query( [] -> do_ldap_query(InstId, [{base, Base} | SearchOptions], State); _ -> - FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data), - %% TODO - Filter = FilterBin, - do_ldap_query( - InstId, - [{base, Base}, {filter, Filter} | SearchOptions], - State - ) + FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data, #{return => rawlist}), + case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of + {ok, Filter} -> + do_ldap_query( + InstId, + [{base, Base}, {filter, Filter} | SearchOptions], + State + ); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "filter_parse_failed", + filter => FilterBin, + reason => Reason + }), + Error + end end. do_ldap_query( diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index ae5c0cdbf..e30531e2a 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -1,5 +1,6 @@ Definitions. +Control = [()&|!=~><:*] NonControl = [^()&|!=~><:*] String = {NonControl}* White = [\s\t\n\r]+ @@ -20,5 +21,7 @@ Rules. dn : {token, {dn, TokenLine}}. {White} : skip_token. {String} : {token, {string, TokenLine, TokenChars}}. +%% Leex will hang if a composite operation is missing a character +{Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}. Erlang code. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl index 2e9cf193d..266d126c1 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -94,6 +94,8 @@ matchingrule -> colon value: {matchingRule, '$2'}. Erlang code. +-export([scan_and_parse/1]). +-ignore_xref({return_error, 2}). 'and'(Value) -> eldap:'and'(Value). @@ -131,3 +133,13 @@ flatten(List) -> lists:flatten(List). get_value({_Token, _Line, Value}) -> Value. + +scan_and_parse(Bin) when is_binary(Bin) -> + scan_and_parse(erlang:binary_to_list(Bin)); +scan_and_parse(String) -> + case emqx_ldap_filter_lexer:string(String) of + {ok, Tokens, _} -> + parse(Tokens); + {error, Reason, _} -> + {error, Reason} + end. From f98f97f37e460fce82894670dfe5e5d11cc11f04 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 1 Aug 2023 18:48:43 +0800 Subject: [PATCH 3/5] feat(ldap): set test env and add test suites --- ...ldap-tcp.yaml => docker-compose-ldap.yaml} | 8 +- .ci/docker-compose-file/openldap/Dockerfile | 16 +- apps/emqx_ldap/docker-ct | 2 +- apps/emqx_ldap/src/emqx_ldap.erl | 26 +- apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl | 4 +- .../emqx_ldap/src/emqx_ldap_filter_parser.yrl | 8 +- apps/emqx_ldap/test/data/emqx.io.ldif | 135 ++++++++++ apps/emqx_ldap/test/data/emqx.schema | 46 ++++ apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 103 +++++--- .../emqx_ldap/test/emqx_ldap_filter_SUITE.erl | 245 ++++++++++++++++++ scripts/ct/run.sh | 3 + 11 files changed, 533 insertions(+), 63 deletions(-) rename .ci/docker-compose-file/{docker-compose-ldap-tcp.yaml => docker-compose-ldap.yaml} (80%) create mode 100644 apps/emqx_ldap/test/data/emqx.io.ldif create mode 100644 apps/emqx_ldap/test/data/emqx.schema create mode 100644 apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-ldap-tcp.yaml b/.ci/docker-compose-file/docker-compose-ldap.yaml similarity index 80% rename from .ci/docker-compose-file/docker-compose-ldap-tcp.yaml rename to .ci/docker-compose-file/docker-compose-ldap.yaml index 61eab91ec..e6c8ba2d8 100644 --- a/.ci/docker-compose-file/docker-compose-ldap-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-ldap.yaml @@ -6,11 +6,11 @@ services: build: context: ../.. dockerfile: .ci/docker-compose-file/openldap/Dockerfile - args: + args: LDAP_TAG: ${LDAP_TAG} - image: openldap - ports: - - 389:389 + image: openldap + #ports: + # - 389:389 restart: always networks: - emqx_bridge diff --git a/.ci/docker-compose-file/openldap/Dockerfile b/.ci/docker-compose-file/openldap/Dockerfile index 88a096066..dd0114b64 100644 --- a/.ci/docker-compose-file/openldap/Dockerfile +++ b/.ci/docker-compose-file/openldap/Dockerfile @@ -1,18 +1,20 @@ -FROM buildpack-deps:stretch +FROM buildpack-deps:bookworm -ARG LDAP_TAG=2.4.50 +ARG LDAP_TAG=2.5.16 RUN apt-get update && apt-get install -y groff groff-base -RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${LDAP_TAG}.tgz \ - && gunzip -c openldap-${LDAP_TAG}.tgz | tar xvfB - \ +RUN wget https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${LDAP_TAG}.tgz \ + && tar xvzf openldap-${LDAP_TAG}.tgz \ && cd openldap-${LDAP_TAG} \ && ./configure && make depend && make && make install \ && cd .. && rm -rf openldap-${LDAP_TAG} COPY .ci/docker-compose-file/openldap/slapd.conf /usr/local/etc/openldap/slapd.conf -COPY apps/emqx_authn/test/data/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif -COPY apps/emqx_authn/test/data/emqx.schema /usr/local/etc/openldap/schema/emqx.schema -COPY apps/emqx_authn/test/data/certs/*.pem /usr/local/etc/openldap/ +COPY apps/emqx_ldap/test/data/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +COPY apps/emqx_ldap/test/data/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +COPY .ci/docker-compose-file/certs/ca.crt /usr/local/etc/openldap/cacert.pem +COPY .ci/docker-compose-file/certs/server.crt /usr/local/etc/openldap/cert.pem +COPY .ci/docker-compose-file/certs/server.key /usr/local/etc/openldap/key.pem RUN mkdir -p /usr/local/etc/openldap/data \ && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf diff --git a/apps/emqx_ldap/docker-ct b/apps/emqx_ldap/docker-ct index 8b1378917..c1142c3c5 100644 --- a/apps/emqx_ldap/docker-ct +++ b/apps/emqx_ldap/docker-ct @@ -1 +1 @@ - +ldap diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 390c34501..173edbe11 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -58,7 +58,8 @@ fields(config) -> desc => ?DESC(base_object), required => true })}, - {filter, ?HOCON(binary(), #{desc => ?DESC(filter), default => ""})}, + {filter, + ?HOCON(binary(), #{desc => ?DESC(filter), default => <<"(objectClass=mqttUser)">>})}, {auto_reconnect, fun ?ECS:auto_reconnect/1} ] ++ emqx_connector_schema_lib:ssl_fields(). @@ -143,10 +144,10 @@ do_get_status(Conn) -> %% =================================================================== connect(Options) -> - #{host := Host, username := Username, password := Password} = + #{hostname := Host, username := Username, password := Password} = Conf = proplists:get_value(options, Options), OpenOpts = maps:to_list(maps:with([port, sslopts], Conf)), - case eldap:open([Host], [{log, fun log/3}, OpenOpts]) of + case eldap:open([Host], [{log, fun log/3} | OpenOpts]) of {ok, Handle} = Ret -> case eldap:simple_bind(Handle, Username, Password) of ok -> Ret; @@ -167,7 +168,7 @@ on_query( [] -> do_ldap_query(InstId, [{base, Base} | SearchOptions], State); _ -> - FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data, #{return => rawlist}), + FilterBin = emqx_placeholder:proc_tmpl(FilterTks, Data), case emqx_ldap_filter_parser:scan_and_parse(FilterBin) of {ok, Filter} -> do_ldap_query( @@ -195,7 +196,7 @@ do_ldap_query( case ecpool:pick_and_do( PoolName, - {eldap, search, SearchOptions}, + {eldap, search, [SearchOptions]}, handover ) of @@ -204,14 +205,9 @@ do_ldap_query( ldap_connector_query_return, #{result => Result} ), - {ok, - case Result#eldap_search_result.entries of - [First | _] -> - %% TODO Support multi entries? - First; - _ -> - undefined - end}; + {ok, Result#eldap_search_result.entries}; + {error, noSuchObject} -> + {ok, []}; {error, Reason} -> ?SLOG( error, @@ -235,4 +231,6 @@ prepare_template(Config, State) -> do_prepare_template([{base_object, V} | T], State) -> do_prepare_template(T, State#{base_tokens => emqx_placeholder:preproc_tmpl(V)}); do_prepare_template([{filter, V} | T], State) -> - do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)}). + do_prepare_template(T, State#{filter_tokens => emqx_placeholder:preproc_tmpl(V)}); +do_prepare_template([], State) -> + State. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index e30531e2a..5c0e178d6 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -1,9 +1,9 @@ Definitions. Control = [()&|!=~><:*] -NonControl = [^()&|!=~><:*] -String = {NonControl}* White = [\s\t\n\r]+ +NonString = [^()&|!=~><:*\s\t\n\r] +String = {NonString}+ Rules. diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl index 266d126c1..84e412f21 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -16,12 +16,12 @@ filtercomp -> filtercomp -> 'or' filterlist: 'or'('$2'). filtercomp -> - 'not' filterlist: 'not'('$2'). + 'not' filter: 'not'('$2'). filtercomp -> item: '$1'. filterlist -> - filter: '$1'. + filter: ['$1']. filterlist -> filter filterlist: ['$1' | '$2']. @@ -71,7 +71,7 @@ extensible -> extensible -> type matchingrule colon equal value: extensible('$5', ['$1', '$2']). extensible -> - type colon equal value: extensible('$4', []). + type colon equal value: extensible('$4', ['$1']). extensible -> dnattrs matchingrule colon equal value: extensible('$5', ['$1', '$2']). @@ -125,7 +125,7 @@ substrings(Attr, List) -> eldap:substrings(Attr, flatten(List)). 'any'(List, Item) -> - [{any, Item} | List]. + [List, {any, Item}]. extensible(Value, Opts) -> eldap:extensibleMatch(Value, Opts). diff --git a/apps/emqx_ldap/test/data/emqx.io.ldif b/apps/emqx_ldap/test/data/emqx.io.ldif new file mode 100644 index 000000000..f9833cd88 --- /dev/null +++ b/apps/emqx_ldap/test/data/emqx.io.ldif @@ -0,0 +1,135 @@ +## create emqx.io + +dn:dc=emqx,dc=io +objectclass: top +objectclass: dcobject +objectclass: organization +dc:emqx +o:emqx,Inc. + +# create testdevice.emqx.io +dn:ou=testdevice,dc=emqx,dc=io +objectClass: top +objectclass:organizationalUnit +ou:testdevice + +# create user admin +dn:uid=admin,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: simpleSecurityObject +objectClass: account +userPassword:: e1NIQX1XNnBoNU1tNVB6OEdnaVVMYlBnekczN21qOWc9 +uid: admin + +## create user=mqttuser0001, +# password=mqttuser0001, +# passhash={SHA}mlb3fat40MKBTXUVZwCKmL73R/0= +# base64passhash=e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 +dn:uid=mqttuser0001,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0001 +isEnabled: TRUE +mqttAccountName: user1 +mqttPublishTopic: mqttuser0001/pub/1 +mqttPublishTopic: mqttuser0001/pub/+ +mqttPublishTopic: mqttuser0001/pub/# +mqttSubscriptionTopic: mqttuser0001/sub/1 +mqttSubscriptionTopic: mqttuser0001/sub/+ +mqttSubscriptionTopic: mqttuser0001/sub/# +mqttPubSubTopic: mqttuser0001/pubsub/1 +mqttPubSubTopic: mqttuser0001/pubsub/+ +mqttPubSubTopic: mqttuser0001/pubsub/# +userPassword:: e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 + +## create user=mqttuser0002 +# password=mqttuser0002, +# passhash={SSHA}n9XdtoG4Q/TQ3TQF4Y+khJbMBH4qXj4M +# base64passhash=e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= +dn:uid=mqttuser0002,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0002 +isEnabled: TRUE +mqttAccountName: user2 +mqttPublishTopic: mqttuser0002/pub/1 +mqttPublishTopic: mqttuser0002/pub/+ +mqttPublishTopic: mqttuser0002/pub/# +mqttSubscriptionTopic: mqttuser0002/sub/1 +mqttSubscriptionTopic: mqttuser0002/sub/+ +mqttSubscriptionTopic: mqttuser0002/sub/# +mqttPubSubTopic: mqttuser0002/pubsub/1 +mqttPubSubTopic: mqttuser0002/pubsub/+ +mqttPubSubTopic: mqttuser0002/pubsub/# +userPassword:: e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= + +## create user mqttuser0003 +# password=mqttuser0003, +# passhash={MD5}ybsPGoaK3nDyiQvveiCOIw== +# base64passhash=e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= +dn:uid=mqttuser0003,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0003 +isEnabled: TRUE +mqttPublishTopic: mqttuser0003/pub/1 +mqttPublishTopic: mqttuser0003/pub/+ +mqttPublishTopic: mqttuser0003/pub/# +mqttSubscriptionTopic: mqttuser0003/sub/1 +mqttSubscriptionTopic: mqttuser0003/sub/+ +mqttSubscriptionTopic: mqttuser0003/sub/# +mqttPubSubTopic: mqttuser0003/pubsub/1 +mqttPubSubTopic: mqttuser0003/pubsub/+ +mqttPubSubTopic: mqttuser0003/pubsub/# +userPassword:: e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= + +## create user mqttuser0004 +# password=mqttuser0004, +# passhash={MD5}2Br6pPDSEDIEvUlu9+s+MA== +# base64passhash=e01ENX0yQnI2cFBEU0VESUV2VWx1OStzK01BPT0= +dn:uid=mqttuser0004,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0004 +isEnabled: TRUE +mqttPublishTopic: mqttuser0004/pub/1 +mqttPublishTopic: mqttuser0004/pub/+ +mqttPublishTopic: mqttuser0004/pub/# +mqttSubscriptionTopic: mqttuser0004/sub/1 +mqttSubscriptionTopic: mqttuser0004/sub/+ +mqttSubscriptionTopic: mqttuser0004/sub/# +mqttPubSubTopic: mqttuser0004/pubsub/1 +mqttPubSubTopic: mqttuser0004/pubsub/+ +mqttPubSubTopic: mqttuser0004/pubsub/# +userPassword: {MD5}2Br6pPDSEDIEvUlu9+s+MA== + +## create user mqttuser0005 +# password=mqttuser0005, +# passhash={SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= +# base64passhash=e1NIQX1qS254ZUVER1IxNGtFOEFSN3l1VkZPZWxoejQ9 +objectClass: top +dn:uid=mqttuser0005,ou=testdevice,dc=emqx,dc=io +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0005 +isEnabled: TRUE +mqttPublishTopic: mqttuser0005/pub/1 +mqttPublishTopic: mqttuser0005/pub/+ +mqttPublishTopic: mqttuser0005/pub/# +mqttSubscriptionTopic: mqttuser0005/sub/1 +mqttSubscriptionTopic: mqttuser0005/sub/+ +mqttSubscriptionTopic: mqttuser0005/sub/# +mqttPubSubTopic: mqttuser0005/pubsub/1 +mqttPubSubTopic: mqttuser0005/pubsub/+ +mqttPubSubTopic: mqttuser0005/pubsub/# +userPassword: {SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= + diff --git a/apps/emqx_ldap/test/data/emqx.schema b/apps/emqx_ldap/test/data/emqx.schema new file mode 100644 index 000000000..55f92269b --- /dev/null +++ b/apps/emqx_ldap/test/data/emqx.schema @@ -0,0 +1,46 @@ +# +# Preliminary Apple OS X Native LDAP Schema +# This file is subject to change. +# +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.1.3 NAME 'isEnabled' + EQUALITY booleanMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + SINGLE-VALUE + USAGE userApplications ) + +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.1 NAME ( 'mqttPublishTopic' 'mpt' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.2 NAME ( 'mqttSubscriptionTopic' 'mst' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.3 NAME ( 'mqttPubSubTopic' 'mpst' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.4 NAME ( 'mqttAccountName' 'man' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) + + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4 NAME 'mqttUser' + AUXILIARY + MAY ( mqttPublishTopic $ mqttSubscriptionTopic $ mqttPubSubTopic $ mqttAccountName) ) + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.2 NAME 'mqttDevice' + SUP top + STRUCTURAL + MUST ( uid ) + MAY ( isEnabled ) ) + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.3 NAME 'mqttSecurity' + SUP top + AUXILIARY + MAY ( userPassword $ userPKCS12 $ pwdAttribute $ pwdLockout ) ) diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 515f8b250..8a2b67929 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -22,18 +22,33 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). +-include_lib("eldap/include/eldap.hrl"). --define(MYSQL_HOST, "ldap"). --define(MYSQL_RESOURCE_MOD, emqx_ldap). +-define(LDAP_HOST, "ldap"). +-define(LDAP_RESOURCE_MOD, emqx_ldap). all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, tcp}, + {group, ssl} + ]. groups() -> - []. + Cases = emqx_common_test_helpers:all(?MODULE), + [ + {tcp, Cases}, + {ssl, Cases} + ]. + +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(_, Config) -> + proplists:delete(group, Config). init_per_suite(Config) -> - case emqx_common_test_helpers:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_DEFAULT_PORT) of + Port = port(tcp), + case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, Port) of true -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), @@ -58,22 +73,22 @@ end_per_testcase(_, _Config) -> % %% Testcases % %%------------------------------------------------------------------------------ -t_lifecycle(_Config) -> +t_lifecycle(Config) -> perform_lifecycle_check( <<"emqx_ldap_SUITE">>, - ldap_config() + ldap_config(Config) ). perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = - emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), + emqx_resource:check_config(?LDAP_RESOURCE_MOD, InitialConfig), {ok, #{ state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( ResourceId, ?CONNECTOR_RESOURCE_GROUP, - ?MYSQL_RESOURCE_MOD, + ?LDAP_RESOURCE_MOD, CheckedConfig, #{} ), @@ -86,15 +101,22 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> emqx_resource:get_instance(ResourceId), ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), ?assertMatch( - {ok, _, [[1]]}, + {ok, [#eldap_entry{attributes = [_, _ | _]}]}, + emqx_resource:query(ResourceId, test_query_no_attr()) + ), + ?assertMatch( + {ok, [#eldap_entry{attributes = [{"mqttAccountName", _}]}]}, + emqx_resource:query(ResourceId, test_query_with_attr()) + ), + ?assertMatch( + {ok, _}, emqx_resource:query( ResourceId, - test_query_with_params_and_timeout() + test_query_with_attr_and_timeout() ) ), + ?assertMatch({ok, []}, emqx_resource:query(ResourceId, test_query_not_exists())), ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. @@ -116,13 +138,13 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = emqx_resource:get_instance(ResourceId), ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), + ?assertMatch({ok, _}, emqx_resource:query(ResourceId, test_query_no_attr())), + ?assertMatch({ok, _}, emqx_resource:query(ResourceId, test_query_with_attr())), ?assertMatch( - {ok, _, [[1]]}, + {ok, _}, emqx_resource:query( ResourceId, - test_query_with_params_and_timeout() + test_query_with_attr_and_timeout() ) ), % Stop and remove the resource in one go. @@ -134,32 +156,51 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> % %%------------------------------------------------------------------------------ % %% Helpers % %%------------------------------------------------------------------------------ - -ldap_config() -> +ldap_config(Config) -> RawConfig = list_to_binary( io_lib:format( "" "\n" " auto_reconnect = true\n" - " database = mqtt\n" - " username= root\n" + " username= \"cn=root,dc=emqx,dc=io\"\n" " password = public\n" " pool_size = 8\n" " server = \"~s:~b\"\n" - " " + " base_object=\"uid=${username},ou=testdevice,dc=emqx,dc=io\"\n" + " filter =\"(objectClass=mqttUser)\"\n" + " ~ts\n" "", - [?MYSQL_HOST, ?MYSQL_DEFAULT_PORT] + [?LDAP_HOST, port(Config), ssl(Config)] ) ), - {ok, Config} = hocon:binary(RawConfig), - #{<<"config">> => Config}. + {ok, LDConfig} = hocon:binary(RawConfig), + #{<<"config">> => LDConfig}. -test_query_no_params() -> - {sql, <<"SELECT 1">>}. +test_query_no_attr() -> + {query, data()}. -test_query_with_params() -> - {sql, <<"SELECT ?">>, [1]}. +test_query_with_attr() -> + {query, data(), ["mqttAccountName"]}. -test_query_with_params_and_timeout() -> - {sql, <<"SELECT ?">>, [1], 1000}. +test_query_with_attr_and_timeout() -> + {query, data(), ["mqttAccountName"], 5000}. + +test_query_not_exists() -> + {query, #{username => <<"not_exists">>}}. + +data() -> + #{username => <<"mqttuser0001">>}. + +port(tcp) -> 389; +port(ssl) -> 636; +port(Config) -> port(proplists:get_value(group, Config)). + +ssl(Config) -> + case proplists:get_value(group, Config) of + tcp -> + "ssl.enable=false"; + ssl -> + "ssl.enable=true\n" + "ssl.cacertfile=\"etc/openldap/cacert.pem\"" + end. diff --git a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl new file mode 100644 index 000000000..8ea322ff1 --- /dev/null +++ b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl @@ -0,0 +1,245 @@ +% %%-------------------------------------------------------------------- +% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +% %% +% %% Licensed under the Apache License, Version 2.0 (the "License"); +% %% you may not use this file except in compliance with the License. +% %% You may obtain a copy of the License at +% %% http://www.apache.org/licenses/LICENSE-2.0 +% %% +% %% Unless required by applicable law or agreed to in writing, software +% %% distributed under the License is distributed on an "AS IS" BASIS, +% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% %% See the License for the specific language governing permissions and +% %% limitations under the License. +% %%-------------------------------------------------------------------- + +-module(emqx_ldap_filter_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-import(eldap, [ + 'and'/1, + 'or'/1, + 'not'/1, + equalityMatch/2, + substrings/2, + present/1, + greaterOrEqual/2, + lessOrEqual/2, + approxMatch/2, + extensibleMatch/2 +]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + _ = application:stop(emqx_connector). + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_and(_Config) -> + ?assertEqual('and'([equalityMatch("a", "1")]), parse("(&(a=1))")), + ?assertEqual( + 'and'([equalityMatch("a", "1"), (equalityMatch("b", "2"))]), + parse("(&(a=1)(b=2))") + ), + ?assertMatch({error, _}, scan_and_parse("(&)")). + +t_or(_Config) -> + ?assertEqual('or'([equalityMatch("a", "1")]), parse("(|(a=1))")), + ?assertEqual( + 'or'([equalityMatch("a", "1"), (equalityMatch("b", "2"))]), + parse("(|(a=1)(b=2))") + ), + ?assertMatch({error, _}, scan_and_parse("(|)")). + +t_not(_Config) -> + ?assertEqual('not'(equalityMatch("a", "1")), parse("(!(a=1))")), + ?assertMatch({error, _}, scan_and_parse("(!)")), + ?assertMatch({error, _}, scan_and_parse("(!(a=1)(b=1))")). + +t_equalityMatch(_Config) -> + ?assertEqual(equalityMatch("attr", "value"), parse("(attr=value)")), + ?assertEqual(equalityMatch("attr", "value"), parse("(attr = value)")), + ?assertMatch({error, _}, scan_and_parse("(attr=)")), + ?assertMatch({error, _}, scan_and_parse("(=)")), + ?assertMatch({error, _}, scan_and_parse("(=value)")). + +t_substrings_initial(_Config) -> + ?assertEqual(substrings("attr", [{initial, "initial"}]), parse("(attr=initial*)")), + ?assertEqual( + substrings("attr", [{initial, "initial"}, {any, "a"}]), + parse("(attr=initial*a*)") + ), + ?assertEqual( + substrings("attr", [{initial, "initial"}, {any, "a"}, {any, "b"}]), + parse("(attr=initial*a*b*)") + ). + +t_substrings_final(_Config) -> + ?assertEqual(substrings("attr", [{final, "final"}]), parse("(attr=*final)")), + ?assertEqual( + substrings("attr", [{any, "a"}, {final, "final"}]), + parse("(attr=*a*final)") + ), + ?assertEqual( + substrings("attr", [{any, "a"}, {any, "b"}, {final, "final"}]), + parse("(attr=*a*b*final)") + ). + +t_substrings_initial_final(_Config) -> + ?assertEqual( + substrings("attr", [{initial, "initial"}, {final, "final"}]), + parse("(attr=initial*final)") + ), + ?assertEqual( + substrings("attr", [{initial, "initial"}, {any, "a"}, {final, "final"}]), + parse("(attr=initial*a*final)") + ), + ?assertEqual( + substrings( + "attr", + [{initial, "initial"}, {any, "a"}, {any, "b"}, {final, "final"}] + ), + parse("(attr=initial*a*b*final)") + ). + +t_substrings_only_any(_Config) -> + ?assertEqual(present("attr"), parse("(attr=*)")), + ?assertEqual(substrings("attr", [{any, "a"}]), parse("(attr=*a*)")), + ?assertEqual( + substrings("attr", [{any, "a"}, {any, "b"}]), + parse("(attr=*a*b*)") + ). + +t_greaterOrEqual(_Config) -> + ?assertEqual(greaterOrEqual("attr", "value"), parse("(attr>=value)")), + ?assertEqual(greaterOrEqual("attr", "value"), parse("(attr >= value )")), + ?assertMatch({error, _}, scan_and_parse("(attr>=)")), + ?assertMatch({error, _}, scan_and_parse("(>=)")), + ?assertMatch({error, _}, scan_and_parse("(>=value)")). + +t_lessOrEqual(_Config) -> + ?assertEqual(lessOrEqual("attr", "value"), parse("(attr<=value)")), + ?assertEqual(lessOrEqual("attr", "value"), parse("( attr <= value )")), + ?assertMatch({error, _}, scan_and_parse("(attr<=)")), + ?assertMatch({error, _}, scan_and_parse("(<=)")), + ?assertMatch({error, _}, scan_and_parse("(<=value)")). + +t_present(_Config) -> + ?assertEqual(present("attr"), parse("(attr=*)")), + ?assertEqual(present("attr"), parse("( attr = * )")). + +t_approxMatch(_Config) -> + ?assertEqual(approxMatch("attr", "value"), parse("(attr~=value)")), + ?assertEqual(approxMatch("attr", "value"), parse("( attr ~= value )")), + ?assertMatch({error, _}, scan_and_parse("(attr~=)")), + ?assertMatch({error, _}, scan_and_parse("(~=)")), + ?assertMatch({error, _}, scan_and_parse("(~=value)")). + +t_extensibleMatch_dn(_Config) -> + ?assertEqual( + extensibleMatch("value", [{type, "attr"}, {dnAttributes, true}]), parse("(attr:dn:=value)") + ), + ?assertEqual( + extensibleMatch("value", [{type, "attr"}, {dnAttributes, true}]), + parse("( attr:dn := value )") + ). + +t_extensibleMatch_rule(_Config) -> + ?assertEqual( + extensibleMatch("value", [{type, "attr"}, {matchingRule, "objectClass"}]), + parse("(attr:objectClass:=value)") + ), + ?assertEqual( + extensibleMatch("value", [{type, "attr"}, {matchingRule, "objectClass"}]), + parse("( attr:objectClass := value )") + ). + +t_extensibleMatch_dn_rule(_Config) -> + ?assertEqual( + extensibleMatch( + "value", + [ + {type, "attr"}, + {dnAttributes, true}, + {matchingRule, "objectClass"} + ] + ), + parse("(attr:dn:objectClass:=value)") + ), + ?assertEqual( + extensibleMatch( + "value", + [ + {type, "attr"}, + {dnAttributes, true}, + {matchingRule, "objectClass"} + ] + ), + parse("( attr:dn:objectClass :=value)") + ). + +t_extensibleMatch_no_dn_rule(_Config) -> + ?assertEqual(extensibleMatch("value", [{type, "attr"}]), parse("(attr:=value)")), + ?assertEqual(extensibleMatch("value", [{type, "attr"}]), parse("( attr := value )")). + +t_extensibleMatch_no_type_dn(_Config) -> + ?assertEqual( + extensibleMatch("value", [{matchingRule, "objectClass"}]), + parse("(:objectClass:=value)") + ), + ?assertEqual( + extensibleMatch("value", [{matchingRule, "objectClass"}]), + parse("( :objectClass := value )") + ). + +t_extensibleMatch_no_type_no_dn(_Config) -> + ?assertEqual( + extensibleMatch( + "value", + [{dnAttributes, true}, {matchingRule, "objectClass"}] + ), + parse("(:dn:objectClass:=value)") + ), + ?assertEqual( + extensibleMatch( + "value", + [{dnAttributes, true}, {matchingRule, "objectClass"}] + ), + parse("( :dn:objectClass :=value)") + ). + +t_extensibleMatch_error(_Config) -> + ?assertMatch({error, _}, scan_and_parse("(:dn:=value)")), + ?assertMatch({error, _}, scan_and_parse("(::=value)")), + ?assertMatch({error, _}, scan_and_parse("(:=)")), + ?assertMatch({error, _}, scan_and_parse("(attr:=)")). + +t_error(_Config) -> + ?assertMatch({error, _}, scan_and_parse("(attr=value")), + ?assertMatch({error, _}, scan_and_parse("attr=value")), + ?assertMatch({error, _}, scan_and_parse("(a=b)(c=d)")). + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ +parse(Str) -> + {ok, Res} = scan_and_parse(Str), + Res. + +scan_and_parse(Str) -> + emqx_ldap_filter_parser:scan_and_parse(Str). diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 578b9c4de..5ad289303 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -225,6 +225,9 @@ for dep in ${CT_DEPS}; do greptimedb) FILES+=( '.ci/docker-compose-file/docker-compose-greptimedb.yaml' ) ;; + ldap) + FILES+=( '.ci/docker-compose-file/docker-compose-ldap.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From b2f3ac9967512209669e2ddb095e908157e3890e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 2 Aug 2023 10:07:27 +0800 Subject: [PATCH 4/5] fix(ldap): fix newline error && correcting files header --- apps/emqx_ldap/.gitignore | 2 +- apps/emqx_ldap/src/emqx_ldap.erl | 1 + apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl | 4 ++++ apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl | 4 ++++ apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 17 +++-------------- apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl | 17 +++-------------- 6 files changed, 16 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ldap/.gitignore b/apps/emqx_ldap/.gitignore index b912731cf..3b0d6b553 100644 --- a/apps/emqx_ldap/.gitignore +++ b/apps/emqx_ldap/.gitignore @@ -1,2 +1,2 @@ src/emqx_ldap_filter_lexer.erl -src/emqx_ldap_filter_parser.erl \ No newline at end of file +src/emqx_ldap_filter_parser.erl diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 173edbe11..7fed819de 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -1,6 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- + -module(emqx_ldap). -include_lib("emqx_connector/include/emqx_connector.hrl"). diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl index 5c0e178d6..6d75c2546 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_lexer.xrl @@ -25,3 +25,7 @@ dn : {token, {dn, TokenLine}}. {Control} : {error, lists:flatten(io_lib:format("Unexpected Tokens:~ts", [TokenChars]))}. Erlang code. + +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- diff --git a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl index 84e412f21..57f526ffd 100644 --- a/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl +++ b/apps/emqx_ldap/src/emqx_ldap_filter_parser.yrl @@ -1,3 +1,7 @@ +Header "%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%--------------------------------------------------------------------". + Nonterminals filter filtercomp filterlist item simple present substring initial any final extensible attr value type dnattrs matchingrule. diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 8a2b67929..9386b1738 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -1,17 +1,6 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(emqx_ldap_SUITE). diff --git a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl index 8ea322ff1..0351fb93d 100644 --- a/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_filter_SUITE.erl @@ -1,17 +1,6 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(emqx_ldap_filter_SUITE). From 9bb5c9de33017349770999c4069863aef07dfe65 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 2 Aug 2023 11:02:02 +0800 Subject: [PATCH 5/5] fix(ldap): make elvis happy --- apps/emqx_ldap/src/emqx_ldap.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 7fed819de..7cc9a6be0 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -207,7 +207,7 @@ do_ldap_query( #{result => Result} ), {ok, Result#eldap_search_result.entries}; - {error, noSuchObject} -> + {error, 'noSuchObject'} -> {ok, []}; {error, Reason} -> ?SLOG(