From a6a9538e73ac059822050660e556b6f671b92167 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 30 Jul 2024 16:05:46 -0300 Subject: [PATCH] refactor: move JWT worker and helpers to separate app Some bridge applications might need to use JWTs before the `emqx_connector` is started, so we must move JWT table initialization to a separate dependency application. --- apps/emqx_bridge_gcp_pubsub/mix.exs | 1 + apps/emqx_bridge_gcp_pubsub/rebar.config | 1 + .../src/emqx_bridge_gcp_pubsub.app.src | 3 +- .../src/emqx_bridge_gcp_pubsub_client.erl | 2 +- .../emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 2 +- apps/emqx_connector/mix.exs | 1 + apps/emqx_connector/rebar.config | 3 +- .../emqx_connector/src/emqx_connector_sup.erl | 14 +------ apps/emqx_connector_jwt/README.md | 3 ++ .../include/emqx_connector_jwt_tables.hrl} | 0 apps/emqx_connector_jwt/mix.exs | 34 ++++++++++++++++ apps/emqx_connector_jwt/rebar.config | 10 +++++ .../src/emqx_connector_jwt.app.src | 16 ++++++++ .../src/emqx_connector_jwt.erl | 12 +++--- .../src/emqx_connector_jwt_app.erl | 39 +++++++++++++++++++ .../src/emqx_connector_jwt_sup.erl | 2 +- .../src/emqx_connector_jwt_worker.erl | 0 .../test/emqx_connector_jwt_SUITE.erl | 11 +++--- .../test/emqx_connector_jwt_worker_SUITE.erl | 0 apps/emqx_machine/priv/reboot_lists.eterm | 1 + 20 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 apps/emqx_connector_jwt/README.md rename apps/{emqx_connector/include/emqx_connector_tables.hrl => emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl} (100%) create mode 100644 apps/emqx_connector_jwt/mix.exs create mode 100644 apps/emqx_connector_jwt/rebar.config create mode 100644 apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt.erl (92%) create mode 100644 apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt_sup.erl (97%) rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt_worker.erl (100%) rename apps/{emqx_connector => emqx_connector_jwt}/test/emqx_connector_jwt_SUITE.erl (94%) rename apps/{emqx_connector => emqx_connector_jwt}/test/emqx_connector_jwt_worker_SUITE.erl (100%) diff --git a/apps/emqx_bridge_gcp_pubsub/mix.exs b/apps/emqx_bridge_gcp_pubsub/mix.exs index 3a9fae0a1..34e02ca9f 100644 --- a/apps/emqx_bridge_gcp_pubsub/mix.exs +++ b/apps/emqx_bridge_gcp_pubsub/mix.exs @@ -23,6 +23,7 @@ defmodule EMQXBridgeGcpPubsub.MixProject do def deps() do [ + {:emqx_connector_jwt, in_umbrella: true}, {:emqx_connector, in_umbrella: true, runtime: false}, {:emqx_resource, in_umbrella: true}, {:emqx_bridge, in_umbrella: true, runtime: false}, diff --git a/apps/emqx_bridge_gcp_pubsub/rebar.config b/apps/emqx_bridge_gcp_pubsub/rebar.config index a6a12b429..e5a65b745 100644 --- a/apps/emqx_bridge_gcp_pubsub/rebar.config +++ b/apps/emqx_bridge_gcp_pubsub/rebar.config @@ -9,6 +9,7 @@ debug_info ]}. {deps, [ + {emqx_connector_jwt, {path, "../../apps/emqx_connector_jwt"}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index eff7847f2..a39c4be99 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -6,7 +6,8 @@ kernel, stdlib, emqx_resource, - ehttpc + ehttpc, + emqx_connector_jwt ]}, {env, [ {emqx_action_info_modules, [ diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl index 67218fcf0..ea6f67112 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl @@ -5,7 +5,7 @@ -module(emqx_bridge_gcp_pubsub_client). -include_lib("jose/include/jose_jwk.hrl"). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include_lib("emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 0e6956d58..656413225 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -594,7 +594,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_rule_engine, emqx_bridge]}, + {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_gcp_pubsub, emqx_bridge]}, {listener_ports, []}, {priv_data_dir, PrivDataDir}, {load_schema, true}, diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs index a641c27fe..a818d8072 100644 --- a/apps/emqx_connector/mix.exs +++ b/apps/emqx_connector/mix.exs @@ -33,6 +33,7 @@ defmodule EMQXConnector.MixProject do [ {:emqx, in_umbrella: true}, {:emqx_resource, in_umbrella: true}, + {:emqx_connector_jwt, in_umbrella: true}, UMP.common_dep(:jose), UMP.common_dep(:ecpool), {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.8.4"}, diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 94da3c580..7e0ff4ea3 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -8,7 +8,8 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, - {emqx_resource, {path, "../emqx_resource"}} + {emqx_resource, {path, "../emqx_resource"}}, + {emqx_connector_jwt, {path, "../emqx_connector_jwt"}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 0d2f337a3..09cd8ea68 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -32,17 +32,5 @@ init([]) -> intensity => 5, period => 20 }, - ChildSpecs = [ - child_spec(emqx_connector_jwt_sup) - ], + ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. - -child_spec(Mod) -> - #{ - id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 3000, - type => supervisor, - modules => [Mod] - }. diff --git a/apps/emqx_connector_jwt/README.md b/apps/emqx_connector_jwt/README.md new file mode 100644 index 000000000..868d7b08f --- /dev/null +++ b/apps/emqx_connector_jwt/README.md @@ -0,0 +1,3 @@ +# emqx_connector_jwt + +This is a small helper application for connectors, actions and sources to generate JWTs. diff --git a/apps/emqx_connector/include/emqx_connector_tables.hrl b/apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl similarity index 100% rename from apps/emqx_connector/include/emqx_connector_tables.hrl rename to apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl diff --git a/apps/emqx_connector_jwt/mix.exs b/apps/emqx_connector_jwt/mix.exs new file mode 100644 index 000000000..752275a56 --- /dev/null +++ b/apps/emqx_connector_jwt/mix.exs @@ -0,0 +1,34 @@ +defmodule EMQXConnectorJWT.MixProject do + use Mix.Project + alias EMQXUmbrella.MixProject, as: UMP + + def project do + [ + app: :emqx_connector_jwt, + version: "0.1.0", + build_path: "../../_build", + erlc_options: UMP.erlc_options(), + erlc_paths: UMP.erlc_paths(), + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [ + extra_applications: UMP.extra_applications(), + mod: {:emqx_connector_jwt_app, []} + ] + end + + def deps() do + [ + {:emqx_resource, in_umbrella: true}, + UMP.common_dep(:jose), + ] + end +end diff --git a/apps/emqx_connector_jwt/rebar.config b/apps/emqx_connector_jwt/rebar.config new file mode 100644 index 000000000..252534c46 --- /dev/null +++ b/apps/emqx_connector_jwt/rebar.config @@ -0,0 +1,10 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {emqx_resource, {path, "../emqx_resource"}} +]}. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src new file mode 100644 index 000000000..e284d7471 --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{application, emqx_connector_jwt, [ + {description, "EMQX JWT Connector Utility"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + {mod, {emqx_connector_jwt_app, []}}, + + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl similarity index 92% rename from apps/emqx_connector/src/emqx_connector_jwt.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt.erl index 60b35ddbb..933a259de 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl @@ -16,7 +16,7 @@ -module(emqx_connector_jwt). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("jose/include/jose_jwt.hrl"). @@ -37,7 +37,7 @@ -type jwt_config() :: #{ expiration := duration(), resource_id := resource_id(), - table := ets:table(), + table => ets:table(), jwk := wrapped_jwk() | jwk(), iss := binary(), sub := binary(), @@ -82,7 +82,8 @@ delete_jwt(TId, ResourceId) -> %% one. -spec ensure_jwt(jwt_config()) -> jwt(). ensure_jwt(JWTConfig) -> - #{resource_id := ResourceId, table := Table} = JWTConfig, + #{resource_id := ResourceId} = JWTConfig, + Table = maps:get(table, JWTConfig, ?JWT_TABLE), case lookup_jwt(Table, ResourceId) of {error, not_found} -> JWT = do_generate_jwt(JWTConfig), @@ -132,8 +133,9 @@ do_generate_jwt(#{ JWT. -spec store_jwt(jwt_config(), jwt()) -> ok. -store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> - true = ets:insert(TId, {{ResourceId, jwt}, JWT}), +store_jwt(#{resource_id := ResourceId} = JWTConfig, JWT) -> + Table = maps:get(table, JWTConfig, ?JWT_TABLE), + true = ets:insert(Table, {{ResourceId, jwt}, JWT}), ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}), ok. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl new file mode 100644 index 000000000..9c9c134de --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_jwt_app). + +-behaviour(application). + +%% `application' API +-export([start/2, stop/1]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% `application' API +%%------------------------------------------------------------------------------ + +start(_StartType, _StartArgs) -> + emqx_connector_jwt_sup:start_link(). + +stop(_State) -> + ok. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ diff --git a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl similarity index 97% rename from apps/emqx_connector/src/emqx_connector_jwt_sup.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl index d50be6395..4579b221c 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl @@ -18,7 +18,7 @@ -behaviour(supervisor). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -export([ start_link/0, diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_jwt_worker.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl similarity index 94% rename from apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl index aef0e660c..a0416b9d5 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl +++ b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl @@ -20,7 +20,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("jose/include/jose_jwt.hrl"). -include_lib("jose/include/jose_jws.hrl"). --include("emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -compile([export_all, nowarn_export_all]). @@ -33,11 +33,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_connector]), - Config. + Apps = emqx_cth_suite:start([emqx_connector_jwt], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_connector]), +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), ok. init_per_testcase(_TestCase, Config) -> diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl similarity index 100% rename from apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 4830f51c4..251b9790a 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -45,6 +45,7 @@ emqx_ds_backends, emqx_http_lib, emqx_resource, + emqx_connector_jwt, emqx_connector, emqx_auth, emqx_auth_http,