refactor: move JWT worker and helpers to separate app

Some bridge applications might need to use JWTs before the `emqx_connector` is started, so
we must move JWT table initialization to a separate dependency application.

apps/emqx_bridge_gcp_pubsub/mix.exs

@@ -23,6 +23,7 @@ defmodule EMQXBridgeGcpPubsub.MixProject do
 
   def deps() do
     [
+      {:emqx_connector_jwt, in_umbrella: true},
       {:emqx_connector, in_umbrella: true, runtime: false},
       {:emqx_resource, in_umbrella: true},
       {:emqx_bridge, in_umbrella: true, runtime: false},

apps/emqx_bridge_gcp_pubsub/rebar.config

@@ -9,6 +9,7 @@
     debug_info
 ]}.
 {deps, [
+    {emqx_connector_jwt, {path, "../../apps/emqx_connector_jwt"}},
     {emqx_connector, {path, "../../apps/emqx_connector"}},
     {emqx_resource, {path, "../../apps/emqx_resource"}},
     {emqx_bridge, {path, "../../apps/emqx_bridge"}},

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src

@@ -6,7 +6,8 @@
         kernel,
         stdlib,
         emqx_resource,
-        ehttpc
+        ehttpc,
+        emqx_connector_jwt
     ]},
     {env, [
         {emqx_action_info_modules, [

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl

@@ -5,7 +5,7 @@
 -module(emqx_bridge_gcp_pubsub_client).
 
 -include_lib("jose/include/jose_jwk.hrl").
--include_lib("emqx_connector/include/emqx_connector_tables.hrl").
+-include_lib("emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl

@@ -594,7 +594,7 @@ cluster(Config) ->
     Cluster = emqx_common_test_helpers:emqx_cluster(
         [core, core],
         [
-            {apps, [emqx_conf, emqx_rule_engine, emqx_bridge]},
+            {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_gcp_pubsub, emqx_bridge]},
             {listener_ports, []},
             {priv_data_dir, PrivDataDir},
             {load_schema, true},

apps/emqx_connector/mix.exs

@@ -33,6 +33,7 @@ defmodule EMQXConnector.MixProject do
     [
       {:emqx, in_umbrella: true},
       {:emqx_resource, in_umbrella: true},
+      {:emqx_connector_jwt, in_umbrella: true},
       UMP.common_dep(:jose),
       UMP.common_dep(:ecpool),
       {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.8.4"},

apps/emqx_connector/rebar.config

@@ -8,7 +8,8 @@
 {deps, [
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
-    {emqx_resource, {path, "../emqx_resource"}}
+    {emqx_resource, {path, "../emqx_resource"}},
+    {emqx_connector_jwt, {path, "../emqx_connector_jwt"}}
 ]}.
 
 {shell, [

apps/emqx_connector/src/emqx_connector_sup.erl

@@ -32,17 +32,5 @@ init([]) ->
         intensity => 5,
         period => 20
     },
-    ChildSpecs = [
-        child_spec(emqx_connector_jwt_sup)
-    ],
+    ChildSpecs = [],
     {ok, {SupFlags, ChildSpecs}}.
-
-child_spec(Mod) ->
-    #{
-        id => Mod,
-        start => {Mod, start_link, []},
-        restart => permanent,
-        shutdown => 3000,
-        type => supervisor,
-        modules => [Mod]
-    }.

apps/emqx_connector_jwt/README.md

@@ -0,0 +1,3 @@
+# emqx_connector_jwt
+
+This is a small helper application for connectors, actions and sources to generate JWTs.

apps/emqx_connector_jwt/mix.exs

@@ -0,0 +1,34 @@
+defmodule EMQXConnectorJWT.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_connector_jwt,
+      version: "0.1.0",
+      build_path: "../../_build",
+      erlc_options: UMP.erlc_options(),
+      erlc_paths: UMP.erlc_paths(),
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [
+      extra_applications: UMP.extra_applications(),
+      mod: {:emqx_connector_jwt_app, []}
+    ]
+  end
+
+  def deps() do
+    [
+      {:emqx_resource, in_umbrella: true},
+      UMP.common_dep(:jose),
+    ]
+  end
+end

apps/emqx_connector_jwt/rebar.config

@@ -0,0 +1,10 @@
+%% -*- mode: erlang -*-
+
+{erl_opts, [
+    nowarn_unused_import,
+    debug_info
+]}.
+
+{deps, [
+    {emqx_resource, {path, "../emqx_resource"}}
+]}.

apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src

@@ -0,0 +1,16 @@
+%% -*- mode: erlang -*-
+{application, emqx_connector_jwt, [
+    {description, "EMQX JWT Connector Utility"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {env, []},
+    {modules, []},
+    {mod, {emqx_connector_jwt_app, []}},
+
+    {licenses, ["Apache 2.0"]},
+    {links, []}
+]}.

apps/emqx_connector_jwt/src/emqx_connector_jwt.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_connector_jwt).
 
--include_lib("emqx_connector/include/emqx_connector_tables.hrl").
+-include("emqx_connector_jwt_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("jose/include/jose_jwt.hrl").
@@ -37,7 +37,7 @@
 -type jwt_config() :: #{
     expiration := duration(),
     resource_id := resource_id(),
-    table := ets:table(),
+    table => ets:table(),
     jwk := wrapped_jwk() | jwk(),
     iss := binary(),
     sub := binary(),
@@ -82,7 +82,8 @@ delete_jwt(TId, ResourceId) ->
 %% one.
 -spec ensure_jwt(jwt_config()) -> jwt().
 ensure_jwt(JWTConfig) ->
-    #{resource_id := ResourceId, table := Table} = JWTConfig,
+    #{resource_id := ResourceId} = JWTConfig,
+    Table = maps:get(table, JWTConfig, ?JWT_TABLE),
     case lookup_jwt(Table, ResourceId) of
         {error, not_found} ->
             JWT = do_generate_jwt(JWTConfig),
@@ -132,8 +133,9 @@ do_generate_jwt(#{
     JWT.
 
 -spec store_jwt(jwt_config(), jwt()) -> ok.
-store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
-    true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
+store_jwt(#{resource_id := ResourceId} = JWTConfig, JWT) ->
+    Table = maps:get(table, JWTConfig, ?JWT_TABLE),
+    true = ets:insert(Table, {{ResourceId, jwt}, JWT}),
     ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}),
     ok.
 

apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl

@@ -0,0 +1,39 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_connector_jwt_app).
+
+-behaviour(application).
+
+%% `application' API
+-export([start/2, stop/1]).
+
+%%------------------------------------------------------------------------------
+%% Type declarations
+%%------------------------------------------------------------------------------
+
+%%------------------------------------------------------------------------------
+%% `application' API
+%%------------------------------------------------------------------------------
+
+start(_StartType, _StartArgs) ->
+    emqx_connector_jwt_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Internal fns
+%%------------------------------------------------------------------------------

apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl

@@ -18,7 +18,7 @@
 
 -behaviour(supervisor).
 
--include_lib("emqx_connector/include/emqx_connector_tables.hrl").
+-include("emqx_connector_jwt_tables.hrl").
 
 -export([
     start_link/0,

apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl

@@ -20,7 +20,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("jose/include/jose_jwt.hrl").
 -include_lib("jose/include/jose_jws.hrl").
--include("emqx_connector_tables.hrl").
+-include("emqx_connector_jwt_tables.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -compile([export_all, nowarn_export_all]).
@@ -33,11 +33,12 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_common_test_helpers:start_apps([emqx_connector]),
-    Config.
+    Apps = emqx_cth_suite:start([emqx_connector_jwt], #{work_dir => emqx_cth_suite:work_dir(Config)}),
+    [{apps, Apps} | Config].
 
-end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_connector]),
+end_per_suite(Config) ->
+    Apps = ?config(apps, Config),
+    emqx_cth_suite:stop(Apps),
     ok.
 
 init_per_testcase(_TestCase, Config) ->

apps/emqx_machine/priv/reboot_lists.eterm

@@ -45,6 +45,7 @@
             emqx_ds_backends,
             emqx_http_lib,
             emqx_resource,
+            emqx_connector_jwt,
             emqx_connector,
             emqx_auth,
             emqx_auth_http,