Merge pull request #5170 from qzhuyan/dev/william/ci-fix-build-package

ci(quic): conditionally quicer build

.ci/build_packages/tests.sh

@@ -38,7 +38,8 @@ emqx_test(){
                 packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
                 unzip -q "${PACKAGE_PATH}/${packagename}"
                 export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
-                       EMQX_MQTT__MAX_TOPIC_ALIAS=10
+                    EMQX_MQTT__MAX_TOPIC_ALIAS=10
+                [[ $(arch) == *arm* || $(arch) == aarch64 ]] && export EMQX_LISTENER__QUIC__EXTERNAL__ENDPOINT=''
                 # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
 
                 echo "running ${packagename} start"
@@ -113,11 +114,25 @@ emqx_test(){
 }
 
 running_test(){
-    export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
-           EMQX_MQTT__MAX_TOPIC_ALIAS=10
     # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
+    emqx_env_vars=$(dirname "$(readlink "$(command -v emqx)")")/../releases/emqx_vars
 
-    if ! emqx start; then
+    if [ -f "$emqx_env_vars" ];
+    then
+        tee -a "$emqx_env_vars" <<EOF
+export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60
+export EMQX_MQTT__MAX_TOPIC_ALIAS=10
+EOF
+        ## for ARM, due to CI env issue, skip start of quic listener for the moment
+        [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee tee -a "$emqx_env_vars" <<EOF
+export EMQX_LISTENER__QUIC__EXTERNAL__ENDPOINT=''
+EOF
+    else
+        echo "Error: cannot locate emqx_vars"
+        exit 1
+    fi
+
+    if ! su - emqx -c "emqx start"; then
         cat /var/log/emqx/erlang.log.1 || true
         cat /var/log/emqx/emqx.log.1 || true
         exit 1
@@ -138,7 +153,6 @@ running_test(){
 
     if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
     || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
-
         if ! service emqx start; then
             cat /var/log/emqx/erlang.log.1 || true
             cat /var/log/emqx/emqx.log.1 || true

.github/workflows/build_packages.yaml

@@ -42,6 +42,7 @@ jobs:
         if: endsWith(github.repository, 'emqx')
         run: |
           make -C source deps-all
+          rm source/rebar.lock
           zip -ryq source.zip source/* source/.[^.]*
       - name: get_all_deps
         if: endsWith(github.repository, 'enterprise')
@@ -63,6 +64,7 @@ jobs:
     if: endsWith(github.repository, 'emqx')
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         exclude:
@@ -131,6 +133,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
@@ -210,6 +213,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:
@@ -336,6 +340,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:

apps/emqx/rebar.config

@@ -20,7 +20,6 @@
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
-    , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.5"}}}
     ]}.
 
 {plugins, [rebar3_proper]}.
@@ -31,7 +30,7 @@
            [ meck
            , {bbmustache,"1.10.0"}
            , {emqx_ct_helpers, {git,"https://github.com/emqx/emqx-ct-helpers", {branch,"hocon"}}}
-           , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.1"}}}
+           , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.2"}}}
            ]},
          {extra_src_dirs, [{"test",[recursive]}]}
        ]}

apps/emqx/rebar.config.script

@@ -1,11 +1,30 @@
-Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}},
+IsCentos6 = fun() ->
-AddBcrypt = fun(C) ->
+                case file:read_file("/etc/centos-release") of
-    {deps, Deps0} = lists:keyfind(deps, 1, C),
+                    {ok, <<"CentOS release 6", _/binary >>} ->
-    Deps = [Bcrypt | Deps0],
+                        true;
-    lists:keystore(deps, 1, C, {deps, Deps})
+                    _ ->
-end,
+                        false
+                end
+            end,
 
-case os:type() of
+IsWin32 = fun() ->
-    {win32, _} -> CONFIG;
+                win32 =:= element(1, os:type())
-    _ -> AddBcrypt(CONFIG)
+          end,
-end.
+
+IsQuicSupp = fun() ->
+                not (IsCentos6() orelse IsWin32() orelse
+                     false =/= os:getenv("EMQX_BUILD_WITHOUT_QUIC")
+                    )
+             end,
+
+Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}},
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {branch, "main"}}},
+
+ExtraDeps = fun(C) ->
+                {deps, Deps0} = lists:keyfind(deps, 1, C),
+                Deps = Deps0 ++ [Bcrypt || not IsWin32()] ++
+                [ Quicer || IsQuicSupp()],
+                lists:keystore(deps, 1, C, {deps, Deps})
+            end,
+
+ExtraDeps(CONFIG).

apps/emqx/src/emqx.app.src

@@ -4,7 +4,7 @@
   {vsn, "5.0.0"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
-  {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,quicer,jiffy]},
+  {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,jiffy]},
   {mod, {emqx_app,[]}},
   {env, []},
   {licenses, ["Apache-2.0"]},

apps/emqx/src/emqx_app.erl

@@ -49,6 +49,8 @@ start(_Type, _Args) ->
     _ = load_ce_modules(),
     ekka:start(),
     ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity),
+    false == os:getenv("EMQX_NO_QUIC")
+        andalso application:ensure_all_started(quicer),
     {ok, Sup} = emqx_sup:start_link(),
     ok = start_autocluster(),
     % ok = emqx_plugins:init(),

apps/emqx/src/emqx_listeners.erl

@@ -106,6 +106,8 @@ format_listen_on(ListenOn) -> format(ListenOn).
 start_listener(#{proto := Proto, name := Name, listen_on := ListenOn, opts := Options}) ->
     ID = identifier(Proto, Name),
     case start_listener(Proto, ListenOn, Options) of
+        {ok, skipped} ->
+            console_print("Start ~s listener on ~s skpped.~n", [ID, format(ListenOn)]);
         {ok, _} ->
             console_print("Start ~s listener on ~s successfully.~n", [ID, format(ListenOn)]);
         {error, Reason} ->
@@ -123,7 +125,7 @@ console_print(_Fmt, _Args) -> ok.
 
 %% Start MQTT/TCP listener
 -spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()])
-      -> {ok, pid()} | {error, term()}).
+      -> {ok, pid() | skipped} | {error, term()}).
 start_listener(tcp, ListenOn, Options) ->
     start_mqtt_listener('mqtt:tcp', ListenOn, Options);
 
@@ -143,24 +145,31 @@ start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss ->
 
 %% Start MQTT/QUIC listener
 start_listener(quic, ListenOn, Options) ->
-    %% @fixme unsure why we need reopen lib and reopen config.
+    case [ A || {quicer, _, _} = A<-application:which_applications() ] of
-    quicer_nif:open_lib(),
+        [_] ->
-    quicer_nif:reg_open(),
+            %% @fixme unsure why we need reopen lib and reopen config.
-    SSLOpts = proplists:get_value(ssl_options, Options),
+            quicer_nif:open_lib(),
-    DefAcceptors = erlang:system_info(schedulers_online) * 8,
+            quicer_nif:reg_open(),
-    ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)}
+            SSLOpts = proplists:get_value(ssl_options, Options),
-                 , {key, proplists:get_value(keyfile, SSLOpts)}
+            DefAcceptors = erlang:system_info(schedulers_online) * 8,
-                 , {alpn, ["mqtt"]}
+            ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)}
-                 , {conn_acceptors, proplists:get_value(acceptors, Options, DefAcceptors)}
+                         , {key, proplists:get_value(keyfile, SSLOpts)}
-                 , {idle_timeout_ms, proplists:get_value(idle_timeout, Options, 60000)}
+                         , {alpn, ["mqtt"]}
-                 ],
+                         , {conn_acceptors, proplists:get_value(acceptors, Options, DefAcceptors)}
-    ConnectionOpts = [ {conn_callback, emqx_quic_connection}
+                         , {idle_timeout_ms, proplists:get_value(idle_timeout, Options, 60000)}
-                     , {peer_unidi_stream_count, 1}
+                         ],
-                     , {peer_bidi_stream_count, 10}
+            ConnectionOpts = [ {conn_callback, emqx_quic_connection}
-                       | Options
+                             , {peer_unidi_stream_count, 1}
-                     ],
+                             , {peer_bidi_stream_count, 10}
-    StreamOpts = [],
+                             | Options
-    quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts}).
+                             ],
+            StreamOpts = [],
+            quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts});
+        [] ->
+            io:format(standard_error, "INFO: quicer application is unavailable/disabled~n",
+                      []),
+            {ok, skipped}
+    end.
 
 replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)].
 

apps/emqx/src/emqx_schema.erl

@@ -333,7 +333,7 @@ fields("quic_listener") ->
     [ {"$name", ref("quic_listener_settings")}];
 
 fields("listener_settings") ->
-    [ {"endpoint", t(union(ip_port(), integer()))}
+    [ {"endpoint", t(union([ip_port(), integer(), ""]))}
     , {"acceptors", t(integer(), undefined, 8)}
     , {"max_connections", t(integer(), undefined, 1024)}
     , {"max_conn_rate", t(integer())}
@@ -785,6 +785,7 @@ tr_listeners(Conf) ->
     TcpListeners = fun(Type, Name) ->
         Prefix = string:join(["listener", Type, Name], "."),
         ListenOnN = case conf_get(Prefix ++ ".endpoint", Conf) of
+                        "" -> [];
                         undefined -> [];
                         ListenOn  -> ListenOn
                     end,
@@ -801,6 +802,8 @@ tr_listeners(Conf) ->
     SslListeners = fun(Type, Name) ->
         Prefix = string:join(["listener", Type, Name], "."),
         case conf_get(Prefix ++ ".endpoint", Conf) of
+            "" ->
+                [];
             undefined ->
                 [];
             ListenOn ->

rebar.config

@@ -55,7 +55,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
     , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.1"}}}
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.2"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
@@ -63,7 +63,6 @@
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
     , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.9.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}}
-    , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.5"}}}
     ]}.
 
 {xref_ignores,

rebar.config.erl

@@ -15,12 +15,14 @@ do(Dir, CONFIG) ->
 bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}.
 
+quicer() ->
+    %% @todo use tag
+    {quicer, {git, "https://github.com/emqx/quic.git", {branch, "main"}}}.
+
 deps(Config) ->
     {deps, OldDeps} = lists:keyfind(deps, 1, Config),
-    MoreDeps = case provide_bcrypt_dep() of
+    MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++
-        true -> [bcrypt()];
+        [quicer() || is_quicer_supported()],
-        false -> []
-    end,
     {HasElixir, ExtraDeps} = extra_deps(),
     {HasElixir, lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps ++ ExtraDeps})}.
 
@@ -78,6 +80,24 @@ is_cover_enabled() ->
 is_enterprise() ->
     filelib:is_regular("EMQX_ENTERPRISE").
 
+is_quicer_supported() ->
+    not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse
+         is_win32() orelse is_centos_6()
+        ).
+
+is_centos_6() ->
+    %% reason:
+    %% glibc is too old
+    case file:read_file("/etc/centos-release") of
+        {ok, <<"CentOS release 6", _/binary >>} ->
+            true;
+        _ ->
+            false
+    end.
+
+is_win32() ->
+    win32 =:= element(1, os:type()).
+
 project_app_dirs() ->
     ["apps/*"] ++
     case is_enterprise() of
@@ -242,7 +262,6 @@ relx_apps(ReleaseType) ->
     , compiler
     , runtime_tools
     , cuttlefish
-    , quicer
     , emqx
     , {mnesia, load}
     , {ekka, load}
@@ -263,6 +282,7 @@ relx_apps(ReleaseType) ->
     , emqx_retainer
     , emqx_statsd
     ]
+    ++ [quicer || is_quicer_supported()]
     ++ [emqx_telemetry || not is_enterprise()]
     ++ [emqx_license || is_enterprise()]
     ++ [bcrypt || provide_bcrypt_release(ReleaseType)]