From 7a44746492451754394a575ceae33fd5f64986ee Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Apr 2024 17:01:13 -0300 Subject: [PATCH 01/11] fix(rule events): don't list validation failures in CE Fixes https://emqx.atlassian.net/browse/EMQX-12284 --- .../emqx_rule_engine/src/emqx_rule_events.erl | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 90bede778..1ff5146e1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -675,7 +675,6 @@ event_info() -> event_info_message_deliver(), event_info_message_acked(), event_info_message_dropped(), - event_info_message_validation_failed(), event_info_client_connected(), event_info_client_disconnected(), event_info_client_connack(), @@ -684,7 +683,27 @@ event_info() -> event_info_session_unsubscribed(), event_info_delivery_dropped(), event_info_bridge_mqtt() + ] ++ ee_event_info(). + +-if(?EMQX_RELEASE_EDITION == ee). +%% ELSE (?EMQX_RELEASE_EDITION == ee). +event_info_message_validation_failed() -> + event_info_common( + 'message.validation_failed', + {<<"message validation failed">>, <<"TODO"/utf8>>}, + {<<"messages that do not pass configured validations">>, <<"TODO"/utf8>>}, + <<"SELECT * FROM \"$events/message_validation_failed\" WHERE topic =~ 't/#'">> + ). +ee_event_info() -> + [ + event_info_message_validation_failed() ]. +-else. +%% END (?EMQX_RELEASE_EDITION == ee). + +ee_event_info() -> + []. +-endif. event_info_message_publish() -> event_info_common( @@ -715,13 +734,6 @@ event_info_message_dropped() -> <<"消息在转发的过程中被丢弃,一般是由于没有订阅者"/utf8>>}, <<"SELECT * FROM \"$events/message_dropped\" WHERE topic =~ 't/#'">> ). -event_info_message_validation_failed() -> - event_info_common( - 'message.validation_failed', - {<<"message validation failed">>, <<"TODO"/utf8>>}, - {<<"messages that do not pass configured validations">>, <<"TODO"/utf8>>}, - <<"SELECT * FROM \"$events/message_validation_failed\" WHERE topic =~ 't/#'">> - ). event_info_delivery_dropped() -> event_info_common( 'delivery.dropped', @@ -793,9 +805,6 @@ event_info_common(Event, {TitleEN, TitleZH}, {DescrEN, DescrZH}, SqlExam) -> test_columns('message.dropped') -> [{<<"reason">>, [<<"no_subscribers">>, <<"the reason of dropping">>]}] ++ test_columns('message.publish'); -test_columns('message.validation_failed') -> - [{<<"validation">>, <<"myvalidation">>}] ++ - test_columns('message.publish'); test_columns('message.publish') -> [ {<<"clientid">>, [<<"c_emqx">>, <<"the clientid of the sender">>]}, @@ -859,7 +868,21 @@ test_columns(<<"$bridges/mqtt", _/binary>>) -> {<<"topic">>, [<<"t/a">>, <<"the topic of the MQTT message">>]}, {<<"qos">>, [1, <<"the QoS of the MQTT message">>]}, {<<"payload">>, [<<"{\"msg\": \"hello\"}">>, <<"the payload of the MQTT message">>]} - ]. + ]; +test_columns(Event) -> + ee_test_columns(Event). + +-if(?EMQX_RELEASE_EDITION == ee). +ee_test_columns('message.validation_failed') -> + [{<<"validation">>, <<"myvalidation">>}] ++ + test_columns('message.publish'). +%% ELSE (?EMQX_RELEASE_EDITION == ee). +-else. +-spec ee_test_columns(_) -> no_return(). +ee_test_columns(Event) -> + error({unknown_event, Event}). +%% END (?EMQX_RELEASE_EDITION == ee). +-endif. columns_with_exam('message.publish') -> [ From 3386f565aaf31bda21c890f19a42fc9684da4388 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 11:21:59 +0200 Subject: [PATCH 02/11] build: replace cloud release type with standard and pass it as argument --- mix.exs | 36 ++++++----- rebar.config.erl | 91 +++++++++++++++------------ scripts/check-elixir-applications.exs | 6 +- scripts/check_missing_reboot_apps.exs | 2 +- 4 files changed, 74 insertions(+), 61 deletions(-) diff --git a/mix.exs b/mix.exs index d77bfc99c..8fb920c87 100644 --- a/mix.exs +++ b/mix.exs @@ -113,8 +113,9 @@ defmodule EMQXUmbrella.MixProject do set_emqx_app_system_env(apps, profile_info, version) end - defp umbrella_apps(profile_info) do - enterprise_apps = enterprise_umbrella_apps() + defp umbrella_apps(profile_info = %{release_type: release_type}) do + enterprise_apps = enterprise_umbrella_apps(release_type) + excluded_apps = excluded_apps(release_type) "apps/*" |> Path.wildcard() @@ -140,10 +141,11 @@ defmodule EMQXUmbrella.MixProject do false end end) + |> Enum.reject(fn {app, _} -> app in excluded_apps end) end - defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do - Enum.map(enterprise_umbrella_apps(), fn app_name -> + defp enterprise_apps(_profile_info = %{release_type: release_type, edition_type: :enterprise}) do + Enum.map(enterprise_umbrella_apps(release_type), fn app_name -> path = "apps/#{app_name}" {app_name, path: path, manager: :rebar3, override: true} end) @@ -154,7 +156,7 @@ defmodule EMQXUmbrella.MixProject do end # need to remove those when listing `/apps/`... - defp enterprise_umbrella_apps() do + defp enterprise_umbrella_apps(_release_type) do MapSet.new([ :emqx_bridge_kafka, :emqx_bridge_confluent, @@ -304,7 +306,7 @@ defmodule EMQXUmbrella.MixProject do end [ - applications: applications(edition_type), + applications: applications(release_type, edition_type), skip_mode_validation_for: [ :emqx_mix, :emqx_gateway, @@ -344,7 +346,7 @@ defmodule EMQXUmbrella.MixProject do ] end - def applications(edition_type) do + def applications(release_type, edition_type) do {:ok, [ %{ @@ -365,7 +367,7 @@ defmodule EMQXUmbrella.MixProject do business_apps = common_business_apps ++ edition_specific_apps - excluded_apps = excluded_apps() + excluded_apps = excluded_apps(release_type) system_apps = Enum.map(system_apps, fn app -> @@ -380,7 +382,7 @@ defmodule EMQXUmbrella.MixProject do |> Keyword.reject(fn {app, _type} -> app in excluded_apps end) end - defp excluded_apps() do + defp excluded_apps(_release_type) do %{ mnesia_rocksdb: enable_rocksdb?(), quicer: enable_quicer?(), @@ -451,19 +453,19 @@ defmodule EMQXUmbrella.MixProject do } = case Mix.env() do :dev -> - {:cloud, :bin, :community} + {:standard, :bin, :community} :emqx -> - {:cloud, :bin, :community} + {:standard, :bin, :community} :"emqx-enterprise" -> - {:cloud, :bin, :enterprise} + {:standard, :bin, :enterprise} :"emqx-pkg" -> - {:cloud, :pkg, :community} + {:standard, :pkg, :community} :"emqx-enterprise-pkg" -> - {:cloud, :pkg, :enterprise} + {:standard, :pkg, :enterprise} end normalize_env!() @@ -566,7 +568,7 @@ defmodule EMQXUmbrella.MixProject do vm_args_template_path = case release_type do - :cloud -> + _ -> "apps/emqx/etc/vm.args.cloud" end @@ -780,10 +782,10 @@ defmodule EMQXUmbrella.MixProject do defp emqx_description(release_type, edition_type) do case {release_type, edition_type} do - {:cloud, :enterprise} -> + {_, :enterprise} -> "EMQX Enterprise" - {:cloud, :community} -> + {_, :community} -> "EMQX" end end diff --git a/rebar.config.erl b/rebar.config.erl index 35f76b187..6493a2428 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -157,17 +157,24 @@ is_rocksdb_supported(_) -> not is_build_without("ROCKSDB"). project_app_dirs() -> - project_app_dirs(get_edition_from_profile_env()). + #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(), + project_app_dirs(Edition, RelType). -project_app_dirs(Edition) -> +project_app_dirs(Edition, RelType) -> IsEnterprise = is_enterprise(Edition), + ExcludedApps = excluded_apps(RelType), UmbrellaApps = [ Path || Path <- filelib:wildcard("apps/*"), - is_community_umbrella_app(Path) orelse IsEnterprise + not project_app_excluded(Path, ExcludedApps) andalso + (is_community_umbrella_app(Path) orelse IsEnterprise) ], UmbrellaApps. +project_app_excluded("apps/" ++ AppStr, ExcludedApps) -> + App = list_to_atom(AppStr), + lists:member(App, ExcludedApps). + plugins() -> [ %{relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}, @@ -196,9 +203,10 @@ test_deps() -> ]. common_compile_opts() -> - common_compile_opts(get_edition_from_profile_env(), undefined). + #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(), + common_compile_opts(Edition, RelType, undefined). -common_compile_opts(Edition, Vsn) -> +common_compile_opts(Edition, _RelType, Vsn) -> % always include debug_info [ debug_info, @@ -224,71 +232,72 @@ warn_profile_env() -> get_edition_from_profile_env() -> case os:getenv("PROFILE") of "emqx-enterprise" ++ _ -> - ee; + #{edition => ee, reltype => standard}; "emqx" ++ _ -> - ce; + #{edition => ce, reltype => standard}; false -> - ee; + #{edition => ee, reltype => standard}; V -> io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]), exit(bad_PROFILE) end. -prod_compile_opts(Edition, Vsn) -> +prod_compile_opts(Edition, RelType, Vsn) -> [ compressed, deterministic, warnings_as_errors - | common_compile_opts(Edition, Vsn) + | common_compile_opts(Edition, RelType, Vsn) ]. prod_overrides() -> [{add, [{erl_opts, [deterministic]}]}]. profiles() -> - case get_edition_from_profile_env() of + #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(), + case Edition of ee -> - profiles_ee(); + profiles_ee(RelType); ce -> - profiles_ce() - end ++ profiles_dev(). + profiles_ce(RelType) + end ++ profiles_dev(RelType). -profiles_ce() -> +profiles_ce(RelType) -> Vsn = get_vsn(emqx), [ {'emqx', [ - {erl_opts, prod_compile_opts(ce, Vsn)}, - {relx, relx(Vsn, cloud, bin, ce)}, + {erl_opts, prod_compile_opts(ce, RelType, Vsn)}, + {relx, relx(Vsn, RelType, bin, ce)}, {overrides, prod_overrides()}, - {project_app_dirs, project_app_dirs(ce)} + {project_app_dirs, project_app_dirs(ce, RelType)} ]}, {'emqx-pkg', [ - {erl_opts, prod_compile_opts(ce, Vsn)}, - {relx, relx(Vsn, cloud, pkg, ce)}, + {erl_opts, prod_compile_opts(ce, RelType, Vsn)}, + {relx, relx(Vsn, RelType, pkg, ce)}, {overrides, prod_overrides()}, - {project_app_dirs, project_app_dirs(ce)} + {project_app_dirs, project_app_dirs(ce, RelType)} ]} ]. -profiles_ee() -> +profiles_ee(RelType) -> Vsn = get_vsn('emqx-enterprise'), [ {'emqx-enterprise', [ - {erl_opts, prod_compile_opts(ee, Vsn)}, - {relx, relx(Vsn, cloud, bin, ee)}, + {erl_opts, prod_compile_opts(ee, RelType, Vsn)}, + {relx, relx(Vsn, RelType, bin, ee)}, {overrides, prod_overrides()}, - {project_app_dirs, project_app_dirs(ee)} + {project_app_dirs, project_app_dirs(ee, RelType)} ]}, {'emqx-enterprise-pkg', [ - {erl_opts, prod_compile_opts(ee, Vsn)}, - {relx, relx(Vsn, cloud, pkg, ee)}, + {erl_opts, prod_compile_opts(ee, RelType, Vsn)}, + {relx, relx(Vsn, RelType, pkg, ee)}, {overrides, prod_overrides()}, - {project_app_dirs, project_app_dirs(ee)} + {project_app_dirs, project_app_dirs(ee, RelType)} ]} ]. %% EE has more files than CE, always test/check with EE options. -profiles_dev() -> +profiles_dev(_RelType) -> [ {check, [ {erl_opts, common_compile_opts()}, @@ -302,7 +311,7 @@ profiles_dev() -> ]} ]. -%% RelType: cloud (full size) +%% RelType: standard %% PkgType: bin | pkg %% Edition: ce (opensource) | ee (enterprise) relx(Vsn, RelType, PkgType, Edition) -> @@ -341,10 +350,10 @@ relform() -> Other -> Other end. -emqx_description(cloud, ee) -> "EMQX Enterprise"; -emqx_description(cloud, ce) -> "EMQX". +emqx_description(_, ee) -> "EMQX Enterprise"; +emqx_description(_, ce) -> "EMQX". -overlay_vars(cloud, PkgType, Edition) -> +overlay_vars(_RelType, PkgType, Edition) -> [ {emqx_default_erlang_cookie, "emqxsecretcookie"} ] ++ @@ -411,15 +420,16 @@ relx_apps(ReleaseType, Edition) -> ce -> CEBusinessApps end, BusinessApps = CommonBusinessApps ++ EditionSpecificApps, + ExcludedApps = excluded_apps(ReleaseType), Apps = - (SystemApps ++ + ([App || App <- SystemApps, not lists:member(App, ExcludedApps)] ++ %% EMQX starts the DB and the business applications: - [{App, load} || App <- DBApps] ++ + [{App, load} || App <- DBApps, not lists:member(App, ExcludedApps)] ++ [emqx_machine] ++ - [{App, load} || App <- BusinessApps]), - lists:foldl(fun proplists:delete/2, Apps, excluded_apps(ReleaseType)). + [{App, load} || App <- BusinessApps, not lists:member(App, ExcludedApps)]), + Apps. -excluded_apps(_ReleaseType) -> +excluded_apps(_RelType) -> OptionalApps = [ {quicer, is_quicer_supported()}, {jq, is_jq_supported()}, @@ -489,7 +499,7 @@ emqx_etc_overlay(ReleaseType) -> emqx_etc_overlay_per_rel(ReleaseType) ++ emqx_etc_overlay(). -emqx_etc_overlay_per_rel(cloud) -> +emqx_etc_overlay_per_rel(_RelType) -> [{"{{base_dir}}/lib/emqx/etc/vm.args.cloud", "etc/vm.args"}]. emqx_etc_overlay() -> @@ -543,10 +553,11 @@ dialyzer(Config) -> end, AppNames = app_names(), + ExcludedApps = excluded_apps(standard), KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], - AppsToExclude = AppNames -- KnownApps, + AppsToExclude = ExcludedApps -- KnownApps ++ AppNames, Extra = [system_monitor, tools, covertool] ++ diff --git a/scripts/check-elixir-applications.exs b/scripts/check-elixir-applications.exs index 42c838199..80a252410 100755 --- a/scripts/check-elixir-applications.exs +++ b/scripts/check-elixir-applications.exs @@ -22,7 +22,7 @@ defmodule CheckElixirApplications do env: [{"DEBUG", "1"}] ) - mix_apps = mix_applications(inputs.edition_type) + mix_apps = mix_applications(inputs.release_type, inputs.edition_type) rebar_apps = rebar_applications(profile) results = diff_apps(mix_apps, rebar_apps) @@ -70,8 +70,8 @@ defmodule CheckElixirApplications do end end - defp mix_applications(edition_type) do - EMQXUmbrella.MixProject.applications(edition_type) + defp mix_applications(release_type, edition_type) do + EMQXUmbrella.MixProject.applications(release_type, edition_type) end defp rebar_applications(profile) do diff --git a/scripts/check_missing_reboot_apps.exs b/scripts/check_missing_reboot_apps.exs index 91d4b39ea..07462a8ad 100755 --- a/scripts/check_missing_reboot_apps.exs +++ b/scripts/check_missing_reboot_apps.exs @@ -14,7 +14,7 @@ profile = Mix.env() # need to use this information because we might have compiled all # applications in the test profile, and thus filter what's in the # release lib directory. -rel_apps = MixProject.applications(inputs.edition_type) +rel_apps = MixProject.applications(inputs.release_type, inputs.edition_type) apps = rel_apps From 035b8480b0954ab20dc9caccee19685859aaef22 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 11:22:49 +0200 Subject: [PATCH 03/11] chore(emqx_conf): extract data dirs to sync to a macro --- apps/emqx_conf/src/emqx_conf_app.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index be42b27aa..75e106c54 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -81,11 +81,13 @@ get_override_config_file() -> end end. +-define(DATA_DIRS, ["authz", "certs"]). + sync_data_from_node() -> Dir = emqx:data_dir(), - TargetDirs = lists:filter(fun(Type) -> filelib:is_dir(filename:join(Dir, Type)) end, [ - "authz", "certs" - ]), + TargetDirs = lists:filter( + fun(Type) -> filelib:is_dir(filename:join(Dir, Type)) end, ?DATA_DIRS + ), Name = "data.zip", case zip:zip(Name, TargetDirs, [memory, {cwd, Dir}]) of {ok, {Name, Bin}} -> {ok, Bin}; From 6c6580bc0872ec6811d8154a9ccb68e75965a9e7 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 11:23:16 +0200 Subject: [PATCH 04/11] build(docker): do not assign default value to the DEBUG var --- deploy/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 1bcc9f30c..56ed7b214 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -16,7 +16,7 @@ FROM builder_${SOURCE_TYPE} as builder ARG PROFILE=emqx ARG IS_ELIXIR=no -ARG DEBUG=0 +ARG DEBUG ENV EMQX_RELUP=false ENV EMQX_REL_FORM='docker' From 26d0c41f025d653a4c4241e583d7f2ff12e2eeb3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 11:24:00 +0200 Subject: [PATCH 05/11] ci: small adjustments in workflows --- .github/workflows/build_docker_for_test.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index 7ae32acfe..8090bcc22 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -28,7 +28,7 @@ jobs: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} env: EMQX_NAME: ${{ matrix.profile }} - PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} + PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }} OTP_VSN: ${{ inputs.otp_vsn }} ELIXIR_VSN: ${{ inputs.elixir_vsn }} diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 8b9261069..9f47ebfa5 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -14,32 +14,30 @@ jobs: check_deps_integrity: runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} container: ${{ inputs.builder }} + env: + MIX_ENV: ${{ matrix.profile }} + PROFILE: ${{ matrix.profile }} + strategy: + matrix: + profile: + - emqx-enterprise steps: - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: make ensure-rebar3 - run: ./scripts/check-deps-integrity.escript - name: Setup mix - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise run: | mix local.hex --force mix local.rebar --force mix deps.get - run: ./scripts/check-elixir-deps-discrepancies.exs - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - run: ./scripts/check-elixir-applications.exs - env: - MIX_ENV: emqx-enterprise - PROFILE: emqx-enterprise - name: Upload produced lock files uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: failure() with: - name: produced_lock_files + name: ${{ matrix.profile }}_produced_lock_files path: | mix.lock rebar.lock From 9ab2eef56c106ed5ef45401463b6a462978d9b37 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 11:25:05 +0200 Subject: [PATCH 06/11] ci: stop building packages for macos 12, add macos 14 --- .github/workflows/build_packages.yaml | 3 +-- .github/workflows/build_slim_packages.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fecb55f3f..e208cfb96 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -76,9 +76,8 @@ jobs: profile: - ${{ inputs.profile }} os: - - macos-12 - - macos-12-arm64 - macos-13 + - macos-14 otp: - ${{ inputs.otp_vsn }} runs-on: ${{ matrix.os }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index c36a6ebd3..f2f45f74a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -111,7 +111,7 @@ jobs: otp: - ${{ inputs.otp_vsn }} os: - - macos-12-arm64 + - macos-14 runs-on: ${{ matrix.os }} env: From 6c3c97d58c232dddc2f6afdaff09ff5e22dd7f65 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 14:01:16 +0200 Subject: [PATCH 07/11] docs: add changelog --- changes/ce/breaking-12957.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/breaking-12957.en.md diff --git a/changes/ce/breaking-12957.en.md b/changes/ce/breaking-12957.en.md new file mode 100644 index 000000000..7fcdd1335 --- /dev/null +++ b/changes/ce/breaking-12957.en.md @@ -0,0 +1,2 @@ +Stop building packages for macOS 12. +Start building packages for macOS 14 (Apple Silicon). From f9211aadd81f561aeed85c2bdd6951179e0fe556 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 14:10:36 +0200 Subject: [PATCH 08/11] ci: switch scheduled workflows from release-56 to release-57 --- .github/workflows/build_packages_cron.yaml | 3 +-- .github/workflows/codeql.yaml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index c3e76d1b2..1ccddd7fb 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -24,9 +24,8 @@ jobs: matrix: profile: - ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2'] - - ['emqx-enterprise', 'release-56', '5.3-4:1.15.7-26.2.1-2'] + - ['emqx', 'release-57', '5.3-4:1.15.7-26.2.1-2'] os: - - debian10 - ubuntu22.04 - amzn2023 diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 5c2a9a4d3..d493c00d9 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -24,7 +24,6 @@ jobs: matrix: branch: - master - - release-56 - release-57 language: - cpp From adb9f6822824de814796d3094d6b3a508e9ace25 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 30 Apr 2024 18:33:58 +0200 Subject: [PATCH 09/11] ci: add mix deps.tree step to check_deps_integrity workflow --- .github/workflows/check_deps_integrity.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 9f47ebfa5..d5584ecb5 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -31,6 +31,8 @@ jobs: mix local.hex --force mix local.rebar --force mix deps.get + - name: print mix dependency tree + run: mix deps.tree - run: ./scripts/check-elixir-deps-discrepancies.exs - run: ./scripts/check-elixir-applications.exs - name: Upload produced lock files From 92359fd3054e59069a51b9c30399da8036505bce Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 2 May 2024 10:24:53 +0200 Subject: [PATCH 10/11] build(rebar.config.erl): adjust AppsToExclude for dyalizer --- rebar.config.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 6493a2428..e2e86bc29 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -553,11 +553,9 @@ dialyzer(Config) -> end, AppNames = app_names(), - ExcludedApps = excluded_apps(standard), - KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], - - AppsToExclude = ExcludedApps -- KnownApps ++ AppNames, + ExcludedApps = excluded_apps(standard), + AppsToExclude = ExcludedApps ++ (AppNames -- KnownApps), Extra = [system_monitor, tools, covertool] ++ From cc025ce4ae771cc508153c1cfd5b703d699dec7e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 2 May 2024 10:29:00 +0200 Subject: [PATCH 11/11] feat: publish packages for Ubuntu 24.04 --- .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .github/workflows/_pr_entrypoint.yaml | 10 +++++----- .github/workflows/_push-entrypoint.yaml | 10 +++++----- .github/workflows/build_and_push_docker_images.yaml | 2 +- .github/workflows/build_packages.yaml | 3 ++- .github/workflows/build_packages_cron.yaml | 4 ++-- .github/workflows/build_slim_packages.yaml | 4 ++-- .github/workflows/codeql.yaml | 2 +- .github/workflows/performance_test.yaml | 2 +- .github/workflows/release.yaml | 2 ++ Makefile | 2 +- build | 2 +- changes/ce/breaking-12957.en.md | 1 - changes/ce/feat-12957.en.md | 2 ++ deploy/docker/Dockerfile | 2 +- scripts/buildx.sh | 4 ++-- scripts/pr-sanity-checks.sh | 2 +- scripts/relup-test/start-relup-test-cluster.sh | 2 +- 19 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 changes/ce/feat-12957.en.md diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 158db93f1..45dfc3fa0 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -18,7 +18,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04 container_name: kdc.emqx.net expose: - 88 # kdc diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index d354a9281..ce98a7ced 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04} env_file: - credentials.env - conf.env diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 779b73d16..4c8349f3f 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -20,15 +20,15 @@ permissions: jobs: sanity-checks: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04" outputs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} - builder: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" - builder_vsn: "5.3-4" + builder: "ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04" + builder_vsn: "5.3-5" otp_vsn: "26.2.1-2" elixir_vsn: "1.15.7" @@ -95,12 +95,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-4", + builder: "5.3-5", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-4", + builder: "5.3-5", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index b141989d5..24ac94db8 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -23,7 +23,7 @@ env: jobs: prepare: runs-on: ubuntu-22.04 - container: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' + container: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04' outputs: profile: ${{ steps.parse-git-ref.outputs.profile }} release: ${{ steps.parse-git-ref.outputs.release }} @@ -31,8 +31,8 @@ jobs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} - builder: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' - builder_vsn: '5.3-4' + builder: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04' + builder_vsn: '5.3-5' otp_vsn: '26.2.1-2' elixir_vsn: '1.15.7' @@ -62,12 +62,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-4", + builder: "5.3-5", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-4", + builder: "5.3-5", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 583b0b42a..dbca3fe38 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -61,7 +61,7 @@ on: builder_vsn: required: false type: string - default: '5.3-4' + default: '5.3-5' permissions: contents: read diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index e208cfb96..7ccf975b6 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -63,7 +63,7 @@ on: builder_vsn: required: false type: string - default: '5.3-4' + default: '5.3-5' permissions: contents: read @@ -110,6 +110,7 @@ jobs: profile: - ${{ inputs.profile }} os: + - ubuntu24.04 - ubuntu22.04 - ubuntu20.04 - ubuntu18.04 diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 1ccddd7fb..8573b6e11 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: profile: - - ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2'] - - ['emqx', 'release-57', '5.3-4:1.15.7-26.2.1-2'] + - ['emqx', 'master', '5.3-5:1.15.7-26.2.1-2'] + - ['emqx', 'release-57', '5.3-5:1.15.7-26.2.1-2'] os: - ubuntu22.04 - amzn2023 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index f2f45f74a..7c8e3aaa4 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,11 +27,11 @@ on: builder: required: false type: string - default: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' + default: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04' builder_vsn: required: false type: string - default: '5.3-4' + default: '5.3-5' otp_vsn: required: false type: string diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index d493c00d9..d0b15b119 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -17,7 +17,7 @@ jobs: actions: read security-events: write container: - image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04 strategy: fail-fast: false diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index fc524a5f9..413ac3728 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -26,7 +26,7 @@ jobs: prepare: runs-on: ubuntu-latest if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu20.04 outputs: BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 34a0e44e5..f40db0039 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,6 +112,8 @@ jobs: push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" + push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-amd64.deb" + push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-arm64.deb" push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm" push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm" push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm" diff --git a/Makefile b/Makefile index 83d041260..f656e74ba 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12 export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 diff --git a/build b/build index 18880ca04..b8deeaa73 100755 --- a/build +++ b/build @@ -395,7 +395,7 @@ function is_ecr_and_enterprise() { ## Build the default docker image based on debian 12. make_docker() { - local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-4}" + local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-5}" local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}" local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}" local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}" diff --git a/changes/ce/breaking-12957.en.md b/changes/ce/breaking-12957.en.md index 7fcdd1335..2f50e3f03 100644 --- a/changes/ce/breaking-12957.en.md +++ b/changes/ce/breaking-12957.en.md @@ -1,2 +1 @@ Stop building packages for macOS 12. -Start building packages for macOS 14 (Apple Silicon). diff --git a/changes/ce/feat-12957.en.md b/changes/ce/feat-12957.en.md new file mode 100644 index 000000000..a18e3ca89 --- /dev/null +++ b/changes/ce/feat-12957.en.md @@ -0,0 +1,2 @@ +Start building packages for macOS 14 (Apple Silicon). +Start building packages for Ubuntu 24.04 Noble Numbat (LTS). diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 56ed7b214..fcff6984e 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12 ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim ARG SOURCE_TYPE=src # tgz diff --git a/scripts/buildx.sh b/scripts/buildx.sh index dbe20d501..3c358a934 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 +## --builder ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12 set -euo pipefail @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12" + echo " E.g. ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12" } die() { diff --git a/scripts/pr-sanity-checks.sh b/scripts/pr-sanity-checks.sh index 61265c7e5..be02a337d 100755 --- a/scripts/pr-sanity-checks.sh +++ b/scripts/pr-sanity-checks.sh @@ -12,7 +12,7 @@ if ! type "yq" > /dev/null; then exit 1 fi -EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-4} +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-5} EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.2.1-2} EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index 862c5377b..557fbbff9 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE"