From a54e108296c0d434d346e4905f7160d2ae10370c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 27 Dec 2021 11:25:35 -0300 Subject: [PATCH] ci(mix): add application mode check This adds a CI check to ensure that applications and their modes are in sync between Elixir and Rebar release builds. --- .github/workflows/elixir_apps_check.yaml | 48 ++++ mix.exs | 94 ++++---- scripts/check-elixir-applications.exs | 282 +++++++++++++++++++++++ 3 files changed, 383 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/elixir_apps_check.yaml create mode 100755 scripts/check-elixir-applications.exs diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml new file mode 100644 index 000000000..04bc8c024 --- /dev/null +++ b/.github/workflows/elixir_apps_check.yaml @@ -0,0 +1,48 @@ +--- + +name: Check Elixir Release Applications + +on: [pull_request] + +jobs: + elixir_apps_check: + runs-on: ubuntu-20.04 + container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + + strategy: + fail-fast: false + matrix: + release_type: + - cloud + - edge + package_type: + - bin + - pkg + edition_type: + - community + - enterprise + exclude: + - release_type: edge + package_type: bin + edition_type: enterprise + - release_type: edge + package_type: pkg + edition_type: enterprise + + steps: + - name: install + run: apk add make bash curl git + - name: Checkout + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - name: ensure rebar + run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1 + - name: check applications + run: ./scripts/check-elixir-applications.exs + env: + EMQX_RELEASE_TYPE: ${{ matrix.release_type }} + EMQX_PACKAGE_TYPE: ${{ matrix.package_type }} + EMQX_EDITION_TYPE: ${{ matrix.edition_type }} + +... diff --git a/mix.exs b/mix.exs index c5efda930..101460929 100644 --- a/mix.exs +++ b/mix.exs @@ -106,47 +106,7 @@ defmodule EMQXUmbrella.MixProject do end [ - applications: [ - logger: :permanent, - crypto: :permanent, - public_key: :permanent, - asn1: :permanent, - syntax_tools: :permanent, - ssl: :permanent, - os_mon: :permanent, - inets: :permanent, - compiler: :permanent, - runtime_tools: :permanent, - hocon: :load, - emqx: :load, - emqx_conf: :load, - emqx_machine: :permanent, - mria: :load, - mnesia: :load, - ekka: :load, - emqx_plugin_libs: :load, - esasl: :load, - emqx_http_lib: :permanent, - emqx_resource: :permanent, - emqx_connector: :permanent, - emqx_authn: :permanent, - emqx_authz: :permanent, - emqx_auto_subscribe: :permanent, - emqx_gateway: :permanent, - emqx_exhook: :permanent, - emqx_bridge: :permanent, - emqx_rule_engine: :permanent, - emqx_modules: :permanent, - emqx_management: :permanent, - emqx_dashboard: :permanent, - emqx_statsd: :permanent, - emqx_retainer: :permanent, - emqx_prometheus: :permanent, - emqx_psk: :permanent, - emqx_slow_subs: :permanent, - emqx_plugins: :permanent, - emqx_mix: :none - ], + applications: applications(release_type), skip_mode_validation_for: [ :emqx_gateway, :emqx_dashboard, @@ -167,6 +127,58 @@ defmodule EMQXUmbrella.MixProject do ] end + def applications(release_type) do + [ + logger: :permanent, + crypto: :permanent, + public_key: :permanent, + asn1: :permanent, + syntax_tools: :permanent, + ssl: :permanent, + os_mon: :permanent, + inets: :permanent, + compiler: :permanent, + runtime_tools: :permanent, + hocon: :load, + emqx: :load, + emqx_conf: :load, + emqx_machine: :permanent, + mria: :load, + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + esasl: :load, + observer_cli: :permanent, + system_monitor: :permanent, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_retainer: :permanent, + emqx_statsd: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent, + emqx_mix: :none + ] ++ + if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ + if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ + if(release_type == :cloud, + do: [xmerl: :permanent, observer: :load], + else: [] + ) + end + defp read_inputs() do release_type = read_enum_env_var( diff --git a/scripts/check-elixir-applications.exs b/scripts/check-elixir-applications.exs new file mode 100755 index 000000000..3ada6d779 --- /dev/null +++ b/scripts/check-elixir-applications.exs @@ -0,0 +1,282 @@ +#!/usr/bin/env elixir + +defmodule CheckElixirApplications do + @default_applications [:kernel, :stdlib, :sasl] + + def main() do + {:ok, _} = Application.ensure_all_started(:mix) + inputs = read_inputs() + # produce `rebar.config.rendered` to consult + profile = profile_of(inputs) + + File.cwd!() + |> Path.join("rebar3") + |> System.cmd(["as", to_string(profile)], + env: [{"DEBUG", "1"}] + ) + + File.cwd!() + |> Path.join("mix.exs") + |> Code.compile_file() + + mix_apps = mix_applications(inputs.release_type) + rebar_apps = rebar_applications(profile) + results = diff_apps(mix_apps, rebar_apps) + + report_discrepancy( + results[:missing_apps], + "* There are missing applications in the Elixir release", + fn %{app: app, mode: mode, after: last_app} -> + IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}") + end + ) + + report_discrepancy( + results[:different_modes], + "* There are applications with different application modes in the Elixir release", + fn %{app: app, rebar_mode: rebar_mode, mix_mode: mix_mode} -> + IO.puts( + " * #{inspect(app)} should have mode #{inspect(rebar_mode)}, but it has mode #{inspect(mix_mode)}" + ) + end + ) + + report_discrepancy( + results[:different_positions], + "* There are applications in the Elixir release in the wrong order", + fn %{app: app, mode: mode, after: last_app} -> + IO.puts(" * #{app}: #{inspect(mode)} should be placed after #{inspect(last_app)}") + end + ) + + success? = + results + |> Map.take([:missing_apps, :different_modes, :different_positions]) + |> Map.values() + |> Enum.concat() + |> Enum.empty?() + + if not success? do + System.halt(1) + else + IO.puts( + IO.ANSI.green() <> + "Mix and Rebar applications OK!" <> + IO.ANSI.reset() + ) + end + end + + defp mix_applications(release_type) do + EMQXUmbrella.MixProject.applications(release_type) + end + + defp rebar_applications(profile) do + {:ok, props} = + File.cwd!() + |> Path.join("rebar.config.rendered") + |> :file.consult() + + props[:profiles][profile][:relx] + |> Enum.find(&(elem(&1, 0) == :release)) + |> elem(2) + |> Enum.map(fn + app when is_atom(app) -> + {app, :permanent} + + {app, mode} -> + {app, mode} + end) + |> Enum.reject(fn {app, _mode} -> + # Elixir already includes those implicitly + app in @default_applications + end) + end + + defp profile_of(%{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + }) do + case {release_type, package_type, edition_type} do + {:cloud, :bin, :community} -> + :emqx + + {:cloud, :pkg, :community} -> + :"emqx-pkg" + + {:cloud, :bin, :enterprise} -> + :"emqx-enterprise" + + {:cloud, :pkg, :enterprise} -> + :"emqx-enterprise-pkg" + + {:edge, :bin, :community} -> + :"emqx-edge" + + {:edge, :pkg, :community} -> + :"emqx-edge-pkg" + end + end + + defp read_inputs() do + release_type = + read_enum_env_var( + "EMQX_RELEASE_TYPE", + [:cloud, :edge], + :cloud + ) + + package_type = + read_enum_env_var( + "EMQX_PACKAGE_TYPE", + [:bin, :pkg], + :bin + ) + + edition_type = + read_enum_env_var( + "EMQX_EDITION_TYPE", + [:community, :enterprise], + :community + ) + + %{ + release_type: release_type, + package_type: package_type, + edition_type: edition_type + } + end + + defp read_enum_env_var(env_var, allowed_values, default_value) do + case System.fetch_env(env_var) do + :error -> + default_value + + {:ok, raw_value} -> + value = + raw_value + |> String.downcase() + |> String.to_atom() + + if value not in allowed_values do + Mix.raise(""" + Invalid value #{raw_value} for variable #{env_var}. + Allowed values are: #{inspect(allowed_values)} + """) + end + + value + end + end + + defp diff_apps(mix_apps, rebar_apps) do + app_names = Keyword.keys(rebar_apps) + mix_apps = Keyword.filter(mix_apps, fn {app, _mode} -> app in app_names end) + + acc = %{ + mix_apps: mix_apps, + missing_apps: [], + different_positions: [], + different_modes: [], + last_app: nil + } + + Enum.reduce( + rebar_apps, + acc, + fn + {rebar_app, rebar_mode}, acc = %{mix_apps: [], last_app: last_app} -> + missing_app = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:missing_apps, &[missing_app | &1]) + |> Map.put(:last_app, rebar_app) + + {rebar_app, rebar_mode}, + acc = %{mix_apps: [{mix_app, mix_mode} | rest], last_app: last_app} -> + case {rebar_app, rebar_mode} do + {^mix_app, ^mix_mode} -> + acc + |> Map.put(:mix_apps, rest) + |> Map.put(:last_app, rebar_app) + + {^mix_app, _mode} -> + different_mode = %{ + app: rebar_app, + rebar_mode: rebar_mode, + mix_mode: mix_mode + } + + acc + |> Map.put(:mix_apps, rest) + |> Map.update!(:different_modes, &[different_mode | &1]) + |> Map.put(:last_app, rebar_app) + + {_app, _mode} -> + case Keyword.pop(rest, rebar_app) do + {nil, _} -> + missing_app = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:missing_apps, &[missing_app | &1]) + |> Map.put(:last_app, rebar_app) + + {^rebar_mode, rest} -> + different_position = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.update!(:different_positions, &[different_position | &1]) + |> Map.put(:last_app, rebar_app) + |> Map.put(:mix_apps, [{mix_app, mix_mode} | rest]) + + {mode, rest} -> + different_mode = %{ + app: rebar_app, + rebar_mode: rebar_mode, + mix_mode: mode + } + + different_position = %{ + app: rebar_app, + mode: rebar_mode, + after: last_app + } + + acc + |> Map.put(:mix_apps, [{mix_app, mix_mode} | rest]) + |> Map.update!(:different_modes, &[different_mode | &1]) + |> Map.update!(:different_positions, &[different_position | &1]) + |> Map.put(:last_app, rebar_app) + end + end + end + ) + end + + defp report_discrepancy(diffs, header, line_fn) do + unless Enum.empty?(diffs) do + IO.puts(IO.ANSI.red() <> header) + + diffs + |> Enum.reverse() + |> Enum.each(line_fn) + + IO.puts(IO.ANSI.reset()) + end + end +end + +CheckElixirApplications.main()