From cc652c33435930cf9b10ee84b6a7c05581557a8c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 24 Dec 2021 14:06:51 -0300 Subject: [PATCH] ci(mix): add elixir deps check Checks whether there are discrepancies between rebar3 and mix deps. --- .github/workflows/elixir_deps_check.yaml | 22 +++++ apps/emqx_retainer/rebar.config | 2 +- mix.exs | 23 +++-- mix.lock | 18 ++-- rebar.config | 4 +- scripts/check-elixir-deps-discrepancies.exs | 101 ++++++++++++++++++++ 6 files changed, 147 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/elixir_deps_check.yaml create mode 100755 scripts/check-elixir-deps-discrepancies.exs diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml new file mode 100644 index 000000000..be637072f --- /dev/null +++ b/.github/workflows/elixir_deps_check.yaml @@ -0,0 +1,22 @@ +--- + +name: Elixir Dependency Version Check + +on: [pull_request] + +jobs: + elixir_deps_check: + runs-on: ubuntu-20.04 + container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0 + + steps: + - name: Checkout + uses: actions/checkout@v2.4.0 + - name: install + run: apk add make bash curl git + - name: ensure rebar + run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1 + - name: check elixir deps + run: ./scripts/check-elixir-deps-discrepancies.exs + +... diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 8ab1ad7b0..e5b53f1be 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -20,6 +20,6 @@ [{test, [{deps, [ - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.0"}}}]} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}]} ]} ]}. diff --git a/mix.exs b/mix.exs index e19b381c1..c8844ba30 100644 --- a/mix.exs +++ b/mix.exs @@ -24,12 +24,12 @@ defmodule EMQXUmbrella.MixProject do {:lc, github: "qzhuyan/lc", tag: "0.1.2"}, {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"}, - {:gproc, "0.8.0", override: true}, + {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true}, {:mria, github: "emqx/mria", tag: "0.1.5", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.11.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.11.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.1"}, @@ -38,7 +38,7 @@ defmodule EMQXUmbrella.MixProject do {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, - {:system_monitor, github: "klarna-incubator/system_monitor", tag: "2.2.0"}, + {:system_monitor, github: "k32/system_monitor", tag: "2.2.1"}, # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true}, @@ -52,13 +52,19 @@ defmodule EMQXUmbrella.MixProject do {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true}, # in conflict by mongodb and eredis_cluster {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true}, - # in conflict by gun and emqtt - {:cowlib, "2.8.0", override: true}, - # in conflict by cowboy_swagger and cowboy - {:ranch, "1.8.0", override: true}, # in conflict by emqx and observer_cli {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, - {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true} + {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, + # dependencies of dependencies; we choose specific refs to match + # what rebar3 chooses. + # in conflict by gun and emqtt + {:cowlib, + github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true}, + # in conflict by cowboy_swagger and cowboy + {:ranch, + github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, + # in conflict by grpc and eetcd + {:gpb, "4.11.2", override: true} ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep() end @@ -240,6 +246,7 @@ defmodule EMQXUmbrella.MixProject do Path.join(etc, "vm.args"), vm_args_rendered ) + File.write!( Path.join(release.version_path, "vm.args"), vm_args_rendered diff --git a/mix.lock b/mix.lock index 3213601fc..c619b1036 100644 --- a/mix.lock +++ b/mix.lock @@ -3,13 +3,11 @@ "bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]}, "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]}, "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]}, - "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, - "earmark": {:hex, :earmark, "1.4.19", "3854a17305c880cc46305af15fb1630568d23a709aba21aaa996ced082fc29d7", [:mix], [{:earmark_parser, ">= 1.4.18", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "d5a8c9f9e37159a8fdd3ea8437fb4e229eaf56d5129b9a011dc4780a4872079d"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"}, + "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363"]}, "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]}, "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]}, "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]}, - "ekka": {:git, "https://github.com/emqx/ekka.git", "005fd6bb94199dc2ecb4ba03284f253b408e02d9", [tag: "0.11.1"]}, + "ekka": {:git, "https://github.com/emqx/ekka.git", "70f2250e5e968e0c1da64e5b4733c5eb0eb402de", [tag: "0.11.2"]}, "eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]}, "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]}, "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]}, @@ -19,11 +17,10 @@ "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]}, "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]}, "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]}, - "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, "gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]}, "getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"}, - "gpb": {:hex, :gpb, "4.19.2", "e841a1c5d5b82b6fbaddd2b6968c66ad48b34adc3b0d974e55cca653bd28ccc5", [:make, :rebar3], [], "hexpm", "3f4cb7f263e94675a598501a6dc96ec17f93ecb28c29bea3bea3a71c53a3317a"}, - "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"}, + "gpb": {:hex, :gpb, "4.11.2", "a2c05241408310b8bd8dbdfd5e1a419799e45fc1408371eaa0f595023c3b21aa", [:make, :rebar], [], "hexpm", "b355a5982b604d6c044ebb6013e5fe65d30c30d0700d35317e7066691eb6bd61"}, + "gproc": {:git, "https://github.com/uwiger/gproc.git", "ce7397809aca0d6eb3aac6db65953752e47fb511", [tag: "0.8.0"]}, "grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]}, "gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]}, "hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]}, @@ -32,20 +29,17 @@ "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]}, "jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]}, "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"}, "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]}, "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]}, "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]}, "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]}, "mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]}, - "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"}, "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"}, "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]}, "poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]}, "prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]}, "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "ranch": {:git, "https://github.com/ninenines/ranch.git", "a692f44567034dacf5efcaa24a24183788594eb7", [ref: "a692f44567034dacf5efcaa24a24183788594eb7"]}, "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]}, "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"}, "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]}, @@ -54,7 +48,7 @@ "snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]}, "ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]}, "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"}, - "system_monitor": {:git, "https://github.com/klarna-incubator/system_monitor.git", "a106f72e8e2251d90e1d95944c139946dac1a07f", [tag: "2.2.0"]}, + "system_monitor": {:git, "https://github.com/k32/system_monitor.git", "3b4b381bf9503695cd764ecf22067ac6542cee89", [tag: "2.2.1"]}, "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"}, "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]}, } diff --git a/rebar.config b/rebar.config index 1894d817d..9cb84e00b 100644 --- a/rebar.config +++ b/rebar.config @@ -53,7 +53,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.5"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} @@ -62,7 +62,7 @@ , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x - , {system_monitor, {git, "https://github.com/klarna-incubator/system_monitor", {tag, "2.2.0"}}} + , {system_monitor, {git, "https://github.com/k32/system_monitor", {tag, "2.2.1"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} diff --git a/scripts/check-elixir-deps-discrepancies.exs b/scripts/check-elixir-deps-discrepancies.exs new file mode 100755 index 000000000..53f5331bb --- /dev/null +++ b/scripts/check-elixir-deps-discrepancies.exs @@ -0,0 +1,101 @@ +#!/usr/bin/env elixir + +# ensure we have a fresh rebar.lock + +case File.stat("rebar.lock") do + {:ok, _} -> + File.rm!("rebar.lock") + + _ -> + :ok +end +{_, 0} = + File.cwd!() + |> Path.join("rebar3") + |> System.cmd(["tree"], into: IO.stream()) + +{:ok, props} = :file.consult("rebar.lock") + +{[{_, rebar_deps}], [props]} = Enum.split_with(props, &is_tuple/1) + +# dpendencies declared as package versions have a "secondary index" +pkg_idx = + props + |> Keyword.fetch!(:pkg_hash) + |> Map.new() + +rebar_deps = Map.new(rebar_deps, fn {name, ref, _} -> + ref = case ref do + {:pkg, _, _} -> + pkg_idx + |> Map.fetch!(name) + |> String.downcase() + + {:git, _, {:ref, ref}} -> + to_string(ref) + end + + {name, ref} +end) + +{mix_deps, []} = Code.eval_file("mix.lock") + +mix_deps = Map.new(mix_deps, fn {name, ref} -> + ref = case ref do + {:git, _, ref, _} -> + ref + + {:hex, _, _, ref, _, _, _, _} -> + ref + end + + {to_string(name), ref} +end) + +diffs = Enum.reduce(rebar_deps, %{}, fn {name, rebar_ref}, acc -> + mix_ref = mix_deps[name] + + cond do + mix_ref && mix_ref != rebar_ref -> + Map.put(acc, name, {rebar_ref, mix_ref}) + + is_nil(mix_ref) -> + Map.put(acc, name, {rebar_ref, nil}) + + :otherwise -> + acc + end +end) + +if diffs == %{} do + IO.puts( + IO.ANSI.green() <> + "* Mix and Rebar3 dependencies OK!" <> + IO.ANSI.reset() + ) + System.halt(0) +else + IO.puts( + IO.ANSI.red() <> + "* Discrepancies between Elixir and Rebar3 dependencies found!" <> + IO.ANSI.reset() + ) + + Enum.each(diffs, fn {name, {rebar_ref, mix_ref}} -> + IO.puts( + IO.ANSI.red() <> + " * #{name}\n" <> + " * Rebar3 ref: #{rebar_ref}\n" <> + " * Mix ref: #{mix_ref}\n" <> + IO.ANSI.reset() + ) + end) + + IO.puts( + IO.ANSI.red() <> + "Update `mix.exs` to match Rebar3's references (use `overwrite: true` if necessary) and try again" <> + IO.ANSI.reset() + ) + + System.halt(1) +end