ci(mix): add elixir deps check

Checks whether there are discrepancies between rebar3 and mix deps.

.github/workflows/elixir_deps_check.yaml

@@ -0,0 +1,22 @@
+---
+
+name: Elixir Dependency Version Check
+
+on: [pull_request]
+
+jobs:
+  elixir_deps_check:
+    runs-on: ubuntu-20.04
+    container: hexpm/elixir:1.13.1-erlang-24.2-alpine-3.15.0
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2.4.0
+      - name: install
+        run: apk add make bash curl git
+      - name: ensure rebar
+        run: ./scripts/ensure-rebar3.sh 3.16.1-emqx-1
+      - name: check elixir deps
+        run: ./scripts/check-elixir-deps-discrepancies.exs
+
+...

apps/emqx_retainer/rebar.config

@@ -20,6 +20,6 @@
  [{test,
    [{deps,
      [
-      {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.0"}}}]}
+      {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}]}
    ]}
  ]}.

mix.exs

@@ -24,12 +24,12 @@ defmodule EMQXUmbrella.MixProject do
       {:lc, github: "qzhuyan/lc", tag: "0.1.2"},
       {:typerefl, github: "k32/typerefl", tag: "0.8.5", override: true},
       {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"},
-      {:gproc, "0.8.0", override: true},
+      {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true},
       {:mria, github: "emqx/mria", tag: "0.1.5", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.11.1", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.11.2", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.5.1", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.2.7", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.1"},
@@ -38,7 +38,7 @@ defmodule EMQXUmbrella.MixProject do
       {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
-      {:system_monitor, github: "klarna-incubator/system_monitor", tag: "2.2.0"},
+      {:system_monitor, github: "k32/system_monitor", tag: "2.2.1"},
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true},
@@ -52,13 +52,19 @@ defmodule EMQXUmbrella.MixProject do
       {:epgsql, github: "epgsql/epgsql", tag: "4.6.0", override: true},
       # in conflict by mongodb and eredis_cluster
       {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true},
-      # in conflict by gun and emqtt
-      {:cowlib, "2.8.0", override: true},
-      # in conflict by cowboy_swagger and cowboy
-      {:ranch, "1.8.0", override: true},
       # in conflict by emqx and observer_cli
       {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
-      {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}
+      {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
+      # dependencies of dependencies; we choose specific refs to match
+      # what rebar3 chooses.
+      # in conflict by gun and emqtt
+      {:cowlib,
+       github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
+      # in conflict by cowboy_swagger and cowboy
+      {:ranch,
+       github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
+      # in conflict by grpc and eetcd
+      {:gpb, "4.11.2", override: true}
     ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
   end
 
@@ -240,6 +246,7 @@ defmodule EMQXUmbrella.MixProject do
       Path.join(etc, "vm.args"),
       vm_args_rendered
     )
+
     File.write!(
       Path.join(release.version_path, "vm.args"),
       vm_args_rendered

mix.lock

@@ -3,13 +3,11 @@
   "bson": {:git, "https://github.com/comtihon/bson-erlang.git", "14308ab927cfa69324742c3de720578094e0bb19", [tag: "v0.2.2"]},
   "cowboy": {:git, "https://github.com/emqx/cowboy.git", "e3ed6c2ab3ac29988d26ed1f176def47b6e8d6de", [tag: "2.9.0"]},
   "cowboy_swagger": {:git, "https://github.com/inaka/cowboy_swagger", "bc441df7988da0f5c5d11ae0861c394dc30995c5", [tag: "2.5.0"]},
-  "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"},
+  "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", [ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363"]},
-  "earmark": {:hex, :earmark, "1.4.19", "3854a17305c880cc46305af15fb1630568d23a709aba21aaa996ced082fc29d7", [:mix], [{:earmark_parser, ">= 1.4.18", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "d5a8c9f9e37159a8fdd3ea8437fb4e229eaf56d5129b9a011dc4780a4872079d"},
-  "earmark_parser": {:hex, :earmark_parser, "1.4.18", "e1b2be73eb08a49fb032a0208bf647380682374a725dfb5b9e510def8397f6f2", [:mix], [], "hexpm", "114a0e85ec3cf9e04b811009e73c206394ffecfcc313e0b346de0d557774ee97"},
   "ecpool": {:git, "https://github.com/emqx/ecpool.git", "0516d2cebd14654ef8c583c347e4a0b01363b86d", [tag: "0.5.1"]},
   "eetcd": {:git, "https://github.com/zhongwencool/eetcd", "69d50aca98247953ee8a3ff58423a693f8318d90", [tag: "v0.3.4"]},
   "ehttpc": {:git, "https://github.com/emqx/ehttpc.git", "7b1a76b2353b385725e62f948cd399c7040467f8", [tag: "0.1.12"]},
-  "ekka": {:git, "https://github.com/emqx/ekka.git", "005fd6bb94199dc2ecb4ba03284f253b408e02d9", [tag: "0.11.1"]},
+  "ekka": {:git, "https://github.com/emqx/ekka.git", "70f2250e5e968e0c1da64e5b4733c5eb0eb402de", [tag: "0.11.2"]},
   "eldap2": {:git, "https://github.com/emqx/eldap2", "f595f67b094db3b9dc07941337706621e815431f", [tag: "v0.2.2"]},
   "emqtt": {:git, "https://github.com/emqx/emqtt.git", "25892ef48a979a9dfbd74d86133cb28cf11f3cf4", [tag: "1.4.3"]},
   "emqx_http_lib": {:git, "https://github.com/emqx/emqx_http_lib.git", "b84d42239fb09fecf50d9469fac914fb9b8efe34", [tag: "0.4.1"]},
@@ -19,11 +17,10 @@
   "esasl": {:git, "https://github.com/emqx/esasl.git", "96d7ac9f6c156017dd35b30df2dd722ae469c7f0", [tag: "0.2.0"]},
   "esockd": {:git, "https://github.com/emqx/esockd.git", "abb01f31c47303b4b4eecdbfe8401feedb6b4216", [tag: "5.9.0"]},
   "estatsd": {:git, "https://github.com/emqx/estatsd", "5184d846b7ecb83509bd4d32695c60428c0198cd", [tag: "0.1.0"]},
-  "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"},
   "gen_rpc": {:git, "https://github.com/emqx/gen_rpc.git", "fb7418dc8cf7e97d153fba073bee0fac07dce753", [tag: "2.5.1"]},
   "getopt": {:hex, :getopt, "1.0.2", "33d9b44289fe7ad08627ddfe1d798e30b2da0033b51da1b3a2d64e72cd581d02", [:rebar3], [], "hexpm", "a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26"},
-  "gpb": {:hex, :gpb, "4.19.2", "e841a1c5d5b82b6fbaddd2b6968c66ad48b34adc3b0d974e55cca653bd28ccc5", [:make, :rebar3], [], "hexpm", "3f4cb7f263e94675a598501a6dc96ec17f93ecb28c29bea3bea3a71c53a3317a"},
+  "gpb": {:hex, :gpb, "4.11.2", "a2c05241408310b8bd8dbdfd5e1a419799e45fc1408371eaa0f595023c3b21aa", [:make, :rebar], [], "hexpm", "b355a5982b604d6c044ebb6013e5fe65d30c30d0700d35317e7066691eb6bd61"},
-  "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"},
+  "gproc": {:git, "https://github.com/uwiger/gproc.git", "ce7397809aca0d6eb3aac6db65953752e47fb511", [tag: "0.8.0"]},
   "grpc": {:git, "https://github.com/emqx/grpc-erl", "9dd00ce65ecbd7fac2de5537edb9976d40b07fe9", [tag: "0.6.4"]},
   "gun": {:git, "https://github.com/emqx/gun.git", "89134e57b3e706c9851701907e00df69d84e9de5", [tag: "1.3.6"]},
   "hocon": {:git, "https://github.com/emqx/hocon.git", "b6baf9c5fcbc3e9f0e72959cb18e863de4cc1d33", [tag: "0.22.0"]},
@@ -32,20 +29,17 @@
   "jose": {:git, "https://github.com/potatosalad/erlang-jose.git", "991649695aaccd92c8effb1c1e88e6159fe8e9a6", [tag: "1.11.2"]},
   "jsx": {:git, "https://github.com/talentdeficit/jsx.git", "bb9b3e570a7efe331eed0900c3a5188043a850d7", [tag: "v3.1.0"]},
   "lc": {:git, "https://github.com/qzhuyan/lc.git", "6f98d098e5aaf4fcd6afbbb2acca96855c474600", [tag: "0.1.2"]},
-  "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"},
-  "makeup_elixir": {:hex, :makeup_elixir, "0.15.2", "dc72dfe17eb240552857465cc00cce390960d9a0c055c4ccd38b70629227e97c", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "fd23ae48d09b32eff49d4ced2b43c9f086d402ee4fd4fcb2d7fad97fa8823e75"},
   "minirest": {:git, "https://github.com/emqx/minirest.git", "f3f80b3e07295d8b6db22ed456318e0cc9dd167f", [tag: "1.2.7"]},
   "mnesia_rocksdb": {:git, "https://github.com/k32/mnesia_rocksdb", "68a80d127c49005480e0dd1f73149e8621052100", [tag: "0.1.5-k32"]},
   "mongodb": {:git, "https://github.com/emqx/mongodb-erlang", "2ffe62f42dafb98eaafead9d340a674c5f9279a5", [tag: "v3.0.10"]},
   "mria": {:git, "https://github.com/emqx/mria.git", "2bf3a71abc3635f910be4b943fa4ccbf8b8257fa", [tag: "0.1.5"]},
   "mysql": {:git, "https://github.com/emqx/mysql-otp", "bdabac44cc8836a9e23897b7e1b77c7df7e04f70", [tag: "1.7.1"]},
-  "nimble_parsec": {:hex, :nimble_parsec, "1.2.0", "b44d75e2a6542dcb6acf5d71c32c74ca88960421b6874777f79153bbbbd7dccc", [:mix], [], "hexpm", "52b2871a7515a5ac49b00f214e4165a40724cf99798d8e4a65e4fd64ebd002c1"},
   "observer_cli": {:hex, :observer_cli, "1.7.1", "c9ca1f623a3ef0158283a3c37cd7b7235bfe85927ad6e26396dd247e2057f5a1", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "4ccafaaa2ce01b85ddd14591f4d5f6731b4e13b610a70fb841f0701178478280"},
   "pbkdf2": {:git, "https://github.com/emqx/erlang-pbkdf2.git", "45d9981209ea07a83a58cf85aaf8236457da4342", [tag: "2.0.4"]},
   "poolboy": {:git, "https://github.com/emqx/poolboy.git", "29be47db8c2be38b18c908e43a80ebb7b9b6116b", [tag: "1.5.2"]},
   "prometheus": {:git, "https://github.com/emqx/prometheus.erl", "9994c76adca40d91a2545102230ccce2423fd8a7", [ref: "9994c76adca40d91a2545102230ccce2423fd8a7"]},
   "quicer": {:git, "https://github.com/emqx/quic.git", "ef73617d0f10f0f30f3aa77eb4a2f6ae071a2e29", [tag: "0.0.9"]},
-  "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
+  "ranch": {:git, "https://github.com/ninenines/ranch.git", "a692f44567034dacf5efcaa24a24183788594eb7", [ref: "a692f44567034dacf5efcaa24a24183788594eb7"]},
   "recon": {:git, "https://github.com/ferd/recon.git", "f7b6c08e6e9e2219db58bfb012c58c178822e01e", [tag: "2.5.1"]},
   "replayq": {:hex, :replayq, "0.3.3", "29344e4fd7c41c232d7f20d7a6e6712169ca585583bbb4bb6dd518f04e0d6cc4", [:rebar3], [], "hexpm", "3a527aff0960cf7ba7d189c79d7f0fbc170adb62e351acc223ccd6d094095c27"},
   "rocksdb": {:git, "https://github.com/k32/erlang-rocksdb.git", "e74972c3da4fe1f08eb66d39fce643a2d25a60be", [tag: "1.7.2-k32"]},
@@ -54,7 +48,7 @@
   "snabbkaffe": {:git, "https://github.com/kafka4beam/snabbkaffe.git", "750ea19ab8fbcb609639d5234b5a2dde75ac38e9", [tag: "0.16.0"]},
   "ssl_verify_fun": {:git, "https://github.com/deadtrickster/ssl_verify_fun.erl.git", "c5718226b0b9f3d1a38ef6ca3c3b4c75f53dda92", [tag: "1.1.4"]},
   "supervisor3": {:hex, :supervisor3, "1.1.9", "f1a3cc12fb6197526f548e79c9fe2b4af0c74efb8a687917b3b1ebe5e9c9368d", [:rebar3], [], "hexpm", "71b177c08f8cab9ec8ecb81c1aa28a23bbc24aac4b468c2db69840229d78d5c4"},
-  "system_monitor": {:git, "https://github.com/klarna-incubator/system_monitor.git", "a106f72e8e2251d90e1d95944c139946dac1a07f", [tag: "2.2.0"]},
+  "system_monitor": {:git, "https://github.com/k32/system_monitor.git", "3b4b381bf9503695cd764ecf22067ac6542cee89", [tag: "2.2.1"]},
   "trails": {:hex, :trails, "2.3.0", "b09703f056705f4943e14fff077b98c711a6f48fad40f4ff0b350794074ad69c", [:rebar3], [{:cowboy, "2.8.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:ranch, "2.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "40804001eb80417aa9d02400f39b7216956c3f251539a8a6096a69b3fac0ea07"},
   "typerefl": {:git, "https://github.com/k32/typerefl.git", "0cafafe1a6ce94f8709f237e890026a290a3e36f", [tag: "0.8.5"]},
 }

rebar.config

@@ -53,7 +53,7 @@
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
     , {mria, {git, "https://github.com/emqx/mria", {tag, "0.1.5"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.2"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.7"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
@@ -62,7 +62,7 @@
     , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
-    , {system_monitor, {git, "https://github.com/klarna-incubator/system_monitor", {tag, "2.2.0"}}}
+    , {system_monitor, {git, "https://github.com/k32/system_monitor", {tag, "2.2.1"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}}
     , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}}

scripts/check-elixir-deps-discrepancies.exs

@@ -0,0 +1,101 @@
+#!/usr/bin/env elixir
+
+# ensure we have a fresh rebar.lock
+
+case File.stat("rebar.lock") do
+  {:ok, _} ->
+    File.rm!("rebar.lock")
+
+  _ ->
+    :ok
+end
+{_, 0} =
+  File.cwd!()
+  |> Path.join("rebar3")
+  |> System.cmd(["tree"], into: IO.stream())
+
+{:ok, props} = :file.consult("rebar.lock")
+
+{[{_, rebar_deps}], [props]} = Enum.split_with(props, &is_tuple/1)
+
+# dpendencies declared as package versions have a "secondary index"
+pkg_idx =
+  props
+  |> Keyword.fetch!(:pkg_hash)
+  |> Map.new()
+
+rebar_deps = Map.new(rebar_deps, fn {name, ref, _} ->
+  ref = case ref  do
+          {:pkg, _, _} ->
+            pkg_idx
+            |> Map.fetch!(name)
+            |> String.downcase()
+
+          {:git, _, {:ref, ref}} ->
+            to_string(ref)
+        end
+
+  {name, ref}
+end)
+
+{mix_deps, []} = Code.eval_file("mix.lock")
+
+mix_deps = Map.new(mix_deps, fn {name, ref} ->
+  ref = case ref do
+          {:git, _, ref, _} ->
+            ref
+
+          {:hex, _, _, ref, _, _, _, _} ->
+            ref
+        end
+
+  {to_string(name), ref}
+end)
+
+diffs = Enum.reduce(rebar_deps, %{}, fn {name, rebar_ref}, acc ->
+  mix_ref = mix_deps[name]
+
+  cond do
+    mix_ref && mix_ref != rebar_ref ->
+      Map.put(acc, name, {rebar_ref, mix_ref})
+
+    is_nil(mix_ref) ->
+      Map.put(acc, name, {rebar_ref, nil})
+
+    :otherwise ->
+      acc
+  end
+end)
+
+if diffs == %{} do
+  IO.puts(
+    IO.ANSI.green() <>
+      "* Mix and Rebar3 dependencies OK!" <>
+      IO.ANSI.reset()
+  )
+  System.halt(0)
+else
+  IO.puts(
+    IO.ANSI.red() <>
+      "* Discrepancies between Elixir and Rebar3 dependencies found!" <>
+      IO.ANSI.reset()
+  )
+
+  Enum.each(diffs, fn {name, {rebar_ref, mix_ref}} ->
+    IO.puts(
+      IO.ANSI.red() <>
+        "  * #{name}\n" <>
+        "    * Rebar3 ref: #{rebar_ref}\n" <>
+        "    * Mix ref: #{mix_ref}\n" <>
+        IO.ANSI.reset()
+    )
+  end)
+
+  IO.puts(
+    IO.ANSI.red() <>
+      "Update `mix.exs` to match Rebar3's references (use `overwrite: true` if necessary) and try again" <>
+      IO.ANSI.reset()
+  )
+
+  System.halt(1)
+end