From 197196624e95093034e041113a2d0cd6a9ca21eb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 22 Jan 2021 20:56:30 +0100 Subject: [PATCH 1/5] chore(CI): Put OTP 23 on trial --- .ci/apps_tests/docker-compose.yaml | 4 ++-- .ci/build_packages/Dockerfile | 2 +- .ci/compatibility_tests/docker-compose-ldap.yaml | 2 +- .../docker-compose-mongo-tls.yaml | 4 ++-- .ci/compatibility_tests/docker-compose-mongo.yaml | 2 +- .../docker-compose-mysql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-mysql.yaml | 2 +- .../docker-compose-pgsql-tls.yaml | 2 +- .ci/compatibility_tests/docker-compose-pgsql.yaml | 2 +- .../docker-compose-redis-cluster-tls.yaml | 2 +- .../docker-compose-redis-cluster.yaml | 2 +- .../docker-compose-redis-sentinel.yaml | 2 +- .../docker-compose-redis-single-tls.yaml | 4 ++-- .../docker-compose-redis-single.yaml | 2 +- .github/workflows/build_packages.yaml | 14 +++++++------- .github/workflows/run_fvt_tests.yaml | 2 +- deploy/docker/Dockerfile | 2 +- docker.mk | 6 +++--- 18 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml index d834da5b2..aadb55245 100644 --- a/.ci/apps_tests/docker-compose.yaml +++ b/.ci/apps_tests/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 depends_on: - mysql_server - redis_server @@ -82,7 +82,7 @@ services: build: context: ../.. dockerfile: .ci/apps_tests/openldap/Dockerfile - args: + args: LDAP_TAG: ${LDAP_TAG} image: emqx-ldap:1.0 ports: diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index adfbb82a2..197b7e731 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl22.3.4.13-ubuntu20.04 +ARG BUILD_FROM=emqx/build-env:erl23.2.2-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/compatibility_tests/docker-compose-ldap.yaml b/.ci/compatibility_tests/docker-compose-ldap.yaml index 1bbdfe2aa..33b37e00c 100644 --- a/.ci/compatibility_tests/docker-compose-ldap.yaml +++ b/.ci/compatibility_tests/docker-compose-ldap.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 depends_on: - ldap_server networks: diff --git a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml index a22642a24..1611534f6 100644 --- a/.ci/compatibility_tests/docker-compose-mongo-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx @@ -14,7 +14,7 @@ services: tty: true mongo_server: - container_name: mongo + container_name: mongo image: mongo:${MONGO_TAG} restart: always environment: diff --git a/.ci/compatibility_tests/docker-compose-mongo.yaml b/.ci/compatibility_tests/docker-compose-mongo.yaml index 8385f1fb0..2f769ac63 100644 --- a/.ci/compatibility_tests/docker-compose-mongo.yaml +++ b/.ci/compatibility_tests/docker-compose-mongo.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml index 50de346c2..8117907f5 100644 --- a/.ci/compatibility_tests/docker-compose-mysql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-mysql.yaml b/.ci/compatibility_tests/docker-compose-mysql.yaml index f7a1b5d39..1f285cc5e 100644 --- a/.ci/compatibility_tests/docker-compose-mysql.yaml +++ b/.ci/compatibility_tests/docker-compose-mysql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml index ff26d88bd..d3d9d93b5 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-pgsql.yaml b/.ci/compatibility_tests/docker-compose-pgsql.yaml index f76768e8e..c5492d971 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../../:/emqx working_dir: /emqx diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml index 592f857cc..06518854f 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster-tls.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml index fa4a93d90..213a06866 100644 --- a/.ci/compatibility_tests/docker-compose-redis-cluster.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-cluster.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml index 2cc3743c0..b2b58fefe 100644 --- a/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-sentinel.yaml @@ -5,7 +5,7 @@ version: '2.4' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml b/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml index 22d44a035..03d643754 100644 --- a/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-single-tls.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx networks: @@ -14,7 +14,7 @@ services: tty: true redis_server: - container_name: redis + container_name: redis image: redis:${REDIS_TAG} volumes: - ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls diff --git a/.ci/compatibility_tests/docker-compose-redis-single.yaml b/.ci/compatibility_tests/docker-compose-redis-single.yaml index bc9b01d12..5d7acb865 100644 --- a/.ci/compatibility_tests/docker-compose-redis-single.yaml +++ b/.ci/compatibility_tests/docker-compose-redis-single.yaml @@ -3,7 +3,7 @@ version: '3' services: erlang: container_name: erlang - image: erlang:22.3.4.13 + image: emqx/build-env:erl23.2.2-ubuntu20.04 volumes: - ../..:/emqx networks: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index cfe434961..4668113b1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -22,7 +22,7 @@ jobs: - uses: gleam-lang/setup-erlang@v1.1.0 id: install_erlang with: - otp-version: 22.3 + otp-version: 23.2 - name: build run: | # set-executionpolicy remotesigned -s cu @@ -81,11 +81,11 @@ jobs: git config --global credential.helper store - name: build erlang run: | - kerl build 22.3.4.13 - kerl install 22.3.4.13 $HOME/.kerl/22.3.4.13 + kerl build 23.2.2 + kerl install 23.2.2 $HOME/.kerl/23.2.2 - name: build run: | - . $HOME/.kerl/22.3.4.13/activate + . $HOME/.kerl/23.2.2/activate make emqx-pkg - name: test run: | @@ -155,7 +155,7 @@ jobs: - uses: actions/checkout@v1 - name: get deps env: - ERL_OTP: erl22.3.4.13 + ERL_OTP: erl23.2.2 run: | docker run -i --rm \ -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ @@ -191,7 +191,7 @@ jobs: - name: build emqx packages if: (matrix.arch == 'amd64' && matrix.emqx == 'emqx') || startsWith(github.ref, 'refs/tags/') env: - ERL_OTP: erl22.3.4.13 + ERL_OTP: erl23.2.2 EMQX: ${{ matrix.emqx }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -245,7 +245,7 @@ jobs: - uses: actions/checkout@v1 - name: get deps env: - ERL_OTP: erl22.3.4.13 + ERL_OTP: erl23.2.2 run: | docker run -i --rm \ -e GITHUB_RUN_ID=$GITHUB_RUN_ID \ diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index fd2c55f05..761daa237 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -117,7 +117,7 @@ jobs: relup_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl22.3.4.13-ubuntu20.04 + container: emqx/build-env:erl23.2.2-ubuntu20.04 defaults: run: shell: bash diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 779e3266e..23b04677d 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl22.3-alpine-amd64 +ARG BUILD_FROM=emqx/build-env:erl23.2.2-alpine-amd64 ARG RUN_FROM=alpine:3.11 FROM ${BUILD_FROM} AS builder diff --git a/docker.mk b/docker.mk index 322b77b38..60e96ae39 100644 --- a/docker.mk +++ b/docker.mk @@ -21,7 +21,7 @@ docker-prepare: # @echo "PREPARE: Setting up dependencies." # @apt update -y # @apt install --only-upgrade docker-ce -y - + ## Update docker configuration to enable docker manifest command @echo "PREPARE: Updating docker configuration" @mkdir -p $$HOME/.docker @@ -45,7 +45,7 @@ docker-build: ## Prepare qemu to build images other then x86_64 on travis @echo "PREPARE: Qemu" \ && docker run --rm --privileged multiarch/qemu-user-static:register --reset - + @mkdir -p tmp \ && cd tmp \ && curl -L -o qemu-$(QEMU_ARCH)-static.tar.gz https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-$(QEMU_ARCH)-static.tar.gz \ @@ -54,7 +54,7 @@ docker-build: @docker build --no-cache \ --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl22.3-alpine-$(ARCH) \ + --build-arg BUILD_FROM=emqx/build-env:erl23.2.2-alpine-$(ARCH) \ --build-arg RUN_FROM=$(ARCH)/alpine:3.11 \ --build-arg EMQX_NAME=$(EMQX_NAME) \ --build-arg QEMU_ARCH=$(QEMU_ARCH) \ From 7edb320ec8551f8f7480758f80df9721a2d9ed22 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 25 Jan 2021 19:09:09 +0100 Subject: [PATCH 2/5] docs(README): update ct command with node name --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d7f162073..96b396e56 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ make eunit ct examples ```bash -./rebar3 ct --dir test,apps/emqx_sn,apps/emqx_coap -./rebar3 ct --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE -./rebar3 ct --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE --case t_rest_api +./rebar3 ct --name 'test@127.0.0.1' -c -v --dir test,apps/emqx_sn,apps/emqx_coap +./rebar3 ct --name 'test@127.0.0.1' -c -v --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE +./rebar3 ct --name 'test@127.0.0.1' -c -v --dir apps/emqx_auth_mnesi --suite emqx_acl_mnesia_SUITE --case t_rest_api ``` NOTE: Do *NOT* use full (relative) path to SUITE files like this `--suite apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl`, From 5477457c7e0e1ad964737fd8cbacaa1cb3fce9e5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 25 Jan 2021 19:09:41 +0100 Subject: [PATCH 3/5] fix(emqx_recon): fix badarg for remote module load --- apps/emqx_recon/src/emqx_recon_cli.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/emqx_recon/src/emqx_recon_cli.erl b/apps/emqx_recon/src/emqx_recon_cli.erl index 2f47fc8cd..587616ba7 100644 --- a/apps/emqx_recon/src/emqx_recon_cli.erl +++ b/apps/emqx_recon/src/emqx_recon_cli.erl @@ -42,7 +42,7 @@ cmd(["node_stats"]) -> recon:node_stats_print(10, 1000); cmd(["remote_load", Mod]) -> - emqx_ctl:print("~p~n", [recon:remote_load(list_to_atom(Mod))]); + emqx_ctl:print("~p~n", [remote_load(list_to_atom(Mod))]); cmd(["proc_count", Attr, N]) -> emqx_ctl:print("~p~n", [recon:proc_count(list_to_atom(Attr), list_to_integer(N))]); @@ -61,3 +61,12 @@ unload() -> concat(Key, Keyword) -> lists:concat([atom_to_list(Key), "/", atom_to_list(Keyword)]). +remote_load(Module) -> remote_load(nodes(), Module). + +%% recon:remote_load/1 has a bug, when nodes() returns [], it is +%% taken by recon as a node name. +%% before OTP 23, the call returns a 'badrpc' tuple +%% after OTP 23, it crashes with 'badarg' error +remote_load([], _Module) -> ok; +remote_load(Nodes, Module) -> recon:remote_load(Nodes, Module). + From 58af9e8474832137b5e3d5b1bd3e27f5338729ac Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 26 Jan 2021 17:34:16 +0800 Subject: [PATCH 4/5] chore(build): rename dashboard download file --- get-dashboard.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/get-dashboard.sh b/get-dashboard.sh index 1e7fa35ab..08879351d 100755 --- a/get-dashboard.sh +++ b/get-dashboard.sh @@ -25,8 +25,8 @@ if [ -d "$DASHBOARD_PATH/www" ] && [ "$(version)" = "$VERSION" ]; then exit 0 fi -curl -f -L "${DOWNLOAD_URL}/${VERSION}/dist.zip" -o ./dist.zip -unzip -q ./dist.zip -d "$DASHBOARD_PATH" +curl -f -L "${DOWNLOAD_URL}/${VERSION}/emqx-dashboard.zip" -o ./emqx-dashboard.zip +unzip -q ./emqx-dashboard.zip -d "$DASHBOARD_PATH" rm -rf "$DASHBOARD_PATH/www" mv "$DASHBOARD_PATH/dist" "$DASHBOARD_PATH/www" -rm -rf dist.zip +rm -rf emqx-dashboard.zip From 6468e7af34f75069dcdd384059928154878a2ba7 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 26 Jan 2021 12:38:36 +0100 Subject: [PATCH 5/5] fix(test): Pin ct-helper 1.3.4 for tls config fix --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index b203b1259..64286de12 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -29,7 +29,7 @@ plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.2"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.4"}}} , meck ].