From fd6cc1a8489d4e6d31a2ee723517ea09908615a2 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 15 Dec 2020 13:47:14 +0800 Subject: [PATCH] test(CI): update actions add fvt tests update test cases --- .ci/apps_tests/.env | 5 + .ci/apps_tests/docker-compose.yaml | 112 ++++++++++ .ci/apps_tests/emqx_ldap/Dockerfile | 26 +++ .ci/apps_tests/emqx_ldap/certs/cacert.pem | 20 ++ .ci/apps_tests/emqx_ldap/certs/cert.pem | 19 ++ .../emqx_ldap/certs/client-cert.pem | 19 ++ .ci/apps_tests/emqx_ldap/certs/client-key.pem | 27 +++ .ci/apps_tests/emqx_ldap/certs/key.pem | 27 +++ .ci/apps_tests/emqx_ldap/schema/emqx.io.ldif | 135 ++++++++++++ .ci/apps_tests/emqx_ldap/schema/emqx.schema | 46 ++++ .ci/apps_tests/emqx_ldap/slapd.conf | 16 ++ .../docker-compose.yaml | 3 + .ci/fvt_tests/http_server/README.md | 30 +++ .ci/fvt_tests/http_server/rebar.config | 10 + .../http_server/src/http_server.app.src | 17 ++ .ci/fvt_tests/http_server/src/http_server.erl | 50 +++++ .ci/fvt_tests/relup.lux | 158 ++++++++++++++ .ci/paho_tests/Makefile | 47 ---- ...ross_packages.yaml => build_packages.yaml} | 19 +- .github/workflows/run_fvt_tests.yaml | 200 ++++++++++++++++++ .github/workflows/run_test_case.yaml | 41 ---- .github/workflows/run_test_cases.yaml | 81 +++++++ Makefile | 4 + apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf | 6 +- 24 files changed, 1018 insertions(+), 100 deletions(-) create mode 100644 .ci/apps_tests/.env create mode 100644 .ci/apps_tests/docker-compose.yaml create mode 100644 .ci/apps_tests/emqx_ldap/Dockerfile create mode 100644 .ci/apps_tests/emqx_ldap/certs/cacert.pem create mode 100644 .ci/apps_tests/emqx_ldap/certs/cert.pem create mode 100644 .ci/apps_tests/emqx_ldap/certs/client-cert.pem create mode 100644 .ci/apps_tests/emqx_ldap/certs/client-key.pem create mode 100644 .ci/apps_tests/emqx_ldap/certs/key.pem create mode 100644 .ci/apps_tests/emqx_ldap/schema/emqx.io.ldif create mode 100644 .ci/apps_tests/emqx_ldap/schema/emqx.schema create mode 100644 .ci/apps_tests/emqx_ldap/slapd.conf rename .ci/{paho_tests => fvt_tests}/docker-compose.yaml (94%) create mode 100644 .ci/fvt_tests/http_server/README.md create mode 100644 .ci/fvt_tests/http_server/rebar.config create mode 100644 .ci/fvt_tests/http_server/src/http_server.app.src create mode 100644 .ci/fvt_tests/http_server/src/http_server.erl create mode 100644 .ci/fvt_tests/relup.lux delete mode 100644 .ci/paho_tests/Makefile rename .github/workflows/{build_cross_packages.yaml => build_packages.yaml} (97%) create mode 100644 .github/workflows/run_fvt_tests.yaml delete mode 100644 .github/workflows/run_test_case.yaml create mode 100644 .github/workflows/run_test_cases.yaml diff --git a/.ci/apps_tests/.env b/.ci/apps_tests/.env new file mode 100644 index 000000000..b8de5b7e6 --- /dev/null +++ b/.ci/apps_tests/.env @@ -0,0 +1,5 @@ +MYSQL_VSN=5.7 +REDIS_VSN=6 +MONGO_VSN=4.1 +PGSQL_VSN=11 +LDAP_VSN=2.4.50 diff --git a/.ci/apps_tests/docker-compose.yaml b/.ci/apps_tests/docker-compose.yaml new file mode 100644 index 000000000..751bd6d7c --- /dev/null +++ b/.ci/apps_tests/docker-compose.yaml @@ -0,0 +1,112 @@ +version: '3' + +services: + erlang: + container_name: erlang + image: erlang:22.3 + depends_on: + - mysql_server + - redis_server + - mongo_server + - pgsql_server + - ldap_server + networks: + - emqx_bridge + volumes: + - ../../.:/emqx + working_dir: /emqx + tty: true + + mysql_server: + container_name: mysql + image: mysql:${MYSQL_VSN} + restart: always + ports: + - 3306:3306 + environment: + MYSQL_ROOT_PASSWORD: public + MYSQL_DATABASE: mqtt + volumes: + - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem:/etc/certs/ca-cert.pem + - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-cert.pem:/etc/certs/server-cert.pem + - ../../apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/server-key.pem:/etc/certs/server-key.pem + command: + --bind-address 0.0.0.0 + --default-authentication-plugin=mysql_native_password + --character-set-server=utf8mb4 + --collation-server=utf8mb4_general_ci + --explicit_defaults_for_timestamp=true + --lower_case_table_names=1 + --max_allowed_packet=128M + --skip-symbolic-links + --ssl-ca=/etc/certs/ca.pem + --ssl-cert=/etc/certs/server-cert.pem + --ssl-key=/etc/certs/server-key.pem + networks: + - emqx_bridge + + redis_server: + container_name: redis + image: redis:${REDIS_VSN} + ports: + - 6379:6379 + command: + - redis-server + - "--bind 0.0.0.0 ::" + - --tls-port 6380 + - --tls-cert-file /tls/redis.crt + - --tls-key-file /tls/redis.key + - --tls-ca-cert-file /tls/ca.crt + volumes: + - ../../apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs:/tls + restart: always + networks: + - emqx_bridge + + mongo_server: + container_name: mongo + image: mongo:${MONGO_VSN} + ports: + - 27017:27017 + restart: always + environment: + MONGO_INITDB_DATABASE: mqtt + volumes: + - ../../apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/mongodb.pem/:/etc/certs/mongodb.pem + command: + --ipv6 + --bind_ip_all + --sslMode requireSSL + --sslPEMKeyFile /etc/certs/mongodb.pem + networks: + - emqx_bridge + + pgsql_server: + container_name: pgsql + image: postgres:${PGSQL_VSN} + ports: + - 5432:5432 + restart: always + environment: + POSTGRES_PASSWORD: public + POSTGRES_USER: root + POSTGRES_DB: mqtt + networks: + - emqx_bridge + + ldap_server: + container_name: openldap + build: + context: ./emqx_ldap + args: + LDAP_VSN: ${LDAP_VSN} + image: emqx-ldap:1.0 + ports: + - 389:389 + restart: always + networks: + - emqx_bridge + +networks: + emqx_bridge: + driver: bridge diff --git a/.ci/apps_tests/emqx_ldap/Dockerfile b/.ci/apps_tests/emqx_ldap/Dockerfile new file mode 100644 index 000000000..79a32069e --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/Dockerfile @@ -0,0 +1,26 @@ +FROM buildpack-deps:stretch + +ARG LDAP_VSN=2.4.50 + +RUN apt-get update && apt-get install -y groff groff-base +RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${LDAP_VSN}.tgz \ + && gunzip -c openldap-${LDAP_VSN}.tgz | tar xvfB - \ + && cd openldap-${LDAP_VSN} \ + && ./configure && make depend && make && make install \ + && cd .. && rm -rf openldap-${LDAP_VSN} + +COPY ./slapd.conf /usr/local/etc/openldap/slapd.conf +COPY ./schema/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +COPY ./schema/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +COPY ./certs/*.pem /usr/local/etc/openldap/ + +RUN mkdir -p /usr/local/etc/openldap/data \ + && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf + +WORKDIR /usr/local/etc/openldap + +EXPOSE 389 636 + +ENTRYPOINT ["/usr/local/libexec/slapd", "-h", "ldap:/// ldaps:///", "-d", "3", "-f", "/usr/local/etc/openldap/slapd.conf"] + +CMD [] diff --git a/.ci/apps_tests/emqx_ldap/certs/cacert.pem b/.ci/apps_tests/emqx_ldap/certs/cacert.pem new file mode 100644 index 000000000..604fd2362 --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/certs/cacert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDUTCCAjmgAwIBAgIJAPPYCjTmxdt/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV +BAYTAkNOMREwDwYDVQQIDAhoYW5nemhvdTEMMAoGA1UECgwDRU1RMQ8wDQYDVQQD +DAZSb290Q0EwHhcNMjAwNTA4MDgwNjUyWhcNMzAwNTA2MDgwNjUyWjA/MQswCQYD +VQQGEwJDTjERMA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UE +AwwGUm9vdENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzcgVLex1 +EZ9ON64EX8v+wcSjzOZpiEOsAOuSXOEN3wb8FKUxCdsGrsJYB7a5VM/Jot25Mod2 +juS3OBMg6r85k2TWjdxUoUs+HiUB/pP/ARaaW6VntpAEokpij/przWMPgJnBF3Ur +MjtbLayH9hGmpQrI5c2vmHQ2reRZnSFbY+2b8SXZ+3lZZgz9+BaQYWdQWfaUWEHZ +uDaNiViVO0OT8DRjCuiDp3yYDj3iLWbTA/gDL6Tf5XuHuEwcOQUrd+h0hyIphO8D +tsrsHZ14j4AWYLk1CPA6pq1HIUvEl2rANx2lVUNv+nt64K/Mr3RnVQd9s8bK+TXQ +KGHd2Lv/PALYuwIDAQABo1AwTjAdBgNVHQ4EFgQUGBmW+iDzxctWAWxmhgdlE8Pj +EbQwHwYDVR0jBBgwFoAUGBmW+iDzxctWAWxmhgdlE8PjEbQwDAYDVR0TBAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAQEAGbhRUjpIred4cFAFJ7bbYD9hKu/yzWPWkMRa +ErlCKHmuYsYk+5d16JQhJaFy6MGXfLgo3KV2itl0d+OWNH0U9ULXcglTxy6+njo5 +CFqdUBPwN1jxhzo9yteDMKF4+AHIxbvCAJa17qcwUKR5MKNvv09C6pvQDJLzid7y +E2dkgSuggik3oa0427KvctFf8uhOV94RvEDyqvT5+pgNYZ2Yfga9pD/jjpoHEUlo +88IGU8/wJCx3Ds2yc8+oBg/ynxG8f/HmCC1ET6EHHoe2jlo8FpU/SgGtghS1YL30 +IWxNsPrUP+XsZpBJy/mvOhE5QXo6Y35zDqqj8tI7AGmAWu22jg== +-----END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/cert.pem b/.ci/apps_tests/emqx_ldap/certs/cert.pem new file mode 100644 index 000000000..092390b1d --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/certs/cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER +MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB +MB4XDTIwMDUwODA4MDcwNVoXDTMwMDUwNjA4MDcwNVowPzELMAkGA1UEBhMCQ04x +ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBlNlcnZl +cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNeWT3pE+QFfiRJzKmn +AMUrWo3K2j/Tm3+Xnl6WLz67/0rcYrJbbKvS3uyRP/stXyXEKw9CepyQ1ViBVFkW +Aoy8qQEOWFDsZc/5UzhXUnb6LXr3qTkFEjNmhj+7uzv/lbBxlUG1NlYzSeOB6/RT +8zH/lhOeKhLnWYPXdXKsa1FL6ij4X8DeDO1kY7fvAGmBn/THh1uTpDizM4YmeI+7 +4dmayA5xXvARte5h4Vu5SIze7iC057N+vymToMk2Jgk+ZZFpyXrnq+yo6RaD3ANc +lrc4FbeUQZ5a5s5Sxgs9a0Y3WMG+7c5VnVXcbjBRz/aq2NtOnQQjikKKQA8GF080 +BQkCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL +BQADggEBAJefnMZpaRDHQSNUIEL3iwGXE9c6PmIsQVE2ustr+CakBp3TZ4l0enLt +iGMfEVFju69cO4oyokWv+hl5eCMkHBf14Kv51vj448jowYnF1zmzn7SEzm5Uzlsa +sqjtAprnLyof69WtLU1j5rYWBuFX86yOTwRAFNjm9fvhAcrEONBsQtqipBWkMROp +iUYMkRqbKcQMdwxov+lHBYKq9zbWRoqLROAn54SRqgQk6c15JdEfgOOjShbsOkIH +UhqcwRkQic7n1zwHVGVDgNIZVgmJ2IdIWBlPEC7oLrRrBD/X1iEEXtKab6p5o22n +KB5mN+iQaE+Oe2cpGKZJiJRdM+IqDDQ= +-----END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/client-cert.pem b/.ci/apps_tests/emqx_ldap/certs/client-cert.pem new file mode 100644 index 000000000..09d855221 --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/certs/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER +MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB +MB4XDTIwMDUwODA4MDY1N1oXDTMwMDUwNjA4MDY1N1owPzELMAkGA1UEBhMCQ04x +ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBkNsaWVu +dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMy4hoksKcZBDbY680u6 +TS25U51nuB1FBcGMlF9B/t057wPOlxF/OcmbxY5MwepS41JDGPgulE1V7fpsXkiW +1LUimYV/tsqBfymIe0mlY7oORahKji7zKQ2UBIVFhdlvQxunlIDnw6F9popUgyHt +dMhtlgZK8oqRwHxO5dbfoukYd6J/r+etS5q26sgVkf3C6dt0Td7B25H9qW+f7oLV +PbcHYCa+i73u9670nrpXsC+Qc7Mygwa2Kq/jwU+ftyLQnOeW07DuzOwsziC/fQZa +nbxR+8U9FNftgRcC3uP/JMKYUqsiRAuaDokARZxVTV5hUElfpO6z6/NItSDvvh3i +eikCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL +BQADggEBABchYxKo0YMma7g1qDswJXsR5s56Czx/I+B41YcpMBMTrRqpUC0nHtLk +M7/tZp592u/tT8gzEnQjZLKBAhFeZaR3aaKyknLqwiPqJIgg0pgsBGITrAK3Pv4z +5/YvAJJKgTe5UdeTz6U4lvNEux/4juZ4pmqH4qSFJTOzQS7LmgSmNIdd072rwXBd +UzcSHzsJgEMb88u/LDLjj1pQ7AtZ4Tta8JZTvcgBFmjB0QUi6fgkHY6oGat/W4kR +jSRUBlMUbM/drr2PVzRc2dwbFIl3X+ZE6n5Sl3ZwRAC/s92JU6CPMRW02muVu6xl +goraNgPISnrbpR6KjxLZkVembXzjNNc= +-----END CERTIFICATE----- diff --git a/.ci/apps_tests/emqx_ldap/certs/client-key.pem b/.ci/apps_tests/emqx_ldap/certs/client-key.pem new file mode 100644 index 000000000..2b3f30cf6 --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/certs/client-key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzLiGiSwpxkENtjrzS7pNLblTnWe4HUUFwYyUX0H+3TnvA86X +EX85yZvFjkzB6lLjUkMY+C6UTVXt+mxeSJbUtSKZhX+2yoF/KYh7SaVjug5FqEqO +LvMpDZQEhUWF2W9DG6eUgOfDoX2milSDIe10yG2WBkryipHAfE7l1t+i6Rh3on+v +561LmrbqyBWR/cLp23RN3sHbkf2pb5/ugtU9twdgJr6Lve73rvSeulewL5BzszKD +BrYqr+PBT5+3ItCc55bTsO7M7CzOIL99BlqdvFH7xT0U1+2BFwLe4/8kwphSqyJE +C5oOiQBFnFVNXmFQSV+k7rPr80i1IO++HeJ6KQIDAQABAoIBAGWgvPjfuaU3qizq +uti/FY07USz0zkuJdkANH6LiSjlchzDmn8wJ0pApCjuIE0PV/g9aS8z4opp5q/gD +UBLM/a8mC/xf2EhTXOMrY7i9p/I3H5FZ4ZehEqIw9sWKK9YzC6dw26HabB2BGOnW +5nozPSQ6cp2RGzJ7BIkxSZwPzPnVTgy3OAuPOiJytvK+hGLhsNaT+Y9bNDvplVT2 +ZwYTV8GlHZC+4b2wNROILm0O86v96O+Qd8nn3fXjGHbMsAnONBq10bZS16L4fvkH +5G+W/1PeSXmtZFppdRRDxIW+DWcXK0D48WRliuxcV4eOOxI+a9N2ZJZZiNLQZGwg +w3A8+mECgYEA8HuJFrlRvdoBe2U/EwUtG74dcyy30L4yEBnN5QscXmEEikhaQCfX +Wm6EieMcIB/5I5TQmSw0cmBMeZjSXYoFdoI16/X6yMMuATdxpvhOZGdUGXxhAH+x +xoTUavWZnEqW3fkUU71kT5E2f2i+0zoatFESXHeslJyz85aAYpP92H0CgYEA2e5A +Yozt5eaA1Gyhd8SeptkEU4xPirNUnVQHStpMWUb1kzTNXrPmNWccQ7JpfpG6DcYl +zUF6p6mlzY+zkMiyPQjwEJlhiHM2NlL1QS7td0R8ewgsFoyn8WsBI4RejWrEG9td +EDniuIw+pBFkcWthnTLHwECHdzgquToyTMjrBB0CgYEA28tdGbrZXhcyAZEhHAZA +Gzog+pKlkpEzeonLKIuGKzCrEKRecIK5jrqyQsCjhS0T7ZRnL4g6i0s+umiV5M5w +fcc292pEA1h45L3DD6OlKplSQVTv55/OYS4oY3YEJtf5mfm8vWi9lQeY8sxOlQpn +O+VZTdBHmTC8PGeTAgZXHZUCgYA6Tyv88lYowB7SN2qQgBQu8jvdGtqhcs/99GCr +H3N0I69LPsKAR0QeH8OJPXBKhDUywESXAaEOwS5yrLNP1tMRz5Vj65YUCzeDG3kx +gpvY4IMp7ArX0bSRvJ6mYSFnVxy3k174G3TVCfksrtagHioVBGQ7xUg5ltafjrms +n8l55QKBgQDVzU8tQvBVqY8/1lnw11Vj4fkE/drZHJ5UkdC1eenOfSWhlSLfUJ8j +ds7vEWpRPPoVuPZYeR1y78cyxKe1GBx6Wa2lF5c7xjmiu0xbRnrxYeLolce9/ntp +asClqpnHT8/VJYTD7Kqj0fouTTZf0zkig/y+2XERppd8k+pSKjUCPQ== +-----END RSA PRIVATE KEY----- diff --git a/.ci/apps_tests/emqx_ldap/certs/key.pem b/.ci/apps_tests/emqx_ldap/certs/key.pem new file mode 100644 index 000000000..6c338216e --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/certs/key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAs15ZPekT5AV+JEnMqacAxStajcraP9Obf5eeXpYvPrv/Stxi +sltsq9Le7JE/+y1fJcQrD0J6nJDVWIFUWRYCjLypAQ5YUOxlz/lTOFdSdvotevep +OQUSM2aGP7u7O/+VsHGVQbU2VjNJ44Hr9FPzMf+WE54qEudZg9d1cqxrUUvqKPhf +wN4M7WRjt+8AaYGf9MeHW5OkOLMzhiZ4j7vh2ZrIDnFe8BG17mHhW7lIjN7uILTn +s36/KZOgyTYmCT5lkWnJeuer7KjpFoPcA1yWtzgVt5RBnlrmzlLGCz1rRjdYwb7t +zlWdVdxuMFHP9qrY206dBCOKQopADwYXTzQFCQIDAQABAoIBAQCuvCbr7Pd3lvI/ +n7VFQG+7pHRe1VKwAxDkx2t8cYos7y/QWcm8Ptwqtw58HzPZGWYrgGMCRpzzkRSF +V9g3wP1S5Scu5C6dBu5YIGc157tqNGXB+SpdZddJQ4Nc6yGHXYERllT04ffBGc3N +WG/oYS/1cSteiSIrsDy/91FvGRCi7FPxH3wIgHssY/tw69s1Cfvaq5lr2NTFzxIG +xCvpJKEdSfVfS9I7LYiymVjst3IOR/w76/ZFY9cRa8ZtmQSWWsm0TUpRC1jdcbkm +ZoJptYWlP+gSwx/fpMYftrkJFGOJhHJHQhwxT5X/ajAISeqjjwkWSEJLwnHQd11C +Zy2+29lBAoGBANlEAIK4VxCqyPXNKfoOOi5dS64NfvyH4A1v2+KaHWc7lqaqPN49 +ezfN2n3X+KWx4cviDD914Yc2JQ1vVJjSaHci7yivocDo2OfZDmjBqzaMp/y+rX1R +/f3MmiTqMa468rjaxI9RRZu7vDgpTR+za1+OBCgMzjvAng8dJuN/5gjlAoGBANNY +uYPKtearBmkqdrSV7eTUe49Nhr0XotLaVBH37TCW0Xv9wjO2xmbm5Ga/DCtPIsBb +yPeYwX9FjoasuadUD7hRvbFu6dBa0HGLmkXRJZTcD7MEX2Lhu4BuC72yDLLFd0r+ +Ep9WP7F5iJyagYqIZtz+4uf7gBvUDdmvXz3sGr1VAoGAdXTD6eeKeiI6PlhKBztF +zOb3EQOO0SsLv3fnodu7ZaHbUgLaoTMPuB17r2jgrYM7FKQCBxTNdfGZmmfDjlLB +0xZ5wL8ibU30ZXL8zTlWPElST9sto4B+FYVVF/vcG9sWeUUb2ncPcJ/Po3UAktDG +jYQTTyuNGtSJHpad/YOZctkCgYBtWRaC7bq3of0rJGFOhdQT9SwItN/lrfj8hyHA +OjpqTV4NfPmhsAtu6j96OZaeQc+FHvgXwt06cE6Rt4RG4uNPRluTFgO7XYFDfitP +vCppnoIw6S5BBvHwPP+uIhUX2bsi/dm8vu8tb+gSvo4PkwtFhEr6I9HglBKmcmog +q6waEQKBgHyecFBeM6Ls11Cd64vborwJPAuxIW7HBAFj/BS99oeG4TjBx4Sz2dFd +rzUibJt4ndnHIvCN8JQkjNG14i9hJln+H3mRss8fbZ9vQdqG+2vOWADYSzzsNI55 +RFY7JjluKcVkp/zCDeUxTU3O6sS+v6/3VE11Cob6OYQx3lN5wrZ3 +-----END RSA PRIVATE KEY----- diff --git a/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif b/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif new file mode 100644 index 000000000..f9833cd88 --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/schema/emqx.io.ldif @@ -0,0 +1,135 @@ +## create emqx.io + +dn:dc=emqx,dc=io +objectclass: top +objectclass: dcobject +objectclass: organization +dc:emqx +o:emqx,Inc. + +# create testdevice.emqx.io +dn:ou=testdevice,dc=emqx,dc=io +objectClass: top +objectclass:organizationalUnit +ou:testdevice + +# create user admin +dn:uid=admin,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: simpleSecurityObject +objectClass: account +userPassword:: e1NIQX1XNnBoNU1tNVB6OEdnaVVMYlBnekczN21qOWc9 +uid: admin + +## create user=mqttuser0001, +# password=mqttuser0001, +# passhash={SHA}mlb3fat40MKBTXUVZwCKmL73R/0= +# base64passhash=e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 +dn:uid=mqttuser0001,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0001 +isEnabled: TRUE +mqttAccountName: user1 +mqttPublishTopic: mqttuser0001/pub/1 +mqttPublishTopic: mqttuser0001/pub/+ +mqttPublishTopic: mqttuser0001/pub/# +mqttSubscriptionTopic: mqttuser0001/sub/1 +mqttSubscriptionTopic: mqttuser0001/sub/+ +mqttSubscriptionTopic: mqttuser0001/sub/# +mqttPubSubTopic: mqttuser0001/pubsub/1 +mqttPubSubTopic: mqttuser0001/pubsub/+ +mqttPubSubTopic: mqttuser0001/pubsub/# +userPassword:: e1NIQX1tbGIzZmF0NDBNS0JUWFVWWndDS21MNzNSLzA9 + +## create user=mqttuser0002 +# password=mqttuser0002, +# passhash={SSHA}n9XdtoG4Q/TQ3TQF4Y+khJbMBH4qXj4M +# base64passhash=e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= +dn:uid=mqttuser0002,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0002 +isEnabled: TRUE +mqttAccountName: user2 +mqttPublishTopic: mqttuser0002/pub/1 +mqttPublishTopic: mqttuser0002/pub/+ +mqttPublishTopic: mqttuser0002/pub/# +mqttSubscriptionTopic: mqttuser0002/sub/1 +mqttSubscriptionTopic: mqttuser0002/sub/+ +mqttSubscriptionTopic: mqttuser0002/sub/# +mqttPubSubTopic: mqttuser0002/pubsub/1 +mqttPubSubTopic: mqttuser0002/pubsub/+ +mqttPubSubTopic: mqttuser0002/pubsub/# +userPassword:: e1NTSEF9bjlYZHRvRzRRL1RRM1RRRjRZK2toSmJNQkg0cVhqNE0= + +## create user mqttuser0003 +# password=mqttuser0003, +# passhash={MD5}ybsPGoaK3nDyiQvveiCOIw== +# base64passhash=e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= +dn:uid=mqttuser0003,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0003 +isEnabled: TRUE +mqttPublishTopic: mqttuser0003/pub/1 +mqttPublishTopic: mqttuser0003/pub/+ +mqttPublishTopic: mqttuser0003/pub/# +mqttSubscriptionTopic: mqttuser0003/sub/1 +mqttSubscriptionTopic: mqttuser0003/sub/+ +mqttSubscriptionTopic: mqttuser0003/sub/# +mqttPubSubTopic: mqttuser0003/pubsub/1 +mqttPubSubTopic: mqttuser0003/pubsub/+ +mqttPubSubTopic: mqttuser0003/pubsub/# +userPassword:: e01ENX15YnNQR29hSzNuRHlpUXZ2ZWlDT0l3PT0= + +## create user mqttuser0004 +# password=mqttuser0004, +# passhash={MD5}2Br6pPDSEDIEvUlu9+s+MA== +# base64passhash=e01ENX0yQnI2cFBEU0VESUV2VWx1OStzK01BPT0= +dn:uid=mqttuser0004,ou=testdevice,dc=emqx,dc=io +objectClass: top +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0004 +isEnabled: TRUE +mqttPublishTopic: mqttuser0004/pub/1 +mqttPublishTopic: mqttuser0004/pub/+ +mqttPublishTopic: mqttuser0004/pub/# +mqttSubscriptionTopic: mqttuser0004/sub/1 +mqttSubscriptionTopic: mqttuser0004/sub/+ +mqttSubscriptionTopic: mqttuser0004/sub/# +mqttPubSubTopic: mqttuser0004/pubsub/1 +mqttPubSubTopic: mqttuser0004/pubsub/+ +mqttPubSubTopic: mqttuser0004/pubsub/# +userPassword: {MD5}2Br6pPDSEDIEvUlu9+s+MA== + +## create user mqttuser0005 +# password=mqttuser0005, +# passhash={SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= +# base64passhash=e1NIQX1qS254ZUVER1IxNGtFOEFSN3l1VkZPZWxoejQ9 +objectClass: top +dn:uid=mqttuser0005,ou=testdevice,dc=emqx,dc=io +objectClass: mqttUser +objectClass: mqttDevice +objectClass: mqttSecurity +uid: mqttuser0005 +isEnabled: TRUE +mqttPublishTopic: mqttuser0005/pub/1 +mqttPublishTopic: mqttuser0005/pub/+ +mqttPublishTopic: mqttuser0005/pub/# +mqttSubscriptionTopic: mqttuser0005/sub/1 +mqttSubscriptionTopic: mqttuser0005/sub/+ +mqttSubscriptionTopic: mqttuser0005/sub/# +mqttPubSubTopic: mqttuser0005/pubsub/1 +mqttPubSubTopic: mqttuser0005/pubsub/+ +mqttPubSubTopic: mqttuser0005/pubsub/# +userPassword: {SHA}jKnxeEDGR14kE8AR7yuVFOelhz4= + diff --git a/.ci/apps_tests/emqx_ldap/schema/emqx.schema b/.ci/apps_tests/emqx_ldap/schema/emqx.schema new file mode 100644 index 000000000..55f92269b --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/schema/emqx.schema @@ -0,0 +1,46 @@ +# +# Preliminary Apple OS X Native LDAP Schema +# This file is subject to change. +# +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.1.3 NAME 'isEnabled' + EQUALITY booleanMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 + SINGLE-VALUE + USAGE userApplications ) + +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.1 NAME ( 'mqttPublishTopic' 'mpt' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.2 NAME ( 'mqttSubscriptionTopic' 'mst' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.3 NAME ( 'mqttPubSubTopic' 'mpst' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) +attributetype ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4.4 NAME ( 'mqttAccountName' 'man' ) + EQUALITY caseIgnoreMatch + SUBSTR caseIgnoreSubstringsMatch + SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 + USAGE userApplications ) + + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.4 NAME 'mqttUser' + AUXILIARY + MAY ( mqttPublishTopic $ mqttSubscriptionTopic $ mqttPubSubTopic $ mqttAccountName) ) + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.2 NAME 'mqttDevice' + SUP top + STRUCTURAL + MUST ( uid ) + MAY ( isEnabled ) ) + +objectclass ( 1.3.6.1.4.1.11.2.53.2.2.3.1.2.3.3 NAME 'mqttSecurity' + SUP top + AUXILIARY + MAY ( userPassword $ userPKCS12 $ pwdAttribute $ pwdLockout ) ) diff --git a/.ci/apps_tests/emqx_ldap/slapd.conf b/.ci/apps_tests/emqx_ldap/slapd.conf new file mode 100644 index 000000000..d6ba20caa --- /dev/null +++ b/.ci/apps_tests/emqx_ldap/slapd.conf @@ -0,0 +1,16 @@ +include /usr/local/etc/openldap/schema/core.schema +include /usr/local/etc/openldap/schema/cosine.schema +include /usr/local/etc/openldap/schema/inetorgperson.schema +include /usr/local/etc/openldap/schema/ppolicy.schema +include /usr/local/etc/openldap/schema/emqx.schema + +TLSCACertificateFile /usr/local/etc/openldap/cacert.pem +TLSCertificateFile /usr/local/etc/openldap/cert.pem +TLSCertificateKeyFile /usr/local/etc/openldap/key.pem + +database bdb +suffix "dc=emqx,dc=io" +rootdn "cn=root,dc=emqx,dc=io" +rootpw {SSHA}eoF7NhNrejVYYyGHqnt+MdKNBh4r1w3W + +directory /usr/local/etc/openldap/data diff --git a/.ci/paho_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml similarity index 94% rename from .ci/paho_tests/docker-compose.yaml rename to .ci/fvt_tests/docker-compose.yaml index 7a3b30478..50e5efa16 100644 --- a/.ci/paho_tests/docker-compose.yaml +++ b/.ci/fvt_tests/docker-compose.yaml @@ -2,6 +2,7 @@ version: '3' services: emqx1: + container_name: node1.emqx.io image: emqx/emqx:build-alpine-amd64 environment: - "EMQX_NAME=emqx" @@ -27,6 +28,7 @@ services: - node1.emqx.io emqx2: + container_name: node2.emqx.io image: emqx/emqx:build-alpine-amd64 environment: - "EMQX_NAME=emqx" @@ -52,6 +54,7 @@ services: - node2.emqx.io client: + container_name: paho_client image: python:3.7.2-alpine3.9 depends_on: - emqx1 diff --git a/.ci/fvt_tests/http_server/README.md b/.ci/fvt_tests/http_server/README.md new file mode 100644 index 000000000..ea14939b3 --- /dev/null +++ b/.ci/fvt_tests/http_server/README.md @@ -0,0 +1,30 @@ +## http_server + + +The http server for emqx functional validation testing + +### Build + + + $ rebar3 compile + +### Getting Started + +``` +1> http_server:start(). +Start http_server listener on 8080 successfully. +ok +2> http_server:stop(). +ok +``` + +### APIS + ++ GET `/counter` + + 返回计数器的值 + ++ POST `/counter` + + 计数器加一 + diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config new file mode 100644 index 000000000..9cc5f3d02 --- /dev/null +++ b/.ci/fvt_tests/http_server/rebar.config @@ -0,0 +1,10 @@ +{erl_opts, [debug_info]}. +{deps, + [ + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.1"}}} + ]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [http_server]} +]}. diff --git a/.ci/fvt_tests/http_server/src/http_server.app.src b/.ci/fvt_tests/http_server/src/http_server.app.src new file mode 100644 index 000000000..f351bb349 --- /dev/null +++ b/.ci/fvt_tests/http_server/src/http_server.app.src @@ -0,0 +1,17 @@ +{application, http_server, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + % {mod, {http_server_app, []}}, + {modules, []}, + {applications, + [kernel, + stdlib, + minirest + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/.ci/fvt_tests/http_server/src/http_server.erl b/.ci/fvt_tests/http_server/src/http_server.erl new file mode 100644 index 000000000..b66b72939 --- /dev/null +++ b/.ci/fvt_tests/http_server/src/http_server.erl @@ -0,0 +1,50 @@ +-module(http_server). + +-import(minirest, [ return/0 + , return/1 + ]). + +-export([ start/0 + , stop/0 + ]). + +-rest_api(#{ name => get_counter + , method => 'GET' + , path => "/counter" + , func => get_counter + , descr => "Check counter" + }). +-rest_api(#{ name => add_counter + , method => 'POST' + , path => "/counter" + , func => add_counter + , descr => "Counter plus one" + }). + +-export([ get_counter/2 + , add_counter/2 + ]). + +start() -> + application:ensure_all_started(minirest), + ets:new(relup_test_message, [named_table, public]), + Handlers = [{"/", minirest:handler(#{modules => [?MODULE]})}], + Dispatch = [{"/[...]", minirest, Handlers}], + minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 8080}]}, Dispatch). + +stop() -> + ets:delete(relup_test_message), + minirest:stop_http(?MODULE). + +get_counter(_Binding, _Params) -> + return({ok, ets:info(relup_test_message, size)}). + +add_counter(_Binding, Params) -> + case lists:keymember(<<"payload">>, 1, Params) of + true -> + {value, {<<"id">>, ID}, Params1} = lists:keytake(<<"id">>, 1, Params), + ets:insert(relup_test_message, {ID, Params1}); + _ -> + ok + end, + return(). diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux new file mode 100644 index 000000000..990606b88 --- /dev/null +++ b/.ci/fvt_tests/relup.lux @@ -0,0 +1,158 @@ +[config var=PACKAGE_PATH] +[config var=BENCH_PATH] +[config var=ONE_MORE_EMQX_PATH] +[config var=VSN] +[config var=OLD_VSNS] + +[config shell_cmd=/bin/bash] +[config timeout=600000] + +[loop old_vsn $OLD_VSNS] + +[shell http_server] + !cd http_server + !rebar3 shell + ???Eshell + ???> + !http_server:start(). + ?Start http_server listener on 8080 successfully. + ?ok + ?> + +[shell emqx] + !cd $PACKAGE_PATH + !unzip -q -o emqx-ubuntu20.04-$old_vsn-x86_64.zip + ?SH-PROMPT + + !cd emqx + !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf + !./bin/emqx start + ?EMQ X Broker $old_vsn is started successfully! + + !./bin/emqx_ctl status + """? + Node 'emqx@127.0.0.1' is started + emqx $old_vsn is running + """ + +[shell emqx2] + !cd $PACKAGE_PATH + !cp -f $ONE_MORE_EMQX_PATH/one_more_emqx.sh . + !./one_more_emqx.sh emqx2 + ?SH-PROMPT + !cd emqx2 + + !./bin/emqx start + ?EMQ X Broker $old_vsn is started successfully! + + !./bin/emqx_ctl status + """? + Node 'emqx2@127.0.0.1' is started + emqx $old_vsn is running + """ + ?SH-PROMPT + + !./bin/emqx_ctl cluster join emqx@127.0.0.1 + ???Join the cluster successfully. + ?SH-PROMPT + + !./bin/emqx_ctl cluster status + """??? + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], + stopped_nodes => []} + """ + ?SH-PROMPT + + !./bin/emqx_ctl resources create 'web_hook' -i 'resource:691c29ba' -c '{"url": "http://127.0.0.1:8080/counter", "method": "POST"}' + ?created + ?SH-PROMPT + !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource": "resource:691c29ba"}}]' + ?created + ?SH-PROMPT + +[shell emqx] + !./bin/emqx_ctl resources list + ?691c29ba + ?SH-PROMPT + !./bin/emqx_ctl rules list + ?691c29ba + ?SH-PROMPT + +[shell bench] + !cd $BENCH_PATH + !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 600 + ???sent + +[shell emqx] + !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !./bin/emqx install $VSN + ?SH-PROMPT + !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" + ?$VSN + ?SH-PROMPT + + !./bin/emqx_ctl cluster status + """??? + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], + stopped_nodes => []} + """ + ?SH-PROMPT + +[shell emqx2] + !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !./bin/emqx install $VSN + ?SH-PROMPT + !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" + ?$VSN + ?SH-PROMPT + + !./bin/emqx_ctl cluster status + """??? + Cluster status: #{running_nodes => ['emqx2@127.0.0.1','emqx@127.0.0.1'], + stopped_nodes => []} + """ + ?SH-PROMPT + +[shell bench] + ???publish complete + ??SH-PROMPT: + !curl http://127.0.0.1:8080/counter + ???{"data":600,"code":0} + ?SH-PROMPT + +[shell http_server] + !http_server:stop(). + ?ok + ?> + !halt(3). + ?SH-PROMPT: + +[shell emqx2] + !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100 + -error + ??SH-PROMPT: + + !./bin/emqx stop + ?ok + ?SH-PROMPT: + + !rm -rf $PACKAGE_PATH/emqx2 + ?SH-PROMPT: + +[shell emqx] + !cat log/emqx.log.1 |grep -v 691c29ba |tail -n 100 + -error + ??SH-PROMPT: + + !./bin/emqx stop + ?ok + ?SH-PROMPT: + + !rm -rf $PACKAGE_PATH/emqx + ?SH-PROMPT: + +[endloop] + +[cleanup] + !echo ==$$?== + ?==0== diff --git a/.ci/paho_tests/Makefile b/.ci/paho_tests/Makefile deleted file mode 100644 index 022364cc9..000000000 --- a/.ci/paho_tests/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -## default globals -TARGET ?= emqx/emqx -EMQX_NAME = $(subst emqx/,,$(TARGET)) - -.PHONY: all -all: test - -define wait_emqx - @while [ "$$(docker inspect -f '{{ .State.Health.Status}}' $$(docker ps -a -q -f name=paho_test_emqx1))" != "healthy" ] || [ "$$(docker inspect -f '{{ .State.Health.Status}}' $$(docker ps -a -q -f name=paho_test_emqx2))" != "healthy" ]; do \ - if [ $$(docker ps -a -f name=paho_test_emqx -f status=exited -q | wc -l) -ne 0 ]; then \ - echo "['$$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop"; \ - exit; \ - else \ - echo "['$$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; \ - sleep 5; \ - fi; \ - done -endef - -.PHONY: create_container -create_container: clean - @docker-compose -p paho_test up -d - - $(call wait_emqx) - -.PHONY: test -test: create_container - @docker exec -i $$(docker ps -a -q -f name=paho_test_client) sh -c "apk update && apk add git curl \ - && git clone -b $(PAHO_BRANCH) https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing \ - && pip install pytest \ - && pytest -v /paho.mqtt.testing/interoperability/test_client/ --host node1.emqx.io" - - @docker-compose -p paho_test down - -.PHONY: cluster_test -cluster_test: create_container - @docker exec -i $$(docker ps -a -q -f name=paho_test_client) sh -c "apk update && apk add git curl \ - && git clone -b $(PAHO_BRANCH) https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing \ - && pip install pytest \ - && pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host node1.emqx.io \ - && pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 node1.emqx.io --host2 node2.emqx.io" - - @docker-compose -p paho_test down - -.PHONY: clean -clean: - @if [ ! -z "$$(docker ps -a -q -f name=paho_test)" ]; then docker-compose -p paho_test down; fi diff --git a/.github/workflows/build_cross_packages.yaml b/.github/workflows/build_packages.yaml similarity index 97% rename from .github/workflows/build_cross_packages.yaml rename to .github/workflows/build_packages.yaml index c380408db..bda61ce06 100644 --- a/.github/workflows/build_cross_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,10 +1,16 @@ name: Cross build packages on: - pull_request: push: tags: - v* + release: + types: + - published + pull_request: + workflow_dispatch: + repository_dispatch: + types: [run_actions] jobs: windows: @@ -275,11 +281,6 @@ jobs: sudo TARGET=emqx/emqx-edge ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker cd _packages/emqx-edge && for var in $(ls emqx-edge-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd - - - name: docker test - if: matrix.arch[0] == 'amd64' - run: | - sudo TARGET=emqx/emqx PAHO_BRANCH="develop-4.0" make -C .ci/paho_tests cluster_test - sudo TARGET=emqx/emqx-edge PAHO_BRANCH="develop-4.0" make -C .ci/paho_tests cluster_test - uses: actions/upload-artifact@v1 with: name: emqx @@ -340,7 +341,7 @@ jobs: with: path: emqx - name: update to github and emqx.io - if: contains(github.ref, 'refs/tags/') + if: github.event_name == 'release' run: | set -e -x -u version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") @@ -357,7 +358,7 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${version}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: push docker image to docker hub - if: contains(github.ref, 'refs/tags/') + if: github.event_name == 'release' run: | set -e -x -u version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") @@ -369,7 +370,7 @@ jobs: sudo TARGET=emqx/emqx-edge make -C emqx docker-push sudo TARGET=emqx/emqx-edge make -C emqx docker-manifest-list - name: update repo.emqx.io - if: contains(github.ref, 'refs/tags/') + if: github.event_name == 'release' run: | set -e -x -u version=$(echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g") diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml new file mode 100644 index 000000000..c76108633 --- /dev/null +++ b/.github/workflows/run_fvt_tests.yaml @@ -0,0 +1,200 @@ +name: Functional Verification Tests + +on: + push: + tags: + - v* + release: + types: + - published + pull_request: + workflow_dispatch: + repository_dispatch: + types: [run_actions] + +jobs: + docker_test: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v1 + - name: make emqx image + run: TARGET=emqx/emqx make docker + - name: run emqx + timeout-minutes: 5 + run: | + set -e -u -x + docker-compose -f .ci/fvt_tests/docker-compose.yaml up -d + while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do + if [ $(docker ps -a -f name=fvt_tests_emqx -f status=exited -q | wc -l) -ne 0 ]; then + echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop"; + exit; + else + echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; + sleep 5; + fi; + done + - name: make paho tests + run: | + docker exec -i paho_client sh -c "apk update && apk add git curl \ + && git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing \ + && pip install pytest \ + && pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host node1.emqx.io \ + && pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 node1.emqx.io --host2 node2.emqx.io \ + && pytest -v /paho.mqtt.testing/interoperability/test_client --host node1.emqx.io" + + helm_test: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v1 + - name: make emqx image + run: TARGET=emqx/emqx make docker + - name: install k3s + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts" + curl -sfL https://get.k3s.io | sh - + sudo chmod 644 /etc/rancher/k3s/k3s.yaml + kubectl cluster-info + - name: install helm + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + sudo chmod 700 get_helm.sh + sudo ./get_helm.sh + helm version + - name: run emqx on chart + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 5 + run: | + version=$(./pkg-vsn.sh) + sudo docker save emqx/emqx:$version -o emqx.tar.gz + sudo k3s ctr image import emqx.tar.gz + + sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml + sed -i -r 's/ pullPolicy: .*$/ pullPolicy: Never/g' deploy/charts/emqx/values.yaml + + helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx --debug --dry-run + helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx + + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + echo "=============================="; + kubectl get pods; + echo "=============================="; + echo "waiting emqx started"; + sleep 10; + done + - uses: actions/checkout@v2 + with: + repository: emqx/paho.mqtt.testing + ref: develop-4.0 + path: paho.mqtt.testing + - name: install pytest + run: | + pip install pytest + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: run paho test + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}") + emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}') + emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}') + + pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc + pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2 + + relup_test: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl22.3-ubuntu20.04 + defaults: + run: + shell: bash + steps: + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + - uses: actions/checkout@v2 + with: + repository: emqx/paho.mqtt.testing + ref: develop-4.0 + path: paho.mqtt.testing + - uses: actions/checkout@v2 + with: + repository: terry-xiaoyu/one_more_emqx + ref: master + path: one_more_emqx + - uses: actions/checkout@v2 + with: + repository: emqx/emqtt-bench + ref: master + path: emqtt-bench + - uses: actions/checkout@v2 + with: + repository: hawk/lux + ref: lux-2.4 + path: lux + - uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} + path: emqx + fetch-depth: 0 + - name: get version + run: | + set -e -x -u + cd emqx + vsn="$(erl -eval '{ok, [{application,emqx, L} | _]} = file:consult("src/emqx.app.src"), {vsn, VSN} = lists:keyfind(vsn,1,L), io:fwrite(VSN), halt().' -noshell)" + echo "VSN=$vsn" >> $GITHUB_ENV + pre_tag="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" + old_vsns="$(git tag -l "$pre_tag.[0-9]" | tr "\n" " " | sed "s/$vsn//")" + echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV + - name: download emqx + run: | + set -e -x -u + cd emqx + old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) + for old_vsn in ${old_vsns[@]}; do + wget https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v$old_vsn/emqx-ubuntu20.04-${old_vsn}-x86_64.zip + done + - name: build emqx + run: make -C emqx emqx-zip + - name: build emqtt-bench + run: make -C emqtt-bench + - name: build lux + run: | + set -e -u -x + cd lux + autoconf + ./configure + make + make install + - name: run relup test + run: | + set -e -x -u + if [ -n "$OLD_VSNS" ]; then + mkdir -p packages + cp emqx/_packages/emqx/*.zip packages + cp emqx/*.zip packages + lux -v \ + --timeout 600000 \ + --var PACKAGE_PATH=$(pwd)/packages \ + --var BENCH_PATH=$(pwd)/emqtt-bench \ + --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ + --var VSN="$VSN" \ + --var OLD_VSNS="$OLD_VSNS" \ + emqx/.ci/fvt_tests/relup.lux + fi + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: lux_logs + path: lux_logs + + + diff --git a/.github/workflows/run_test_case.yaml b/.github/workflows/run_test_case.yaml deleted file mode 100644 index 0384af554..000000000 --- a/.github/workflows/run_test_case.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: Run test case - -on: [push, pull_request] - -jobs: - - run_test_case: - - runs-on: ubuntu-latest - - container: - image: erlang:22.1 - - steps: - - uses: actions/checkout@v1 - - name: Code dialyzer - run: | - make xref - make dialyzer - rm -f rebar.lock - - name: Run tests - run: | - make eunit - rm -f rebar.lock - make ct - rm -f rebar.lock - make cover - - name: Coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - make coveralls - - uses: actions/upload-artifact@v1 - if: always() - with: - name: logs - path: _build/test/logs - - uses: actions/upload-artifact@v1 - with: - name: cover - path: _build/test/cover diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml new file mode 100644 index 000000000..d4434a87f --- /dev/null +++ b/.github/workflows/run_test_cases.yaml @@ -0,0 +1,81 @@ +name: Run test case + +on: + push: + tags: + - v* + release: + types: + - published + pull_request: + workflow_dispatch: + repository_dispatch: + types: [run_actions] + +jobs: + run_test_case: + runs-on: ubuntu-latest + + strategy: + matrix: + mysql_vsn: [5.7, 8] + redis_vsn: [6] + mongo_vsn: [3, 4] + pgsql_vsn: [11, 12, 13] + ldap_vsn: [2.4.50] + + steps: + - uses: actions/checkout@v2 + - name: set up + env: + MYSQL_VSN: ${{ matrix.mysql_vsn }} + REDIS_VSN: ${{ matrix.redis_vsn }} + MONGO_VSN: ${{ matrix.mongo_vsn }} + PGSQL_VSN: ${{ matrix.pgsql_vsn }} + LDAP_VSN: ${{ matrix.ldap_vsn }} + run: | + cp -f apps/emqx_auth_ldap/emqx.io.ldif .ci/apps_tests/emqx_ldap/schema + cp -f apps/emqx_auth_ldap/emqx.schema .ci/apps_tests/emqx_ldap/schema + cp -f apps/emqx_auth_ldap/test/certs/* .ci/apps_tests/emqx_ldap/certs + docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache + docker-compose -f .ci/apps_tests/docker-compose.yaml up -d + - name: set config files + run: | + sed -i "/auth.mysql.server/c auth.mysql.server = mysql_server:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo 'auth.mysql.ssl = on' >> apps/emqx_auth_redis/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.cafile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + + sed -i "/auth.redis.server/c auth.redis.server = redis_server:6379" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.ssl = on' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.cafile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + + sed -i "/auth.mongo.server/c auth.mongo.server = mongo_server:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl = true' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl_opts.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl_opts.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl_opts.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + + sed -i "/auth.pgsql.server/c auth.pgsql.server = pgsql_server:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "/auth.ldap.servers/c auth.ldap.servers = ldap_server" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + - name: run tests + run: | + docker exec -i erlang bash -c "make ct" + docker exec -i erlang bash -c "make cover" + - name: coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + make coveralls + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: _build/test/logs + - uses: actions/upload-artifact@v1 + with: + name: cover + path: _build/test/cover diff --git a/Makefile b/Makefile index 4f2476827..a0fed9c71 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,10 @@ eunit: $(REBAR) ct: $(REBAR) $(REBAR) ct +.PHONY: cover +cover: $(REBAR) + $(REBAR) cover + .PHONY: $(REL_PROFILES) $(REL_PROFILES:%=%): $(REBAR) ifneq ($(shell echo $(@) |grep edge),) diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index 0efccce29..c502e36b1 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -17,12 +17,12 @@ auth.mysql.pool = 8 ## MySQL username. ## ## Value: String -## auth.mysql.username = +auth.mysql.username = root ## MySQL password. ## ## Value: String -## auth.mysql.password = +auth.mysql.password = public ## MySQL database. ## @@ -98,7 +98,7 @@ auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic f ## Mysql ssl configuration. ## ## Value: on | off -auth.mysql.ssl = off +## auth.mysql.ssl = off ## CA certificate. ##