Merge pull request #12085 from zmstone/1201-otp-26-in-ci

ci: upgrade to otp 26
This commit is contained in:
Zaiming (Stone) Shi 2023-12-15 10:23:39 +01:00 committed by GitHub
commit 7ad2acb337
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
54 changed files with 256 additions and 186 deletions

View File

@ -18,7 +18,7 @@ services:
- /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
kdc:
hostname: kdc.emqx.net
image: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04
image: ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04
container_name: kdc.emqx.net
expose:
- 88 # kdc

View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04}
env_file:
- credentials.env
- conf.env

View File

@ -3,7 +3,7 @@ inputs:
profile: # emqx, emqx-enterprise
required: true
type: string
otp: # 25.3.2-2
otp:
required: true
type: string
os:

View File

@ -17,17 +17,17 @@ env:
jobs:
sanity-checks:
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04"
container: "ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04"
outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
builder: "ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04"
builder_vsn: "5.2-3"
otp_vsn: "25.3.2-2"
elixir_vsn: "1.14.5"
builder: "ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04"
builder_vsn: "5.2-8"
otp_vsn: "26.1.2-2"
elixir_vsn: "1.15.7"
steps:
- uses: actions/checkout@v3
@ -92,14 +92,14 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.2-3",
otp: "25.3.2-2",
elixir: "1.14.5"
builder: "5.2-8",
otp: "26.1.2-2",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.2-3",
otp: ["25.3.2-2"][],
elixir: "1.14.5"
builder: "5.2-8",
otp: ["26.1.2-2"][],
elixir: "1.15.7"
})
]
')"

View File

@ -20,7 +20,7 @@ env:
jobs:
prepare:
runs-on: ubuntu-22.04
container: 'ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04'
container: 'ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04'
outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
release: ${{ steps.parse-git-ref.outputs.release }}
@ -29,10 +29,10 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
builder: 'ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04'
builder_vsn: '5.2-3'
otp_vsn: '25.3.2-2'
elixir_vsn: '1.14.5'
builder: 'ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04'
builder_vsn: '5.2-8'
otp_vsn: '26.1.2-2'
elixir_vsn: '1.15.7'
steps:
- uses: actions/checkout@v3
@ -62,14 +62,14 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.2-3",
otp: "25.3.2-2",
elixir: "1.14.5"
builder: "5.2-8",
otp: "26.1.2-2",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.2-3",
otp: ["25.3.2-2"][],
elixir: "1.14.5"
builder: "5.2-8",
otp: ["26.1.2-2"][],
elixir: "1.15.7"
})
]
')"

View File

@ -58,15 +58,15 @@ on:
otp_vsn:
required: false
type: string
default: '25.3.2-2'
default: '26.1.2-2'
elixir_vsn:
required: false
type: string
default: '1.14.5'
default: '1.15.7'
builder_vsn:
required: false
type: string
default: '5.2-3'
default: '5.2-8'
permissions:
contents: read

View File

@ -54,15 +54,15 @@ on:
otp_vsn:
required: false
type: string
default: '25.3.2-2'
default: '26.1.2-2'
elixir_vsn:
required: false
type: string
default: '1.14.5'
default: '1.15.7'
builder_vsn:
required: false
type: string
default: '5.2-3'
default: '5.2-8'
jobs:
mac:

View File

@ -14,7 +14,7 @@ jobs:
if: github.repository_owner == 'emqx'
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
container:
image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}-${{ matrix.os }}"
strategy:
fail-fast: false
@ -22,8 +22,6 @@ jobs:
profile:
- ['emqx', 'master']
- ['emqx-enterprise', 'release-54']
otp:
- 25.3.2-2
arch:
- x64
os:
@ -31,9 +29,7 @@ jobs:
- ubuntu22.04
- amzn2023
builder:
- 5.2-3
elixir:
- 1.14.5
- 5.2-8:1.15.7-26.1.2-2
defaults:
run:
@ -98,7 +94,7 @@ jobs:
branch:
- master
otp:
- 25.3.2-2
- 26.1.2-2
os:
- macos-12-arm64

View File

@ -27,19 +27,19 @@ on:
builder:
required: false
type: string
default: 'ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04'
default: 'ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04'
builder_vsn:
required: false
type: string
default: '5.2-3'
default: '5.2-8'
otp_vsn:
required: false
type: string
default: '25.3.2-2'
default: '26.1.2-2'
elixir_vsn:
required: false
type: string
default: '1.14.5'
default: '1.15.7'
jobs:
linux:
@ -51,8 +51,8 @@ jobs:
fail-fast: false
matrix:
profile:
- ["emqx", "25.3.2-2", "ubuntu20.04", "elixir"]
- ["emqx-enterprise", "25.3.2-2", "ubuntu20.04", "erlang"]
- ["emqx", "26.1.2-2", "ubuntu20.04", "elixir"]
- ["emqx-enterprise", "26.1.2-2", "ubuntu20.04", "erlang"]
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"

View File

@ -20,7 +20,7 @@ jobs:
actions: read
security-events: write
container:
image: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04
image: ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04
strategy:
fail-fast: false

View File

@ -26,7 +26,7 @@ jobs:
prepare:
runs-on: ubuntu-latest
if: github.repository_owner == 'emqx'
container: ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu20.04
outputs:
BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

View File

@ -74,7 +74,7 @@ jobs:
steps:
- uses: erlef/setup-beam@v1.16.0
with:
otp-version: 25.3.2
otp-version: 26.1.2
- uses: actions/checkout@v3
with:
repository: hawk/lux

View File

@ -1,2 +1,2 @@
erlang 26.1.2-1
erlang 26.1.2-2
elixir 1.15.7-otp-26

View File

@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-debian11
export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:11-slim
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
@ -102,7 +102,7 @@ endif
# Allow user-set GROUPS environment variable
ifneq ($(GROUPS),)
GROUPS_ARG := --groups $(GROUPS)
GROUPS_ARG := --group $(GROUPS)
endif
ifeq ($(ENABLE_COVER_COMPILE),1)

View File

@ -83,7 +83,9 @@ docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p
Ветка `master` предназначена для последней версии 5, переключитесь на ветку `main-v4.4` для версии 4.4.
EMQX требует OTP 24 для версии 4.4. Версию 5.0 можно собирать с OTP 24 или 25.
EMQX требует OTP 24 для версии 4.4.
Версии 5.0 ~ 5.3 могут быть собраны с OTP 24 или 25.
Версия 5.4 и новее могут быть собраны с OTP 25 или 26.
```bash
git clone https://github.com/emqx/emqx.git

View File

@ -95,7 +95,9 @@ For more organised improvement proposals, you can send pull requests to [EIP](ht
The `master` branch tracks the latest version 5. For version 4.4 checkout the `main-v4.4` branch.
EMQX 4.4 requires OTP 24. EMQX 5.0 and 5.1 can be built with OTP 24 or 25.
EMQX 4.4 requires OTP 24.
EMQX 5.0 ~ 5.3 can be built with OTP 24 or 25.
EMQX 5.4 and newer can be built with OTP 24 or 25.
```bash
git clone https://github.com/emqx/emqx.git

View File

@ -516,7 +516,7 @@ do_t_session_expiration(_Config, Opts) ->
t_session_gc(Config) ->
GCInterval = ?config(gc_interval, Config),
[Node1, Node2, Node3] = Nodes = ?config(nodes, Config),
[Node1, Node2, _Node3] = Nodes = ?config(nodes, Config),
CoreNodes = [Node1, Node2],
[
Port1,

View File

@ -66,7 +66,7 @@
{plt_location, "."},
{plt_prefix, "emqx_dialyzer"},
{plt_apps, all_apps},
{plt_extra_apps, [hocon]},
{plt_extra_apps, [hocon,mnesia,bcrypt,os_mon,pbkdf2,emqx_http_lib, recon]},
{statistics, true}
]}.

View File

@ -175,19 +175,11 @@
%% Behaviour
%% -------------------------------------------------------------------
-if(?OTP_RELEASE < 26).
-callback create(clientinfo(), conninfo(), conf()) ->
term().
-callback open(clientinfo(), conninfo(), conf()) ->
term().
-callback destroy(t() | clientinfo()) -> ok.
-else.
-callback create(clientinfo(), conninfo(), conf()) ->
t().
-callback open(clientinfo(), conninfo(), conf()) ->
{_IsPresent :: true, t(), _ReplayContext} | false.
-callback destroy(t() | clientinfo()) -> ok.
-endif.
%%--------------------------------------------------------------------
%% Create a Session

View File

@ -94,7 +94,7 @@ init_per_group(quic, Config) ->
[
{conn_fun, quic_connect},
{port, emqx_config:get([listeners, quic, test, bind])},
{ssl_opts, emqx_common_test_helpers:client_ssl_twoway()},
{ssl_opts, emqx_common_test_helpers:client_mtls()},
{ssl, true},
{group_apps, Apps}
| Config

View File

@ -395,7 +395,7 @@ t_peercert_preserved_before_connected(_) ->
?HP_HIGHEST
),
ClientId = atom_to_binary(?FUNCTION_NAME),
SslConf = emqx_common_test_helpers:client_ssl_twoway(default),
SslConf = emqx_common_test_helpers:client_mtls(default),
{ok, Client} = emqtt:start_link([
{port, 8883},
{clientid, ClientId},
@ -455,7 +455,7 @@ tls_certcn_as_clientid(TLSVsn) ->
tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) ->
CN = <<"Client">>,
emqx_config:put_zone_conf(default, [mqtt, peer_cert_as_clientid], cn),
SslConf = emqx_common_test_helpers:client_ssl_twoway(TLSVsn),
SslConf = emqx_common_test_helpers:client_mtls(TLSVsn),
{ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
{ok, _} = emqtt:connect(Client),
#{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),

View File

@ -48,8 +48,10 @@
-export([
client_ssl/0,
client_ssl/1,
client_ssl_twoway/0,
client_ssl_twoway/1,
client_mtls/0,
client_mtls/1,
ssl_verify_fun_allow_any_host/0,
ssl_verify_fun_allow_any_host_impl/3,
ensure_mnesia_stopped/0,
ensure_quic_listener/2,
ensure_quic_listener/3,
@ -430,11 +432,11 @@ flush(Msgs) ->
after 0 -> lists:reverse(Msgs)
end.
client_ssl_twoway() ->
client_ssl_twoway(default).
client_mtls() ->
client_mtls(default).
client_ssl_twoway(TLSVsn) ->
client_certs() ++ ciphers(TLSVsn).
client_mtls(TLSVsn) ->
ssl_verify_fun_allow_any_host() ++ client_certs() ++ ciphers(TLSVsn).
%% Paths prepended to cert filenames
client_certs() ->
@ -1421,3 +1423,24 @@ group_path(Config) ->
_:_ ->
[]
end.
%% almost verify_none equivalent, but only ignores 'hostname_check_failed'
ssl_verify_fun_allow_any_host_impl(_Cert, Event, State) ->
case Event of
valid ->
{valid, State};
valid_peer ->
{valid, State};
{bad_cert, hostname_check_failed} ->
{valid, State};
{bad_cert, _} ->
{fail, Event};
{extension, _} ->
{unknown, State}
end.
ssl_verify_fun_allow_any_host() ->
[
{verify, verify_peer},
{verify_fun, {fun ?MODULE:ssl_verify_fun_allow_any_host_impl/3, _State = #{}}}
].

View File

@ -387,7 +387,8 @@ assert_successful_connection(Config, ClientNum) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -601,7 +602,8 @@ t_cache(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -619,7 +621,8 @@ t_cache(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -829,7 +832,8 @@ t_filled_cache(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -853,7 +857,8 @@ t_not_cached_and_unreachable(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -871,7 +876,8 @@ t_revoked(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -893,7 +899,8 @@ t_revoke_then_refresh(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -916,7 +923,8 @@ t_revoke_then_refresh(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -961,7 +969,8 @@ do_t_update_listener(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),
@ -999,7 +1008,8 @@ do_t_update_listener(Config) ->
{ssl, true},
{ssl_opts, [
{certfile, ClientCert},
{keyfile, ClientKey}
{keyfile, ClientKey},
{verify, verify_none}
]},
{port, 8883}
]),

View File

@ -915,10 +915,8 @@ do_t_validations(_Config) ->
#{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} =
emqx_utils_json:decode(ResRaw3, [return_maps]),
%% we can't remove certfile now, because it has default value.
?assertMatch(
<<"{bad_ssl_config,#{file_read => enoent,pem_check => invalid_pem", _/binary>>,
MsgRaw3
),
?assertMatch({match, _}, re:run(MsgRaw3, <<"enoent">>)),
?assertMatch({match, _}, re:run(MsgRaw3, <<"invalid_pem">>)),
ok.
t_unknown_error_fetching_ocsp_response(_Config) ->

View File

@ -115,7 +115,7 @@ init_per_group(quic, Config) ->
[
{port, get_listener_port(quic, test)},
{conn_fun, quic_connect},
{ssl_opts, emqx_common_test_helpers:client_ssl_twoway()},
{ssl_opts, emqx_common_test_helpers:client_mtls()},
{ssl, true},
{group_apps, Apps}
| Config

View File

@ -739,11 +739,16 @@ t_qos1_random_dispatch_if_all_members_are_down(Config) when is_list(Config) ->
?assert(is_process_alive(Pid2)),
{ok, _} = emqtt:publish(ConnPub, Topic, <<"hello11">>, 1),
ct:sleep(100),
Msgs1 = emqx_mqueue:to_list(get_mqueue(Pid1)),
Msgs2 = emqx_mqueue:to_list(get_mqueue(Pid2)),
%% assert the message is in mqueue (because socket is closed)
?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2),
?retry(
100,
10,
begin
Msgs1 = emqx_mqueue:to_list(get_mqueue(Pid1)),
Msgs2 = emqx_mqueue:to_list(get_mqueue(Pid2)),
%% assert the message is in mqueue (because socket is closed)
?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2)
end
),
emqtt:stop(ConnPub),
ok.

View File

@ -1,6 +1,6 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0-emqx-2"}}}
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0.3"}}}
, {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}

View File

@ -205,7 +205,9 @@ on_start(
http ->
{tcp, []};
https ->
SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)),
SSLConf = maps:get(ssl, Config),
%% force enable ssl
SSLOpts = emqx_tls_lib:to_client_opts(SSLConf#{enable => true}),
{tls, SSLOpts}
end,
NTransportOpts = emqx_utils:ipv6_probe(TransportOpts),

View File

@ -124,7 +124,7 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)).
t_tls_verify_none(Config) ->
PoolName = <<"emqx_bridge_influxdb_connector_SUITE">>,
PoolName = <<"testpool-1">>,
Host = ?config(influxdb_tls_host, Config),
Port = ?config(influxdb_tls_port, Config),
InitialConfig = influxdb_config(Host, Port, true, <<"verify_none">>),
@ -135,7 +135,7 @@ t_tls_verify_none(Config) ->
ok.
t_tls_verify_peer(Config) ->
PoolName = <<"emqx_bridge_influxdb_connector_SUITE">>,
PoolName = <<"testpool-2">>,
Host = ?config(influxdb_tls_host, Config),
Port = ?config(influxdb_tls_port, Config),
InitialConfig = influxdb_config(Host, Port, true, <<"verify_peer">>),
@ -157,7 +157,11 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) ->
to_client_opts,
fun(Opts) ->
Verify = {verify_fun, {custom_verify(), {return, VerifyReturn}}},
[Verify | meck:passthrough([Opts])]
[
Verify,
{cacerts, public_key:cacerts_get()}
| meck:passthrough([Opts])
]
end
),
try

View File

@ -1,6 +1,6 @@
%% -*- mode: erlang; -*-
{erl_opts, [debug_info]}.
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0-emqx-2"}}}
{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0.3"}}}
, {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}
, {emqx_bridge, {path, "../../apps/emqx_bridge"}}

View File

@ -230,19 +230,18 @@ t_conf_bridge_authn_passfile(Config) ->
?assertReceive(
{authenticate, #{username := Username2, password := Password2}}
),
?assertMatch(
{ok, 201, #{
<<"status">> := <<"disconnected">>,
<<"status_reason">> := <<"#{msg => failed_to_read_secret_file", _/bytes>>
}},
{ok, 201, #{
<<"status">> := <<"disconnected">>,
<<"status_reason">> := Reason
}} =
request_json(
post,
uri(["bridges"]),
?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{
<<"name">> => <<"t_conf_bridge_authn_no_passfile">>
}
)
).
),
?assertMatch({match, _}, re:run(Reason, <<"failed_to_read_secret_file">>)).
hook_authenticate() ->
emqx_hooks:add('client.authenticate', {?MODULE, authenticate, [self()]}, ?HP_HIGHEST).

View File

@ -385,7 +385,6 @@ start_consumer(TestCase, Config) ->
<<Scheme/binary, (list_to_binary(PulsarHost))/binary, ":",
(integer_to_binary(PulsarPort))/binary>>
),
ConnOpts = #{},
ConsumerClientId = list_to_atom(
atom_to_list(TestCase) ++ integer_to_list(erlang:unique_integer())
),
@ -396,15 +395,9 @@ start_consumer(TestCase, Config) ->
certfile => filename:join([CertsPath, "cert.pem"]),
cacertfile => filename:join([CertsPath, "cacert.pem"])
},
{ok, _ClientPid} = pulsar:ensure_supervised_client(
ConsumerClientId,
[URL],
#{
conn_opts => ConnOpts,
ssl_opts => emqx_tls_lib:to_client_opts(SSLOpts)
}
),
ConsumerOpts = #{
Opts = #{enable_ssl => UseTLS, ssl_opts => emqx_tls_lib:to_client_opts(SSLOpts)},
{ok, _ClientPid} = pulsar:ensure_supervised_client(ConsumerClientId, [URL], Opts),
ConsumerOpts = Opts#{
cb_init_args => #{send_to => self()},
cb_module => pulsar_echo_consumer,
sub_type => 'Shared',
@ -414,8 +407,7 @@ start_consumer(TestCase, Config) ->
%% id, or else weird bugs will happen, like the
%% consumer never starts...
name => list_to_atom("test_consumer" ++ integer_to_list(erlang:unique_integer())),
consumer_id => 1,
conn_opts => ConnOpts
consumer_id => 1
},
{ok, Consumer} = pulsar:ensure_supervised_consumers(
ConsumerClientId,

View File

@ -4,7 +4,7 @@
%% The following two are dependencies of rabbit_common
{thoas, {git, "https://github.com/emqx/thoas.git", {tag, "v1.0.0"}}}
, {credentials_obfuscation, {git, "https://github.com/emqx/credentials-obfuscation.git", {tag, "v3.2.0"}}}
%% The v3.11.13_with_app_src tag, employed in the next two dependencies,
%% The v3.11.13.2 (v3.11.13_with_app_src branch), employed in the next two dependencies,
%% represents a fork of the official RabbitMQ v3.11.13 tag. This fork diverges
%% from the official version as it includes app and hrl files
%% generated by make files in subdirectories deps/rabbit_common and
@ -17,11 +17,11 @@
%% packages that we don't have control over.
, {rabbit_common, {git_subdir,
"https://github.com/emqx/rabbitmq-server.git",
{tag, "v3.11.13-emqx"},
{tag, "v3.11.13.2"},
"deps/rabbit_common"}}
, {amqp_client, {git_subdir,
"https://github.com/emqx/rabbitmq-server.git",
{tag, "v3.11.13-emqx"},
{tag, "v3.11.13.2"},
"deps/amqp_client"}}
, {emqx_connector, {path, "../../apps/emqx_connector"}}
, {emqx_resource, {path, "../../apps/emqx_resource"}}

View File

@ -308,7 +308,7 @@ t_lookup_badarg(_Config) ->
ok.
t_start_supervised_worker(_Config) ->
{ok, _} = emqx_connector_jwt_sup:start_link(),
{ok, Sup} = emqx_connector_jwt_sup:start_link(),
Config = #{resource_id := ResourceId, table := TId} = generate_config(),
{ok, Pid} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config),
Ref = emqx_connector_jwt_worker:ensure_jwt(Pid),
@ -333,7 +333,7 @@ t_start_supervised_worker(_Config) ->
?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)),
%% ensure the specs are removed from the supervision tree.
?assertEqual([], supervisor:which_children(emqx_connector_jwt_sup)),
ok.
ok = stop_jwt_sup(Sup).
t_start_supervised_worker_already_started(_Config) ->
{ok, _} = emqx_connector_jwt_sup:start_link(),
@ -344,17 +344,28 @@ t_start_supervised_worker_already_started(_Config) ->
ok.
t_start_supervised_worker_already_present(_Config) ->
{ok, _} = emqx_connector_jwt_sup:start_link(),
{ok, Sup} = emqx_connector_jwt_sup:start_link(),
Config = #{resource_id := ResourceId} = generate_config(),
{ok, Pid0} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config),
Ref = monitor(process, Pid0),
exit(Pid0, kill),
receive
{'DOWN', Ref, process, Pid0, killed} -> ok
after 1_000 -> error(worker_didnt_stop)
end,
{ok, Pid1} = emqx_connector_jwt_sup:ensure_worker_present(ResourceId, Config),
?assertNotEqual(Pid0, Pid1),
?assert(is_process_alive(Pid1)),
ok = stop_jwt_sup(Sup).
stop_jwt_sup(Sup) ->
Ref = monitor(process, Sup),
unlink(Sup),
exit(Sup, shutdown),
receive
{'DOWN', Ref, process, Sup, shutdown} -> ok
after 1000 ->
error(timeout)
end,
ok.
t_unknown_requests(_Config) ->

View File

@ -261,9 +261,10 @@ request_dashboard(Method, Url, Auth) ->
request_dashboard(Method, Url, QueryParams, Auth) ->
Request = {Url ++ "?" ++ QueryParams, [Auth]},
do_request_dashboard(Method, Request).
do_request_dashboard(Method, Request) ->
do_request_dashboard(Method, {Url, _} = Request) ->
ct:pal("Method: ~p, Request: ~p", [Method, Request]),
case httpc:request(Method, Request, [], []) of
case httpc:request(Method, Request, maybe_ssl(Url), []) of
{error, socket_closed_remotely} ->
{error, socket_closed_remotely};
{ok, {{"HTTP/1.1", Code, _}, _Headers, Return}} when
@ -276,6 +277,9 @@ do_request_dashboard(Method, Request) ->
{error, Reason}
end.
maybe_ssl("http://" ++ _) -> [];
maybe_ssl("https://" ++ _) -> [{ssl, [{verify, verify_none}]}].
auth_header_() ->
auth_header_(<<"admin">>, <<"public">>).

View File

@ -198,8 +198,25 @@ t_verify_cacertfile(_Config) ->
VerifyPeerConf1,
naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>)
),
validate_https(VerifyPeerConf2, MaxConnection, DefaultSSLCert, verify_peer),
ok.
%% we always test client with verify_none and no client cert is sent
%% since the server is configured with verify_peer
%% hence the expected observation on the client side is an error
ErrorReason =
try
validate_https(VerifyPeerConf2, MaxConnection, DefaultSSLCert, verify_peer)
catch
error:{https_client_error, Reason} ->
Reason
end,
%% There seems to be a race-condition causing the return value to vary a bit
case ErrorReason of
socket_closed_remotely ->
ok;
{ssl_error, _SslSock, {tls_alert, {certificate_required, _}}} ->
ok;
Other ->
throw({unexpected, Other})
end.
t_bad_certfile(_Config) ->
Conf = #{
@ -219,9 +236,12 @@ t_bad_certfile(_Config) ->
validate_https(Conf, MaxConnection, SSLCert, Verify) ->
emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf),
emqx_mgmt_api_test_util:init_suite([emqx_management], fun(X) -> X end),
assert_ranch_options(MaxConnection, SSLCert, Verify),
assert_https_request(),
emqx_mgmt_api_test_util:end_suite([emqx_management]).
try
assert_ranch_options(MaxConnection, SSLCert, Verify),
assert_https_request()
after
emqx_mgmt_api_test_util:end_suite([emqx_management])
end.
assert_ranch_options(MaxConnections0, SSLCert, Verify) ->
Middlewares = [emqx_dashboard_middleware, cowboy_router, cowboy_handler],
@ -286,10 +306,10 @@ assert_https_request() ->
lists:foreach(
fun(Path) ->
ApiPath = https_api_path([Path]),
?assertMatch(
{ok, _},
emqx_dashboard_SUITE:request_dashboard(get, ApiPath, Headers)
)
case emqx_dashboard_SUITE:request_dashboard(get, ApiPath, Headers) of
{ok, _} -> ok;
{error, Reason} -> error({https_client_error, Reason})
end
end,
?OVERVIEWS
).

View File

@ -315,6 +315,8 @@ t_none_ref(_Config) ->
),
ok.
namespace() -> undefined.
t_sub_fields(_Config) ->
Spec = #{
post => #{

View File

@ -125,6 +125,7 @@ t_bad_messages(Config) ->
).
t_nonexistent_file(_Config) ->
erlang:process_flag(trap_exit, true),
?assertEqual(
{error, enoent},
emqx_ft_storage_fs_reader:start_link(self(), "/a/b/c/bar")

View File

@ -143,11 +143,11 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) ->
false ->
[];
true ->
Opts1 = maps:get(ssl, Options, #{}),
Opts2 = maps:without([handshake_timeout], Opts1),
SSLOpts = emqx_tls_lib:to_server_opts(tls, Opts2),
[
{ssl_options,
maps:to_list(
maps:without([enable, handshake_timeout], maps:get(ssl, Options, #{}))
)}
{ssl_options, SSLOpts}
]
end,
ListenOnStr = emqx_listeners:format_bind(ListenOn),
@ -200,20 +200,20 @@ start_grpc_client_channel(
}}
)
end,
case emqx_utils_maps:deep_get([ssl_options, enable], Options, false) of
SSLOpts = emqx_utils_maps:deep_get([ssl_options], Options, #{}),
case maps:get(enable, SSLOpts, false) of
false ->
SvrAddr = compose_http_uri(http, Host, Port),
grpc_client_sup:create_channel_pool(GwName, SvrAddr, #{});
true ->
SslOpts = maps:to_list(maps:get(ssl, Options, #{})),
SSLOpts1 = [{nodelay, true} | emqx_tls_lib:to_client_opts(SSLOpts)],
ClientOpts = #{
gun_opts =>
#{
transport => ssl,
transport_opts => SslOpts
transport_opts => SSLOpts1
}
},
SvrAddr = compose_http_uri(https, Host, Port),
grpc_client_sup:create_channel_pool(GwName, SvrAddr, ClientOpts)
end;

View File

@ -180,7 +180,7 @@ set_special_cfg(_, _, _, _) ->
listener_confs(Type) ->
Default = #{bind => 7993, acceptors => 8},
#{Type => #{'default' => maps:merge(Default, socketopts(Type))}}.
#{Type => #{'default' => maps:merge(Default, server_socketopts(Type))}}.
default_config() ->
?CONF_DEFAULT.
@ -593,11 +593,11 @@ open(udp) ->
{ok, Sock} = gen_udp:open(0, ?TCPOPTS),
{udp, Sock};
open(ssl) ->
SslOpts = maps:to_list(client_ssl_opts()),
SslOpts = client_ssl_opts(),
{ok, SslSock} = ssl:connect("127.0.0.1", 7993, ?TCPOPTS ++ SslOpts),
{ssl, SslSock};
open(dtls) ->
SslOpts = maps:to_list(client_ssl_opts()),
SslOpts = client_ssl_opts(),
{ok, SslSock} = ssl:connect("127.0.0.1", 7993, ?DTLSOPTS ++ SslOpts),
{dtls, SslSock}.
@ -635,24 +635,24 @@ close({dtls, Sock}) ->
%%--------------------------------------------------------------------
%% Server-Opts
socketopts(tcp) ->
#{tcp_options => tcp_opts()};
socketopts(ssl) ->
server_socketopts(tcp) ->
#{tcp_options => server_tcp_opts()};
server_socketopts(ssl) ->
#{
tcp_options => tcp_opts(),
ssl_options => ssl_opts()
tcp_options => server_tcp_opts(),
ssl_options => server_ssl_opts()
};
socketopts(udp) ->
#{udp_options => udp_opts()};
socketopts(dtls) ->
server_socketopts(udp) ->
#{udp_options => server_udp_opts()};
server_socketopts(dtls) ->
#{
udp_options => udp_opts(),
dtls_options => dtls_opts()
udp_options => server_udp_opts(),
dtls_options => server_dtls_opts()
}.
tcp_opts() ->
server_tcp_opts() ->
maps:merge(
udp_opts(),
server_udp_opts(),
#{
send_timeout => 15000,
send_timeout_close => true,
@ -661,7 +661,7 @@ tcp_opts() ->
}
).
udp_opts() ->
server_udp_opts() ->
#{
recbuf => 1024,
sndbuf => 1024,
@ -669,7 +669,7 @@ udp_opts() ->
reuseaddr => true
}.
ssl_opts() ->
server_ssl_opts() ->
Certs = certs("key.pem", "cert.pem", "cacert.pem"),
maps:merge(
Certs,
@ -684,14 +684,15 @@ ssl_opts() ->
}
).
dtls_opts() ->
maps:merge(ssl_opts(), #{versions => ['dtlsv1.2', 'dtlsv1']}).
server_dtls_opts() ->
maps:merge(server_ssl_opts(), #{versions => ['dtlsv1.2', 'dtlsv1']}).
%%--------------------------------------------------------------------
%% Client-Opts
client_ssl_opts() ->
certs("client-key.pem", "client-cert.pem", "cacert.pem").
OptsWithCerts = certs("client-key.pem", "client-cert.pem", "cacert.pem"),
[{verify, verify_none} | maps:to_list(OptsWithCerts)].
certs(Key, Cert, CACert) ->
CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"),

View File

@ -150,12 +150,13 @@ tr_handler_conf(#{logs := LogsConf, exporter := ExporterConf}) ->
ssl_opts(Endpoint, SSLOpts) ->
case is_ssl(Endpoint) of
true ->
emqx_tls_lib:to_client_opts(SSLOpts);
%% force enable ssl
emqx_tls_lib:to_client_opts(SSLOpts#{enable => true});
false ->
[]
end.
is_ssl(<<"https://", _/binary>> = _Endpoint) ->
is_ssl(<<"https://", _/binary>>) ->
true;
is_ssl(_Endpoint) ->
is_ssl(<<"http://", _/binary>>) ->
false.

View File

@ -2,7 +2,7 @@
{erl_opts, [debug_info]}.
{deps, [
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.1.1"}}},
{emqx_connector, {path, "../../apps/emqx_connector"}},
{emqx_resource, {path, "../../apps/emqx_resource"}}
]}.

View File

@ -1,6 +1,6 @@
{deps, [
{emqx, {path, "../../apps/emqx"}},
{erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0-emqx-2"}}},
{erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.7.0.3"}}},
{emqx_bridge_http, {path, "../emqx_bridge_http"}}
]}.

View File

@ -129,7 +129,7 @@ t_url(Config) ->
?assertMatch(
{ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}},
httpc:request(Url)
httpc:request(get, {Url, []}, [{ssl, [{verify, verify_none}]}], [])
).
t_no_acl(Config) ->

View File

@ -64,13 +64,13 @@ groups() ->
{tcp_cases, [
t_config_switch,
t_config_switch_http_settings,
t_too_large,
t_no_profile
]},
{tls_cases, [
t_tls_error
t_tls_error,
t_config_switch_http_settings
]},
{noconn_errors, [{group, transport_errors}]},
@ -205,7 +205,8 @@ t_signed_url_download(_Config) ->
emqx_s3_client:uri(Client, Key)
end),
{ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []),
HttpOpts = [{ssl, [{verify, verify_none}]}],
{ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, HttpOpts, []),
?assertEqual(
iolist_to_binary(Data),
@ -222,7 +223,8 @@ t_signed_nonascii_url_download(_Config) ->
emqx_s3_client:uri(Client, Key)
end),
{ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []),
HttpOpts = [{ssl, [{verify, verify_none}]}],
{ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, HttpOpts, []),
?assertEqual(
iolist_to_binary(Data),

6
build
View File

@ -387,10 +387,10 @@ docker_cleanup() {
## Build the default docker image based on debian 11.
make_docker() {
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.1-4}"
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.2-8}"
local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian11}"
local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}"
local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.14.5}"
local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-26.1.2-2}"
local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}"
local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}}
local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"

View File

@ -0,0 +1 @@
Upgrade to OTP 26.1.2-2

View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-debian11
ARG RUN_FROM=public.ecr.aws/debian/debian:11-slim
FROM ${BUILD_FROM} AS builder
ARG DEBUG=0

View File

@ -79,7 +79,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by ehttpc and emqtt
{:gun, github: "emqx/gun", tag: "1.3.9", override: true},
# in conflict by emqx_connector and system_monitor
{:epgsql, github: "emqx/epgsql", tag: "4.7.0.1", override: true},
{:epgsql, github: "emqx/epgsql", tag: "4.7.1.1", override: true},
# in conflict by emqx and observer_cli
{:recon, github: "ferd/recon", tag: "2.5.1", override: true},
{:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
@ -97,6 +97,7 @@ defmodule EMQXUmbrella.MixProject do
{:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
# set by hackney (dependency)
{:ssl_verify_fun, "1.1.7", override: true},
{:rfc3339, github: "emqx/rfc3339", tag: "0.2.3", override: true},
{:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
{:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}
] ++
@ -214,12 +215,12 @@ defmodule EMQXUmbrella.MixProject do
github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
{:rabbit_common,
github: "emqx/rabbitmq-server",
tag: "v3.11.13-emqx",
tag: "v3.11.13.2",
sparse: "deps/rabbit_common",
override: true},
{:amqp_client,
github: "emqx/rabbitmq-server",
tag: "v3.11.13-emqx",
tag: "v3.11.13.2",
sparse: "deps/amqp_client",
override: true}
]

View File

@ -96,6 +96,7 @@
, {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}}
, {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}}
, {ssl_verify_fun, "1.1.7"}
, {rfc3339, {git, "https://github.com/emqx/rfc3339.git", {tag, "0.2.3"}}}
]}.
{xref_ignores,

View File

@ -562,7 +562,7 @@ dialyzer(Config) ->
AppsToExclude = AppNames -- KnownApps,
Extra =
[os_mon, system_monitor, covertool] ++
[os_mon, system_monitor, tools, covertool] ++
[bcrypt || provide_bcrypt_dep()] ++
[jq || is_jq_supported()] ++
[quicer || is_quicer_supported()],

View File

@ -9,7 +9,7 @@
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
## --builder ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-debian11
set -euo pipefail
@ -24,7 +24,7 @@ help() {
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11"
echo " E.g. ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-debian11"
}
die() {

View File

@ -12,9 +12,9 @@ if ! type "yq" > /dev/null; then
exit 1
fi
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.1-4}
EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-25.3.2-2}
EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.14.5}
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.2-8}
EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.1.2-2}
EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7}
EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}
EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}}

View File

@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
BENCH="bench.$NET"
COOKIE='this-is-a-secret'
## Erlang image is needed to run webhook server and emqtt-bench
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04"
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.2-8:1.15.7-26.1.2-2-ubuntu22.04"
# builder has emqtt-bench installed
BENCH_IMAGE="$ERLANG_IMAGE"