Merge pull request #12957 from id/0430-adjust-build-scripts

adjust build scripts
This commit is contained in:
Ivan Dyachkov 2024-05-02 17:05:06 +02:00 committed by GitHub
commit 3064a1cbae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 126 additions and 110 deletions

View File

@ -18,7 +18,7 @@ services:
- /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
kdc: kdc:
hostname: kdc.emqx.net hostname: kdc.emqx.net
image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 image: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04
container_name: kdc.emqx.net container_name: kdc.emqx.net
expose: expose:
- 88 # kdc - 88 # kdc

View File

@ -3,7 +3,7 @@ version: '3.9'
services: services:
erlang: erlang:
container_name: erlang container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04} image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04}
env_file: env_file:
- credentials.env - credentials.env
- conf.env - conf.env

View File

@ -20,15 +20,15 @@ permissions:
jobs: jobs:
sanity-checks: sanity-checks:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" container: "ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04"
outputs: outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }} ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }} ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
builder: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" builder: "ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04"
builder_vsn: "5.3-4" builder_vsn: "5.3-5"
otp_vsn: "26.2.1-2" otp_vsn: "26.2.1-2"
elixir_vsn: "1.15.7" elixir_vsn: "1.15.7"
@ -95,12 +95,12 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c ' MATRIX="$(echo "${APPS}" | jq -c '
[ [
(.[] | select(.profile == "emqx") | . + { (.[] | select(.profile == "emqx") | . + {
builder: "5.3-4", builder: "5.3-5",
otp: "26.2.1-2", otp: "26.2.1-2",
elixir: "1.15.7" elixir: "1.15.7"
}), }),
(.[] | select(.profile == "emqx-enterprise") | . + { (.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-4", builder: "5.3-5",
otp: ["26.2.1-2"][], otp: ["26.2.1-2"][],
elixir: "1.15.7" elixir: "1.15.7"
}) })

View File

@ -23,7 +23,7 @@ env:
jobs: jobs:
prepare: prepare:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
container: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' container: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04'
outputs: outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }} profile: ${{ steps.parse-git-ref.outputs.profile }}
release: ${{ steps.parse-git-ref.outputs.release }} release: ${{ steps.parse-git-ref.outputs.release }}
@ -31,8 +31,8 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }} ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }} ct-docker: ${{ steps.matrix.outputs.ct-docker }}
builder: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' builder: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04'
builder_vsn: '5.3-4' builder_vsn: '5.3-5'
otp_vsn: '26.2.1-2' otp_vsn: '26.2.1-2'
elixir_vsn: '1.15.7' elixir_vsn: '1.15.7'
@ -62,12 +62,12 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c ' MATRIX="$(echo "${APPS}" | jq -c '
[ [
(.[] | select(.profile == "emqx") | . + { (.[] | select(.profile == "emqx") | . + {
builder: "5.3-4", builder: "5.3-5",
otp: "26.2.1-2", otp: "26.2.1-2",
elixir: "1.15.7" elixir: "1.15.7"
}), }),
(.[] | select(.profile == "emqx-enterprise") | . + { (.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-4", builder: "5.3-5",
otp: ["26.2.1-2"][], otp: ["26.2.1-2"][],
elixir: "1.15.7" elixir: "1.15.7"
}) })

View File

@ -61,7 +61,7 @@ on:
builder_vsn: builder_vsn:
required: false required: false
type: string type: string
default: '5.3-4' default: '5.3-5'
permissions: permissions:
contents: read contents: read

View File

@ -28,7 +28,7 @@ jobs:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
env: env:
EMQX_NAME: ${{ matrix.profile }} EMQX_NAME: ${{ matrix.profile }}
PKG_VSN: ${{ startsWith(matrix.profile, 'emqx-enterprise') && inputs.version-emqx-enterprise || inputs.version-emqx }} PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
OTP_VSN: ${{ inputs.otp_vsn }} OTP_VSN: ${{ inputs.otp_vsn }}
ELIXIR_VSN: ${{ inputs.elixir_vsn }} ELIXIR_VSN: ${{ inputs.elixir_vsn }}

View File

@ -63,7 +63,7 @@ on:
builder_vsn: builder_vsn:
required: false required: false
type: string type: string
default: '5.3-4' default: '5.3-5'
permissions: permissions:
contents: read contents: read
@ -76,9 +76,8 @@ jobs:
profile: profile:
- ${{ inputs.profile }} - ${{ inputs.profile }}
os: os:
- macos-12
- macos-12-arm64
- macos-13 - macos-13
- macos-14
otp: otp:
- ${{ inputs.otp_vsn }} - ${{ inputs.otp_vsn }}
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
@ -111,6 +110,7 @@ jobs:
profile: profile:
- ${{ inputs.profile }} - ${{ inputs.profile }}
os: os:
- ubuntu24.04
- ubuntu22.04 - ubuntu22.04
- ubuntu20.04 - ubuntu20.04
- ubuntu18.04 - ubuntu18.04

View File

@ -23,10 +23,9 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
profile: profile:
- ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2'] - ['emqx', 'master', '5.3-5:1.15.7-26.2.1-2']
- ['emqx-enterprise', 'release-56', '5.3-4:1.15.7-26.2.1-2'] - ['emqx', 'release-57', '5.3-5:1.15.7-26.2.1-2']
os: os:
- debian10
- ubuntu22.04 - ubuntu22.04
- amzn2023 - amzn2023

View File

@ -27,11 +27,11 @@ on:
builder: builder:
required: false required: false
type: string type: string
default: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' default: 'ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04'
builder_vsn: builder_vsn:
required: false required: false
type: string type: string
default: '5.3-4' default: '5.3-5'
otp_vsn: otp_vsn:
required: false required: false
type: string type: string
@ -111,7 +111,7 @@ jobs:
otp: otp:
- ${{ inputs.otp_vsn }} - ${{ inputs.otp_vsn }}
os: os:
- macos-12-arm64 - macos-14
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
env: env:

View File

@ -14,32 +14,32 @@ jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
container: ${{ inputs.builder }} container: ${{ inputs.builder }}
env:
MIX_ENV: ${{ matrix.profile }}
PROFILE: ${{ matrix.profile }}
strategy:
matrix:
profile:
- emqx-enterprise
steps: steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
- run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- run: make ensure-rebar3 - run: make ensure-rebar3
- run: ./scripts/check-deps-integrity.escript - run: ./scripts/check-deps-integrity.escript
- name: Setup mix - name: Setup mix
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
run: | run: |
mix local.hex --force mix local.hex --force
mix local.rebar --force mix local.rebar --force
mix deps.get mix deps.get
- name: print mix dependency tree
run: mix deps.tree
- run: ./scripts/check-elixir-deps-discrepancies.exs - run: ./scripts/check-elixir-deps-discrepancies.exs
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- run: ./scripts/check-elixir-applications.exs - run: ./scripts/check-elixir-applications.exs
env:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- name: Upload produced lock files - name: Upload produced lock files
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure() if: failure()
with: with:
name: produced_lock_files name: ${{ matrix.profile }}_produced_lock_files
path: | path: |
mix.lock mix.lock
rebar.lock rebar.lock

View File

@ -17,14 +17,13 @@ jobs:
actions: read actions: read
security-events: write security-events: write
container: container:
image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 image: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
branch: branch:
- master - master
- release-56
- release-57 - release-57
language: language:
- cpp - cpp

View File

@ -26,7 +26,7 @@ jobs:
prepare: prepare:
runs-on: ubuntu-latest runs-on: ubuntu-latest
if: github.repository_owner == 'emqx' if: github.repository_owner == 'emqx'
container: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu20.04
outputs: outputs:
BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

View File

@ -112,6 +112,8 @@ jobs:
push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb"
push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb" push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb"
push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb" push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb"
push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-amd64.deb"
push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-arm64.deb"
push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm" push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm"
push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm" push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm"
push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm" push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm"

View File

@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12
export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim
export EMQX_REL_FORM ?= tgz export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1 export QUICER_DOWNLOAD_FROM_RELEASE = 1

View File

@ -81,11 +81,13 @@ get_override_config_file() ->
end end
end. end.
-define(DATA_DIRS, ["authz", "certs"]).
sync_data_from_node() -> sync_data_from_node() ->
Dir = emqx:data_dir(), Dir = emqx:data_dir(),
TargetDirs = lists:filter(fun(Type) -> filelib:is_dir(filename:join(Dir, Type)) end, [ TargetDirs = lists:filter(
"authz", "certs" fun(Type) -> filelib:is_dir(filename:join(Dir, Type)) end, ?DATA_DIRS
]), ),
Name = "data.zip", Name = "data.zip",
case zip:zip(Name, TargetDirs, [memory, {cwd, Dir}]) of case zip:zip(Name, TargetDirs, [memory, {cwd, Dir}]) of
{ok, {Name, Bin}} -> {ok, Bin}; {ok, {Name, Bin}} -> {ok, Bin};

2
build
View File

@ -395,7 +395,7 @@ function is_ecr_and_enterprise() {
## Build the default docker image based on debian 12. ## Build the default docker image based on debian 12.
make_docker() { make_docker() {
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-4}" local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-5}"
local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}" local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}"
local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}" local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}"
local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}" local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}"

View File

@ -0,0 +1 @@
Stop building packages for macOS 12.

View File

@ -0,0 +1,2 @@
Start building packages for macOS 14 (Apple Silicon).
Start building packages for Ubuntu 24.04 Noble Numbat (LTS).

View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12
ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim
ARG SOURCE_TYPE=src # tgz ARG SOURCE_TYPE=src # tgz
@ -16,7 +16,7 @@ FROM builder_${SOURCE_TYPE} as builder
ARG PROFILE=emqx ARG PROFILE=emqx
ARG IS_ELIXIR=no ARG IS_ELIXIR=no
ARG DEBUG=0 ARG DEBUG
ENV EMQX_RELUP=false ENV EMQX_RELUP=false
ENV EMQX_REL_FORM='docker' ENV EMQX_REL_FORM='docker'

36
mix.exs
View File

@ -113,8 +113,9 @@ defmodule EMQXUmbrella.MixProject do
set_emqx_app_system_env(apps, profile_info, version) set_emqx_app_system_env(apps, profile_info, version)
end end
defp umbrella_apps(profile_info) do defp umbrella_apps(profile_info = %{release_type: release_type}) do
enterprise_apps = enterprise_umbrella_apps() enterprise_apps = enterprise_umbrella_apps(release_type)
excluded_apps = excluded_apps(release_type)
"apps/*" "apps/*"
|> Path.wildcard() |> Path.wildcard()
@ -140,10 +141,11 @@ defmodule EMQXUmbrella.MixProject do
false false
end end
end) end)
|> Enum.reject(fn {app, _} -> app in excluded_apps end)
end end
defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do defp enterprise_apps(_profile_info = %{release_type: release_type, edition_type: :enterprise}) do
Enum.map(enterprise_umbrella_apps(), fn app_name -> Enum.map(enterprise_umbrella_apps(release_type), fn app_name ->
path = "apps/#{app_name}" path = "apps/#{app_name}"
{app_name, path: path, manager: :rebar3, override: true} {app_name, path: path, manager: :rebar3, override: true}
end) end)
@ -154,7 +156,7 @@ defmodule EMQXUmbrella.MixProject do
end end
# need to remove those when listing `/apps/`... # need to remove those when listing `/apps/`...
defp enterprise_umbrella_apps() do defp enterprise_umbrella_apps(_release_type) do
MapSet.new([ MapSet.new([
:emqx_bridge_kafka, :emqx_bridge_kafka,
:emqx_bridge_confluent, :emqx_bridge_confluent,
@ -304,7 +306,7 @@ defmodule EMQXUmbrella.MixProject do
end end
[ [
applications: applications(edition_type), applications: applications(release_type, edition_type),
skip_mode_validation_for: [ skip_mode_validation_for: [
:emqx_mix, :emqx_mix,
:emqx_gateway, :emqx_gateway,
@ -344,7 +346,7 @@ defmodule EMQXUmbrella.MixProject do
] ]
end end
def applications(edition_type) do def applications(release_type, edition_type) do
{:ok, {:ok,
[ [
%{ %{
@ -365,7 +367,7 @@ defmodule EMQXUmbrella.MixProject do
business_apps = common_business_apps ++ edition_specific_apps business_apps = common_business_apps ++ edition_specific_apps
excluded_apps = excluded_apps() excluded_apps = excluded_apps(release_type)
system_apps = system_apps =
Enum.map(system_apps, fn app -> Enum.map(system_apps, fn app ->
@ -380,7 +382,7 @@ defmodule EMQXUmbrella.MixProject do
|> Keyword.reject(fn {app, _type} -> app in excluded_apps end) |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
end end
defp excluded_apps() do defp excluded_apps(_release_type) do
%{ %{
mnesia_rocksdb: enable_rocksdb?(), mnesia_rocksdb: enable_rocksdb?(),
quicer: enable_quicer?(), quicer: enable_quicer?(),
@ -451,19 +453,19 @@ defmodule EMQXUmbrella.MixProject do
} = } =
case Mix.env() do case Mix.env() do
:dev -> :dev ->
{:cloud, :bin, :community} {:standard, :bin, :community}
:emqx -> :emqx ->
{:cloud, :bin, :community} {:standard, :bin, :community}
:"emqx-enterprise" -> :"emqx-enterprise" ->
{:cloud, :bin, :enterprise} {:standard, :bin, :enterprise}
:"emqx-pkg" -> :"emqx-pkg" ->
{:cloud, :pkg, :community} {:standard, :pkg, :community}
:"emqx-enterprise-pkg" -> :"emqx-enterprise-pkg" ->
{:cloud, :pkg, :enterprise} {:standard, :pkg, :enterprise}
end end
normalize_env!() normalize_env!()
@ -566,7 +568,7 @@ defmodule EMQXUmbrella.MixProject do
vm_args_template_path = vm_args_template_path =
case release_type do case release_type do
:cloud -> _ ->
"apps/emqx/etc/vm.args.cloud" "apps/emqx/etc/vm.args.cloud"
end end
@ -780,10 +782,10 @@ defmodule EMQXUmbrella.MixProject do
defp emqx_description(release_type, edition_type) do defp emqx_description(release_type, edition_type) do
case {release_type, edition_type} do case {release_type, edition_type} do
{:cloud, :enterprise} -> {_, :enterprise} ->
"EMQX Enterprise" "EMQX Enterprise"
{:cloud, :community} -> {_, :community} ->
"EMQX" "EMQX"
end end
end end

View File

@ -157,17 +157,24 @@ is_rocksdb_supported(_) ->
not is_build_without("ROCKSDB"). not is_build_without("ROCKSDB").
project_app_dirs() -> project_app_dirs() ->
project_app_dirs(get_edition_from_profile_env()). #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(),
project_app_dirs(Edition, RelType).
project_app_dirs(Edition) -> project_app_dirs(Edition, RelType) ->
IsEnterprise = is_enterprise(Edition), IsEnterprise = is_enterprise(Edition),
ExcludedApps = excluded_apps(RelType),
UmbrellaApps = [ UmbrellaApps = [
Path Path
|| Path <- filelib:wildcard("apps/*"), || Path <- filelib:wildcard("apps/*"),
is_community_umbrella_app(Path) orelse IsEnterprise not project_app_excluded(Path, ExcludedApps) andalso
(is_community_umbrella_app(Path) orelse IsEnterprise)
], ],
UmbrellaApps. UmbrellaApps.
project_app_excluded("apps/" ++ AppStr, ExcludedApps) ->
App = list_to_atom(AppStr),
lists:member(App, ExcludedApps).
plugins() -> plugins() ->
[ [
%{relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}, %{relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}},
@ -196,9 +203,10 @@ test_deps() ->
]. ].
common_compile_opts() -> common_compile_opts() ->
common_compile_opts(get_edition_from_profile_env(), undefined). #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(),
common_compile_opts(Edition, RelType, undefined).
common_compile_opts(Edition, Vsn) -> common_compile_opts(Edition, _RelType, Vsn) ->
% always include debug_info % always include debug_info
[ [
debug_info, debug_info,
@ -224,71 +232,72 @@ warn_profile_env() ->
get_edition_from_profile_env() -> get_edition_from_profile_env() ->
case os:getenv("PROFILE") of case os:getenv("PROFILE") of
"emqx-enterprise" ++ _ -> "emqx-enterprise" ++ _ ->
ee; #{edition => ee, reltype => standard};
"emqx" ++ _ -> "emqx" ++ _ ->
ce; #{edition => ce, reltype => standard};
false -> false ->
ee; #{edition => ee, reltype => standard};
V -> V ->
io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]), io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [V]),
exit(bad_PROFILE) exit(bad_PROFILE)
end. end.
prod_compile_opts(Edition, Vsn) -> prod_compile_opts(Edition, RelType, Vsn) ->
[ [
compressed, compressed,
deterministic, deterministic,
warnings_as_errors warnings_as_errors
| common_compile_opts(Edition, Vsn) | common_compile_opts(Edition, RelType, Vsn)
]. ].
prod_overrides() -> prod_overrides() ->
[{add, [{erl_opts, [deterministic]}]}]. [{add, [{erl_opts, [deterministic]}]}].
profiles() -> profiles() ->
case get_edition_from_profile_env() of #{edition := Edition, reltype := RelType} = get_edition_from_profile_env(),
case Edition of
ee -> ee ->
profiles_ee(); profiles_ee(RelType);
ce -> ce ->
profiles_ce() profiles_ce(RelType)
end ++ profiles_dev(). end ++ profiles_dev(RelType).
profiles_ce() -> profiles_ce(RelType) ->
Vsn = get_vsn(emqx), Vsn = get_vsn(emqx),
[ [
{'emqx', [ {'emqx', [
{erl_opts, prod_compile_opts(ce, Vsn)}, {erl_opts, prod_compile_opts(ce, RelType, Vsn)},
{relx, relx(Vsn, cloud, bin, ce)}, {relx, relx(Vsn, RelType, bin, ce)},
{overrides, prod_overrides()}, {overrides, prod_overrides()},
{project_app_dirs, project_app_dirs(ce)} {project_app_dirs, project_app_dirs(ce, RelType)}
]}, ]},
{'emqx-pkg', [ {'emqx-pkg', [
{erl_opts, prod_compile_opts(ce, Vsn)}, {erl_opts, prod_compile_opts(ce, RelType, Vsn)},
{relx, relx(Vsn, cloud, pkg, ce)}, {relx, relx(Vsn, RelType, pkg, ce)},
{overrides, prod_overrides()}, {overrides, prod_overrides()},
{project_app_dirs, project_app_dirs(ce)} {project_app_dirs, project_app_dirs(ce, RelType)}
]} ]}
]. ].
profiles_ee() -> profiles_ee(RelType) ->
Vsn = get_vsn('emqx-enterprise'), Vsn = get_vsn('emqx-enterprise'),
[ [
{'emqx-enterprise', [ {'emqx-enterprise', [
{erl_opts, prod_compile_opts(ee, Vsn)}, {erl_opts, prod_compile_opts(ee, RelType, Vsn)},
{relx, relx(Vsn, cloud, bin, ee)}, {relx, relx(Vsn, RelType, bin, ee)},
{overrides, prod_overrides()}, {overrides, prod_overrides()},
{project_app_dirs, project_app_dirs(ee)} {project_app_dirs, project_app_dirs(ee, RelType)}
]}, ]},
{'emqx-enterprise-pkg', [ {'emqx-enterprise-pkg', [
{erl_opts, prod_compile_opts(ee, Vsn)}, {erl_opts, prod_compile_opts(ee, RelType, Vsn)},
{relx, relx(Vsn, cloud, pkg, ee)}, {relx, relx(Vsn, RelType, pkg, ee)},
{overrides, prod_overrides()}, {overrides, prod_overrides()},
{project_app_dirs, project_app_dirs(ee)} {project_app_dirs, project_app_dirs(ee, RelType)}
]} ]}
]. ].
%% EE has more files than CE, always test/check with EE options. %% EE has more files than CE, always test/check with EE options.
profiles_dev() -> profiles_dev(_RelType) ->
[ [
{check, [ {check, [
{erl_opts, common_compile_opts()}, {erl_opts, common_compile_opts()},
@ -302,7 +311,7 @@ profiles_dev() ->
]} ]}
]. ].
%% RelType: cloud (full size) %% RelType: standard
%% PkgType: bin | pkg %% PkgType: bin | pkg
%% Edition: ce (opensource) | ee (enterprise) %% Edition: ce (opensource) | ee (enterprise)
relx(Vsn, RelType, PkgType, Edition) -> relx(Vsn, RelType, PkgType, Edition) ->
@ -341,10 +350,10 @@ relform() ->
Other -> Other Other -> Other
end. end.
emqx_description(cloud, ee) -> "EMQX Enterprise"; emqx_description(_, ee) -> "EMQX Enterprise";
emqx_description(cloud, ce) -> "EMQX". emqx_description(_, ce) -> "EMQX".
overlay_vars(cloud, PkgType, Edition) -> overlay_vars(_RelType, PkgType, Edition) ->
[ [
{emqx_default_erlang_cookie, "emqxsecretcookie"} {emqx_default_erlang_cookie, "emqxsecretcookie"}
] ++ ] ++
@ -411,15 +420,16 @@ relx_apps(ReleaseType, Edition) ->
ce -> CEBusinessApps ce -> CEBusinessApps
end, end,
BusinessApps = CommonBusinessApps ++ EditionSpecificApps, BusinessApps = CommonBusinessApps ++ EditionSpecificApps,
ExcludedApps = excluded_apps(ReleaseType),
Apps = Apps =
(SystemApps ++ ([App || App <- SystemApps, not lists:member(App, ExcludedApps)] ++
%% EMQX starts the DB and the business applications: %% EMQX starts the DB and the business applications:
[{App, load} || App <- DBApps] ++ [{App, load} || App <- DBApps, not lists:member(App, ExcludedApps)] ++
[emqx_machine] ++ [emqx_machine] ++
[{App, load} || App <- BusinessApps]), [{App, load} || App <- BusinessApps, not lists:member(App, ExcludedApps)]),
lists:foldl(fun proplists:delete/2, Apps, excluded_apps(ReleaseType)). Apps.
excluded_apps(_ReleaseType) -> excluded_apps(_RelType) ->
OptionalApps = [ OptionalApps = [
{quicer, is_quicer_supported()}, {quicer, is_quicer_supported()},
{jq, is_jq_supported()}, {jq, is_jq_supported()},
@ -489,7 +499,7 @@ emqx_etc_overlay(ReleaseType) ->
emqx_etc_overlay_per_rel(ReleaseType) ++ emqx_etc_overlay_per_rel(ReleaseType) ++
emqx_etc_overlay(). emqx_etc_overlay().
emqx_etc_overlay_per_rel(cloud) -> emqx_etc_overlay_per_rel(_RelType) ->
[{"{{base_dir}}/lib/emqx/etc/vm.args.cloud", "etc/vm.args"}]. [{"{{base_dir}}/lib/emqx/etc/vm.args.cloud", "etc/vm.args"}].
emqx_etc_overlay() -> emqx_etc_overlay() ->
@ -543,10 +553,9 @@ dialyzer(Config) ->
end, end,
AppNames = app_names(), AppNames = app_names(),
KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)], KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)],
ExcludedApps = excluded_apps(standard),
AppsToExclude = AppNames -- KnownApps, AppsToExclude = ExcludedApps ++ (AppNames -- KnownApps),
Extra = Extra =
[system_monitor, tools, covertool] ++ [system_monitor, tools, covertool] ++

View File

@ -9,7 +9,7 @@
## example: ## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 ## --builder ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12
set -euo pipefail set -euo pipefail
@ -24,7 +24,7 @@ help() {
echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD" echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull" echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12" echo " E.g. ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-debian12"
} }
die() { die() {

View File

@ -22,7 +22,7 @@ defmodule CheckElixirApplications do
env: [{"DEBUG", "1"}] env: [{"DEBUG", "1"}]
) )
mix_apps = mix_applications(inputs.edition_type) mix_apps = mix_applications(inputs.release_type, inputs.edition_type)
rebar_apps = rebar_applications(profile) rebar_apps = rebar_applications(profile)
results = diff_apps(mix_apps, rebar_apps) results = diff_apps(mix_apps, rebar_apps)
@ -70,8 +70,8 @@ defmodule CheckElixirApplications do
end end
end end
defp mix_applications(edition_type) do defp mix_applications(release_type, edition_type) do
EMQXUmbrella.MixProject.applications(edition_type) EMQXUmbrella.MixProject.applications(release_type, edition_type)
end end
defp rebar_applications(profile) do defp rebar_applications(profile) do

View File

@ -14,7 +14,7 @@ profile = Mix.env()
# need to use this information because we might have compiled all # need to use this information because we might have compiled all
# applications in the test profile, and thus filter what's in the # applications in the test profile, and thus filter what's in the
# release lib directory. # release lib directory.
rel_apps = MixProject.applications(inputs.edition_type) rel_apps = MixProject.applications(inputs.release_type, inputs.edition_type)
apps = apps =
rel_apps rel_apps

View File

@ -12,7 +12,7 @@ if ! type "yq" > /dev/null; then
exit 1 exit 1
fi fi
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-4} EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-5}
EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.2.1-2} EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.2.1-2}
EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7} EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7}
EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}

View File

@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
BENCH="bench.$NET" BENCH="bench.$NET"
COOKIE='this-is-a-secret' COOKIE='this-is-a-secret'
## Erlang image is needed to run webhook server and emqtt-bench ## Erlang image is needed to run webhook server and emqtt-bench
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-5:1.15.7-26.2.1-2-ubuntu22.04"
# builder has emqtt-bench installed # builder has emqtt-bench installed
BENCH_IMAGE="$ERLANG_IMAGE" BENCH_IMAGE="$ERLANG_IMAGE"