yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #6242 from emqx/update-otp-version-and-pkg-name-scheme 

Update otp version and pkg name scheme
This commit is contained in:
Zaiming (Stone) Shi 2021-11-22 10:34:24 +01:00 committed by GitHub
parent 6da039cf4f 188f1b52c7
commit 90a65b8d04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 466 additions and 535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml
View File

@ -1,5 +1,4 @@
x-default-emqx: &default-emqx x-default-emqx: &default-emqx
image: $TARGET:$EMQX_TAG
env_file: env_file:
- conf.cluster.env - conf.cluster.env
healthcheck: healthcheck:
@ -15,8 +14,8 @@ services:
restart: on-failure restart: on-failure
environment: environment:
- "EMQX_HOST=node1.emqx.io" - "EMQX_HOST=node1.emqx.io"
- "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_DB__BACKEND=rlog"
- "EMQX_CLUSTER__RLOG__ROLE=core" - "EMQX_DB__ROLE=core"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"
- "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false"
- "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false"
@ -29,9 +28,9 @@ services:
restart: on-failure restart: on-failure
environment: environment:
- "EMQX_HOST=node2.emqx.io" - "EMQX_HOST=node2.emqx.io"
- "EMQX_CLUSTER__DB_BACKEND=rlog" - "EMQX_DB__BACKEND=rlog"
- "EMQX_CLUSTER__RLOG__ROLE=replicant" - "EMQX_DB__ROLE=replicant"
- "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io" - "EMQX_DB__CORE_NODES=emqx@node1.emqx.io"
- "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"
- "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false"
- "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false"

2
.ci/docker-compose-file/docker-compose-emqx-cluster.yaml
View File

@ -1,7 +1,7 @@
version: '3.9' version: '3.9'
x-default-emqx: &default-emqx x-default-emqx: &default-emqx
image: $TARGET:$EMQX_TAG image: ${_EMQX_DOCKER_IMAGE_TAG}
env_file: env_file:
- conf.cluster.env - conf.cluster.env
healthcheck: healthcheck:

4
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services: services:
erlang23: erlang23:
container_name: erlang23 container_name: erlang23
image: ghcr.io/emqx/emqx-builder/5.0:23.2.7.2-emqx-2-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:
@ -23,7 +23,7 @@ services:
erlang24: erlang24:
container_name: erlang24 container_name: erlang24
image: ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04 image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04
env_file: env_file:
- conf.env - conf.env
environment: environment:

6
.ci/docker-compose-file/scripts/run-emqx.sh
View File

@ -1,7 +1,11 @@
#!/bin/bash #!/bin/bash
set -euxo pipefail set -euxo pipefail
if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] # _EMQX_DOCKER_IMAGE_TAG is shared with docker-compose file
export _EMQX_DOCKER_IMAGE_TAG="$1"
_EMQX_TEST_DB_BACKEND="${2:-${_EMQX_TEST_DB_BACKEND:-mnesia}}"
if [ "$_EMQX_TEST_DB_BACKEND" = "rlog" ]
then then
CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml" CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml"
else else

195
.github/workflows/build_packages.yaml vendored
View File

@ -11,92 +11,69 @@ on:
types: types:
- published - published
workflow_dispatch: workflow_dispatch:
inputs:
which_branch:
required: false
jobs: jobs:
prepare: prepare:
strategy:
matrix:
otp:
- "23.2.7.2-emqx-2"
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" # prepare source with any OTP version, no need for a matrix
container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
outputs: outputs:
profiles: ${{ steps.set_profile.outputs.profiles }} old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }}
old_vsns: ${{ steps.set_profile.outputs.old_vsns }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
ref: ${{ github.event.inputs.which_branch }}
path: source path: source
fetch-depth: 0 fetch-depth: 0
- name: set profile - name: find old versions
id: set_profile id: find_old_versons
shell: bash shell: bash
working-directory: source working-directory: source
run: | run: |
vsn="$(./pkg-vsn.sh)" vsn="$(./pkg-vsn.sh)"
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if make emqx-ee --dry-run > /dev/null 2>&1; then
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx-ee\"]"
else
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" - name: get_all_deps
fi
- name: get otp version
id: get_otp_version
run: | run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)" make -C source deps-all
echo "::set-output name=otp::$otp" zip -ryq source.zip source/* source/.[^.]*
- name: set get token
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: get deps
working-directory: source
run: |
make ensure-rebar3
./rebar3 as default get-deps
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: source-${{ steps.get_otp_version.outputs.otp }} name: source
path: source-${{ steps.get_otp_version.outputs.otp }}.zip path: source.zip
windows: windows:
runs-on: windows-2019 runs-on: windows-2019
needs: prepare needs: prepare
if: endsWith(github.repository, 'emqx')
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # only CE for windows
exclude: - emqx
- profile: emqx-edge otp:
- 23.2
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-23.2.7.2-emqx-2 name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: Expand-Archive -Path source-23.2.7.2-emqx-2.zip -DestinationPath ./ run: Expand-Archive -Path source.zip -DestinationPath ./
- uses: ilammy/msvc-dev-cmd@v1 - uses: ilammy/msvc-dev-cmd@v1
- uses: gleam-lang/setup-erlang@v1.1.2 - uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang id: install_erlang
## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
with: with:
otp-version: 23.2 otp-version: ${{ matrix.otp }}
- name: build - name: build
env: env:
PYTHON: python PYTHON: python
@ -108,10 +85,10 @@ jobs:
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
} }
else { else {
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
} }
## We do not build/release bcrypt and quic for windows package ## We do not build/release bcrypt and quic for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
@ -140,34 +117,32 @@ jobs:
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
with: with:
name: ${{ matrix.profile }}-23.2.7.2-emqx-2 name: ${{ matrix.profile }}
path: source/_packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
mac: mac:
needs: prepare needs: prepare
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # no EDGE for mac
- emqx
- emqx-ee
otp:
- 24.1.5-2
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15
otp:
- 24.1.1-emqx-1
exclude: exclude:
- profile: emqx-edge - profile: emqx-edge
runs-on: ${{ matrix.macos }} runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: prepare - name: prepare
run: | run: |
brew update brew update
@ -182,8 +157,12 @@ jobs:
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
run: | run: |
kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} kerl update releases
kerl build ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build - name: build
working-directory: source working-directory: source
@ -191,11 +170,12 @@ jobs:
. $HOME/.kerl/${{ matrix.otp }}/activate . $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-zip
- name: test - name: test
working-directory: source working-directory: source
run: | run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip | head) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q $pkg_name unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
@ -230,7 +210,15 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ## all editions for linux
- emqx-edge
- emqx
- emqx-ee
otp:
- 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions
arch:
- amd64
- arm64
os: os:
- ubuntu20.04 - ubuntu20.04
- ubuntu18.04 - ubuntu18.04
@ -240,18 +228,9 @@ jobs:
# - opensuse # - opensuse
- centos8 - centos8
- centos7 - centos7
- centos6
- raspbian10 - raspbian10
# - raspbian9 # - raspbian9
arch:
- amd64
- arm64
otp:
- 23.2.7.2-emqx-2
- 24.1.1-emqx-1
exclude: exclude:
- os: centos6
arch: arm64
- os: raspbian9 - os: raspbian9
arch: amd64 arch: amd64
- os: raspbian10 - os: raspbian10
@ -277,12 +256,13 @@ jobs:
platforms: all platforms: all
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: downloads old emqx zip packages - name: downloads old emqx zip packages
env: env:
OTP_VSN: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
@ -302,10 +282,11 @@ jobs:
cd _upgrade_base cd _upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do for tag in ${old_vsns[@]}; do
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
fi fi
done done
- name: build emqx packages - name: build emqx packages
@ -320,7 +301,7 @@ jobs:
-v $(pwd):/emqx \ -v $(pwd):/emqx \
--workdir /emqx \ --workdir /emqx \
--platform linux/$ARCH \ --platform linux/$ARCH \
ghcr.io/emqx/emqx-builder/5.0:$OTP-$SYSTEM \ ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
@ -349,17 +330,21 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: # all editions for docker
- emqx-edge
- emqx
- emqx-ee
# NOTE: for docker, only support latest otp version, not a matrix
otp: otp:
- 24.1.1-emqx-1 - 24.1.5-2 # update to latest
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- uses: docker/setup-buildx-action@v1 - uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1 - uses: docker/setup-qemu-action@v1
with: with:
@ -376,7 +361,8 @@ jobs:
type=ref,event=pr type=ref,event=pr
type=ref,event=tag type=ref,event=tag
type=semver,pattern={{version}} type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}} labels:
org.opencontainers.image.otp.version=${{ matrix.otp }}
- uses: docker/login-action@v1 - uses: docker/login-action@v1
if: github.event_name == 'release' if: github.event_name == 'release'
with: with:
@ -384,32 +370,26 @@ jobs:
password: ${{ secrets.DOCKER_HUB_TOKEN }} password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/build-push-action@v2 - uses: docker/build-push-action@v2
with: with:
push: ${{ github.event_name == 'release' }} push: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
pull: true pull: true
no-cache: true no-cache: true
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}
build-args: | build-args: |
BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-alpine3.14 BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14 RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }} EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile file: source/deploy/docker/Dockerfile
context: source context: source
delete-artifact: delete-artifact:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy:
matrix:
otp:
- 23.2.7.2-emqx-2
- 24.1.1-emqx-1
needs: [prepare, mac, linux, docker] needs: [prepare, mac, linux, docker]
steps: steps:
- uses: geekyeggo/delete-artifact@v1 - uses: geekyeggo/delete-artifact@v1
with: with:
name: source-${{ matrix.otp }} name: source
upload: upload:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -420,9 +400,12 @@ jobs:
strategy: strategy:
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile:
- emqx-edge
- emqx
- emqx-ee
otp: otp:
- 24.1.1-emqx-1 - 24.1.5-2
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -461,17 +444,11 @@ jobs:
aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
- uses: Rory-Z/upload-release-asset@v1 - uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile != 'emqx-ee' if: github.event_name == 'release'
with: with:
repo: emqx repo: emqx
path: "_packages/${{ matrix.profile }}/emqx-*" path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }} token: ${{ github.token }}
- uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
with:
repo: emqx-enterprise
path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }}
- name: update to emqx.io - name: update to emqx.io
if: github.event_name == 'release' if: github.event_name == 'release'
run: | run: |
@ -484,32 +461,28 @@ jobs:
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }} ${{ secrets.EMQX_IO_RELEASE_API }}
- name: update repo.emqx.io - name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' if: github.event_name == 'release'
run: | run: |
if [ "${{ matrix. profile }}" = 'emqx-ee' ]; then
BOOL_FLAG_NAME="emqx_ee"
else
BOOL_FLAG_NAME="emqx_ce"
fi
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \ -H "Accept: application/vnd.github.v3+json" \
-X POST \ -X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"${BOOL_FLAG_NAME}\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
run: |
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update homebrew packages - name: update homebrew packages
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' if: github.event_name == 'release' && matrix.profile == 'emqx'
run: | run: |
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \ -H "Accept: application/vnd.github.v3+json" \
-X POST \ -X POST \
-d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
fi fi
- uses: geekyeggo/delete-artifact@v1 - uses: geekyeggo/delete-artifact@v1

75
.github/workflows/build_slim_packages.yaml vendored
View File

@ -4,7 +4,6 @@ concurrency:
group: slim-${{ github.event_name }}-${{ github.ref }} group: slim-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true cancel-in-progress: true
on: on:
push: push:
tags: tags:
@ -14,48 +13,37 @@ on:
workflow_dispatch: workflow_dispatch:
jobs: jobs:
build: linux:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile:
- emqx-edge
- emqx
- emqx-ee
otp: otp:
- 24.1.1-emqx-1 - 24.1.5-2
os: os:
- ubuntu20.04 - ubuntu20.04
- centos7 - centos7
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-${{ matrix.os }}" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-${{ matrix.os }}"
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v1
- name: prepare - name: build zip package
run: | run: make ${{ matrix.profile }}-zip
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
else
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
fi
- name: build zip packages
run: make ${EMQX_NAME}-zip
- name: build deb/rpm packages - name: build deb/rpm packages
run: make ${EMQX_NAME}-pkg run: make ${{ matrix.profile }}-pkg
- uses: actions/upload-artifact@v1
if: failure()
with:
name: rebar3.crashdump
path: ./rebar3.crashdump
- name: packages test - name: packages test
run: | run: |
export CODE_PATH=$GITHUB_WORKSPACE export CODE_PATH=$GITHUB_WORKSPACE
.ci/build_packages/tests.sh EMQX_NAME=${{ matrix.profile }} .ci/build_packages/tests.sh
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: ${{ matrix.os }} name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
path: _packages/**/*.zip path: _packages/**/*.zip
mac: mac:
@ -63,32 +51,25 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
profile:
- emqx
- emqx-ee
otp:
- 24.1.5-2
macos: macos:
- macos-11 - macos-11
- macos-10.15 - macos-10.15
otp:
- 24.1.1-emqx-1
runs-on: ${{ matrix.macos }} runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v2
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
else
echo "EMQX_NAME=emqx" >> $GITHUB_ENV
fi
- name: prepare - name: prepare
run: | run: |
brew update brew update
brew install curl zip unzip gnu-sed kerl unixodbc freetds brew install curl zip unzip gnu-sed kerl unixodbc freetds
echo "/usr/local/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
- uses: actions/cache@v2 - uses: actions/cache@v2
id: cache id: cache
with: with:
@ -97,23 +78,23 @@ jobs:
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
run: | run: |
kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} kerl update releases
kerl build ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build - name: build ${{ matrix.profile }}
run: | run: |
. $HOME/.kerl/${{ matrix.otp }}/activate . $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
make ${EMQX_NAME}-zip make ${{ matrix.profile }}-zip
- uses: actions/upload-artifact@v1
if: failure()
with:
name: rebar3.crashdump
path: ./rebar3.crashdump
- name: test - name: test
run: | run: |
unzip -q $(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip | head) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs: jobs:
check_deps_integrity: check_deps_integrity:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

12
.github/workflows/code_style_check.yaml vendored
View File

@ -9,19 +9,9 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
fetch-depth: 1000 fetch-depth: 1000
- name: Set git token
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: Run elvis check - name: Run elvis check
run: | run: |
set -e ./scripts/elvis-check.sh $GITHUB_BASE_REF
if [ -f EMQX_ENTERPRISE ]; then
./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise
else
./scripts/elvis-check.sh $GITHUB_BASE_REF emqx
fi
- name: Check line-break at EOF - name: Check line-break at EOF
run: | run: |
./scripts/check-nl-at-eof.sh ./scripts/check-nl-at-eof.sh

6
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -12,11 +12,11 @@ jobs:
strategy: strategy:
matrix: matrix:
otp: otp:
- "23.2.7.2-emqx-2" - 23.3.4.9-3
- "24.1.1-emqx-1" - 24.1.5-2
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

84
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -8,58 +8,27 @@ on:
push: push:
tags: tags:
- v* - v*
- e*
pull_request: pull_request:
jobs: jobs:
prepare: prepare:
strategy:
matrix:
otp:
- "23.2.7.2-emqx-2"
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" # prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
outputs:
profile: ${{ steps.profile.outputs.profile }}
steps: steps:
- name: get otp version
id: get_otp_version
run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
echo "::set-output name=otp::$otp"
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
path: source path: source
fetch-depth: 0 fetch-depth: 0
- name: set profile
id: profile
shell: bash
working-directory: source
run: |
vsn="$(./pkg-vsn.sh)"
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "::set-output name=profile::emqx-ee"
else
echo "::set-output name=profile::emqx"
fi
- name: get deps - name: get deps
working-directory: source
run: | run: |
make ensure-rebar3 make -C source deps-all
./rebar3 as default get-deps zip -ryq source.zip source/* source/.[^.]*
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: source-${{ steps.get_otp_version.outputs.otp }} name: source
path: source-${{ steps.get_otp_version.outputs.otp }}.zip path: source.zip
docker_test: docker_test:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -68,35 +37,34 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
otp: profile:
- 23.2.7.2-emqx-2 - emqx-edge
- 24.1.1-emqx-1 - emqx
- emqx-ee
cluster_db_backend: cluster_db_backend:
- "mnesia" - mnesia
- "rlog" - rlog
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
OTP: ${{ matrix.otp }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
run: | run: |
make ${{ needs.prepare.outputs.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- name: run emqx - name: run emqx
timeout-minutes: 5 timeout-minutes: 5
working-directory: source working-directory: source
run: | run: |
set -x set -x
export EMQX_TEST_DB_BACKEND="${{ matrix.cluster_db_backend }}" IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh)
./.ci/docker-compose-file/scripts/run-emqx.sh ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
- name: make paho tests - name: make paho tests
run: | run: |
if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
@ -118,24 +86,24 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
otp: profile:
- 23.2.7.2-emqx-2 - emqx
- 24.1.1-emqx-1 # - emqx-ee # TODO test enterprise
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source-${{ matrix.otp }} name: source
path: . path: .
- name: unzip source code - name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip run: unzip -q source.zip
- name: make docker image - name: make docker image
working-directory: source working-directory: source
env: env:
OTP: ${{ matrix.otp }} EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
run: | run: |
make ${{ needs.prepare.outputs.profile }}-docker make ${{ matrix.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- run: minikube start - run: minikube start
- name: run emqx on chart - name: run emqx on chart

32
.github/workflows/run_relup_tests.yaml vendored
View File

@ -15,12 +15,14 @@ jobs:
relup_test: relup_test:
strategy: strategy:
matrix: matrix:
otp: profile:
- "23.2.7.2-emqx-2" - emqx
- "24.1.1-emqx-1" - emqx-ee
otp_vsn:
- 24.1.5-2
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04"
defaults: defaults:
run: run:
@ -55,26 +57,18 @@ jobs:
repository: ${{ github.repository }} repository: ${{ github.repository }}
path: emqx path: emqx
fetch-depth: 0 fetch-depth: 0
- name: prepare
run: |
if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: get version - name: get version
run: | run: |
set -e -x -u set -e -x -u
cd emqx cd emqx
export PROFILE=${{ matrix.profile }}
export OTP_VSN=${{ matrix.otp_vsn }}
echo "PROFILE=$PROFILE" >> $GITHUB_ENV
echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV
if [ $PROFILE = "emqx" ];then if [ $PROFILE = "emqx" ];then
broker="emqx-ce" broker="emqx-ce"
edition='opensource'
else else
broker="emqx-ee" broker="emqx-ee"
edition='enterprise'
fi fi
echo "BROKER=$broker" >> $GITHUB_ENV echo "BROKER=$broker" >> $GITHUB_ENV
@ -82,11 +76,7 @@ jobs:
echo "VSN=$vsn" >> $GITHUB_ENV echo "VSN=$vsn" >> $GITHUB_ENV
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if [ $PROFILE = "emqx" ]; then
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
else
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
fi
echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
- name: download emqx - name: download emqx
run: | run: |
@ -95,7 +85,7 @@ jobs:
cd emqx/_upgrade_base cd emqx/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for old_vsn in ${old_vsns[@]}; do for old_vsn in ${old_vsns[@]}; do
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip
done done
- name: build emqx - name: build emqx
run: make -C emqx ${PROFILE}-zip run: make -C emqx ${PROFILE}-zip

33
.github/workflows/run_test_cases.yaml vendored
View File

@ -15,12 +15,11 @@ jobs:
run_static_analysis: run_static_analysis:
strategy: strategy:
matrix: matrix:
otp: emqx_builder:
- "23.2.7.2-emqx-2" - 5.0-2:24.1.5-2 # run dialyzer on latest OTP
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -38,12 +37,11 @@ jobs:
run_proper_test: run_proper_test:
strategy: strategy:
matrix: matrix:
otp: emqx_builder:
- "23.2.7.2-emqx-2" - 5.0-2:24.1.5-2
- "24.1.1-emqx-1"
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04"
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -67,32 +65,13 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: set edition
id: set_edition
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "EDITION=enterprise" >> $GITHUB_ENV
else
echo "EDITION=opensource" >> $GITHUB_ENV
fi
- name: docker compose up - name: docker compose up
if: env.EDITION == 'opensource'
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | run: |
docker-compose \ docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \
up -d --build up -d --build
- name: docker compose up
if: env.EDITION == 'enterprise'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
timeout-minutes: 20
run: |
docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise.yaml \
up -d --build
- name: run eunit - name: run eunit
run: | run: |
docker exec -i ${{ matrix.otp_release }} bash -c "make eunit" docker exec -i ${{ matrix.otp_release }} bash -c "make eunit"

27
Makefile
View File

@ -3,16 +3,20 @@ REBAR_VERSION = 3.16.1-emqx-1
REBAR = $(CURDIR)/rebar3 REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts SCRIPTS = $(CURDIR)/scripts
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14
export EMQX_DEFAULT_RUNNER = alpine:3.14
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
export EMQX_DESC ?= EMQ X
export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.18 export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.18
export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none export REBAR_COLOR=none
endif endif
PROFILE ?= emqx PROFILE ?= emqx
REL_PROFILES := emqx emqx-edge REL_PROFILES := emqx emqx-edge emqx-ee
PKG_PROFILES := emqx-pkg emqx-edge-pkg PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-ee-pkg
PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
CT_NODE_NAME ?= 'test@127.0.0.1' CT_NODE_NAME ?= 'test@127.0.0.1'
@ -85,7 +89,6 @@ coveralls: $(REBAR)
@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
.PHONY: $(REL_PROFILES) .PHONY: $(REL_PROFILES)
$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
@$(REBAR) as $(@) do compile,release @$(REBAR) as $(@) do compile,release
@ -98,8 +101,10 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
clean: $(PROFILES:%=clean-%) clean: $(PROFILES:%=clean-%)
$(PROFILES:%=clean-%): $(PROFILES:%=clean-%):
@if [ -d _build/$(@:clean-%=%) ]; then \ @if [ -d _build/$(@:clean-%=%) ]; then \
rm rebar.lock \
rm -rf _build/$(@:clean-%=%)/rel; \ rm -rf _build/$(@:clean-%=%)/rel; \
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
find _build/$(@:clean-%=%) -type l -delete; \
fi fi
.PHONY: clean-all .PHONY: clean-all
@ -109,6 +114,7 @@ clean-all:
.PHONY: deps-all .PHONY: deps-all
deps-all: $(REBAR) $(PROFILES:%=deps-%) deps-all: $(REBAR) $(PROFILES:%=deps-%)
@make clean # ensure clean at the end
## deps-<profile> is used in CI scripts to download deps and the ## deps-<profile> is used in CI scripts to download deps and the
## share downloads between CI steps and/or copied into containers ## share downloads between CI steps and/or copied into containers
@ -116,6 +122,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
.PHONY: $(PROFILES:%=deps-%) .PHONY: $(PROFILES:%=deps-%)
$(PROFILES:%=deps-%): $(REBAR) get-dashboard $(PROFILES:%=deps-%): $(REBAR) get-dashboard
@$(REBAR) as $(@:deps-%=%) get-deps @$(REBAR) as $(@:deps-%=%) get-deps
@rm -f rebar.lock
.PHONY: xref .PHONY: xref
xref: $(REBAR) xref: $(REBAR)
@ -174,5 +181,17 @@ endef
ALL_ZIPS = $(REL_PROFILES) ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing
## emqx-ee-docker-testing
## is to directly copy a unzipped zip-package to a
## base image such as ubuntu20.04. Mostly for testing
.PHONY: $(REL_PROFILES:%=%-docker-testing)
define gen-docker-target-testing
$1-docker-testing: $(COMMON_DEPS)
@$(BUILD) $1 docker-testing
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs: conf-segs:
@scripts/merge-config.escript @scripts/merge-config.escript

14
apps/emqx/include/emqx_release.hrl
View File

@ -14,9 +14,6 @@
%% limitations under the License. %% limitations under the License.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-ifndef(EMQX_RELEASE_HRL).
-define(EMQX_RELEASE_HRL, true).
%% NOTE: this is the release version which is not always the same %% NOTE: this is the release version which is not always the same
%% as the emqx app version defined in emqx.app.src %% as the emqx app version defined in emqx.app.src
%% App (plugin) versions are bumped independently. %% App (plugin) versions are bumped independently.
@ -27,13 +24,4 @@
%% NOTE: This version number should be manually bumped for each release %% NOTE: This version number should be manually bumped for each release
-ifndef(EMQX_ENTERPRISE). -define(EMQX_RELEASE, "5.0-beta.2").
-define(EMQX_RELEASE, {opensource, "5.0-beta.1"}).
-else.
-endif.
-endif.

28
apps/emqx/src/emqx.erl
View File

@ -20,6 +20,7 @@
-include("logger.hrl"). -include("logger.hrl").
-include("types.hrl"). -include("types.hrl").
-elvis([{elvis_style, god_modules, disable}]).
%% Start/Stop the application %% Start/Stop the application
-export([ start/0 -export([ start/0
@ -51,10 +52,6 @@
, run_fold_hook/3 , run_fold_hook/3
]). ]).
%% Troubleshooting
-export([ set_debug_secret/1
]).
%% Configs APIs %% Configs APIs
-export([ get_config/1 -export([ get_config/1
, get_config/2 , get_config/2
@ -71,29 +68,6 @@
-define(APP, ?MODULE). -define(APP, ?MODULE).
%% @hidden Path to the file which has debug_info encryption secret in it.
%% Evaluate this function if there is a need to access encrypted debug_info.
%% NOTE: Do not change the API to accept the secret text because it may
%% get logged everywhere.
set_debug_secret(PathToSecretFile) ->
SecretText =
case file:read_file(PathToSecretFile) of
{ok, Secret} ->
try string:trim(binary_to_list(Secret))
catch _ : _ -> error({badfile, PathToSecretFile})
end;
{error, Reason} ->
?ULOG("Failed to read debug_info encryption key file ~ts: ~p~n",
[PathToSecretFile, Reason]),
error(Reason)
end,
F = fun(init) -> ok;
(clear) -> ok;
({debug_info, _Mode, _Module, _Filename}) -> SecretText
end,
_ = beam_lib:clear_crypto_key_fun(),
ok = beam_lib:crypto_key_fun(F).
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Bootstrap, is_running... %% Bootstrap, is_running...
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------

29
apps/emqx/src/emqx_app.erl
View File

@ -30,7 +30,6 @@
]). ]).
-include("emqx.hrl"). -include("emqx.hrl").
-include("emqx_release.hrl").
-include("logger.hrl"). -include("logger.hrl").
-define(APP, emqx). -define(APP, emqx).
@ -40,6 +39,7 @@
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
start(_Type, _Args) -> start(_Type, _Args) ->
ok = emqx_release:put_edition(),
ok = maybe_load_config(), ok = maybe_load_config(),
ok = emqx_persistent_session:init_db_backend(), ok = emqx_persistent_session:init_db_backend(),
ok = maybe_start_quicer(), ok = maybe_start_quicer(),
@ -107,30 +107,7 @@ is_quicer_app_present() ->
is_quic_listener_configured() -> is_quic_listener_configured() ->
emqx_listeners:has_enabled_listener_conf_by_type(quic). emqx_listeners:has_enabled_listener_conf_by_type(quic).
get_description() -> get_description() -> emqx_release:description().
{ok, Descr0} = application:get_key(?APP, description),
case os:getenv("EMQX_DESCRIPTION") of
false -> Descr0;
"" -> Descr0;
Str -> string:strip(Str, both, $\n)
end.
get_release() -> get_release() ->
case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of emqx_release:version().
false -> %% For TEST build or depedency build.
release_in_macro();
{_, Vsn} -> %% For emqx release build
VsnStr = release_in_macro(),
case string:str(Vsn, VsnStr) of
1 -> ok;
_ ->
erlang:error(#{ reason => version_mismatch
, source => VsnStr
, built_for => Vsn
})
end,
Vsn
end.
release_in_macro() ->
element(2, ?EMQX_RELEASE).

25
apps/emqx/src/emqx_misc.erl
View File

@ -17,6 +17,7 @@
-module(emqx_misc). -module(emqx_misc).
-compile(inline). -compile(inline).
-elvis([{elvis_style, god_modules, disable}]).
-include("types.hrl"). -include("types.hrl").
-include("logger.hrl"). -include("logger.hrl").
@ -65,21 +66,13 @@ maybe_parse_ip(Host) ->
end. end.
%% @doc Add `ipv6_probe' socket option if it's supported. %% @doc Add `ipv6_probe' socket option if it's supported.
%% gen_tcp:ipv6_probe() -> true. is added to EMQ's OTP forks
ipv6_probe(Opts) -> ipv6_probe(Opts) ->
case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of case erlang:function_exported(gen_tcp, ipv6_probe, 0) of
unknown -> true -> [{ipv6_probe, true} | Opts];
%% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 false -> Opts
OtpVsn = emqx_vm:get_otp_version(),
Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])),
_ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool),
ipv6_probe(Bool, Opts);
Bool ->
ipv6_probe(Bool, Opts)
end. end.
ipv6_probe(false, Opts) -> Opts;
ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts].
%% @doc Merge options %% @doc Merge options
-spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()).
merge_opts(Defaults, Options) -> merge_opts(Defaults, Options) ->
@ -339,6 +332,12 @@ pad(L, Count) ->
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
ipv6_probe_test() -> ipv6_probe_test() ->
?assertEqual([{ipv6_probe, true}], ipv6_probe([])). try gen_tcp:ipv6_probe() of
true ->
?assertEqual([{ipv6_probe, true}], ipv6_probe([]))
catch
_ : _ ->
ok
end.
-endif. -endif.

86
apps/emqx/src/emqx_release.erl Normal file
View File

@ -0,0 +1,86 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_release).
-export([ edition/0
, put_edition/0
, put_edition/1
, description/0
, version/0
]).
-include("emqx_release.hrl").
%% @doc Return EMQ X description.
description() ->
case os:getenv("EMQX_DESCRIPTION") of
false -> "EMQ X Community Edition";
"" -> "EMQ X Community Edition";
Str -> string:strip(Str, both, $\n)
end.
%% @doc Return EMQ X edition info.
%% Read info from persistent_term at runtime.
%% Or meck this function to run tests for another eidtion.
-spec edition() -> ce | ee | edge.
edition() ->
try persistent_term:get(emqx_edition)
catch error : badarg -> get_edition() end.
%% @private initiate EMQ X edition info in persistent_term.
put_edition() ->
ok = put_edition(get_edition()).
%% @hidden This function is mostly for testing.
%% Switch to another eidtion at runtime to run edition-specific tests.
-spec put_edition(ce | ee | edge) -> ok.
put_edition(Which) ->
persistent_term:put(emqx_edition, Which),
ok.
-spec get_edition() -> ce | ee | edge.
get_edition() ->
edition(description()).
edition(Desc) ->
case re:run(Desc, "enterprise", [caseless]) of
{match, _} ->
ee;
_ ->
case re:run(Desc, "edge", [caseless]) of
{match, _} -> edge;
_ -> ce
end
end.
%% @doc Return the release version.
version() ->
case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of
false -> %% For TEST build or depedency build.
?EMQX_RELEASE;
{_, Vsn} -> %% For emqx release build
VsnStr = ?EMQX_RELEASE,
case string:str(Vsn, VsnStr) of
1 -> ok;
_ ->
erlang:error(#{ reason => version_mismatch
, source => VsnStr
, built_for => Vsn
})
end,
Vsn
end.

2
apps/emqx/test/emqx_authentication_SUITE.erl
View File

@ -270,7 +270,7 @@ t_restart(Config) when is_list(Config) ->
?assertEqual({ok, [test_chain]}, ?AUTHN:list_chain_names()); ?assertEqual({ok, [test_chain]}, ?AUTHN:list_chain_names());
t_restart({'end', Config}) -> t_restart({'end', _Config}) ->
?AUTHN:delete_chain(test_chain), ?AUTHN:delete_chain(test_chain),
ok. ok.

8
apps/emqx_dashboard/src/emqx_dashboard_admin.erl
View File

@ -171,7 +171,7 @@ check(_, undefined) ->
check(Username, Password) -> check(Username, Password) ->
case lookup_user(Username) of case lookup_user(Username) of
[#?ADMIN{pwdhash = <<Salt:4/binary, Hash/binary>>}] -> [#?ADMIN{pwdhash = <<Salt:4/binary, Hash/binary>>}] ->
case Hash =:= sha3_hash(Salt, Password) of case Hash =:= sha256(Salt, Password) of
true -> ok; true -> ok;
false -> {error, <<"BAD_USERNAME_OR_PASSWORD">>} false -> {error, <<"BAD_USERNAME_OR_PASSWORD">>}
end; end;
@ -206,10 +206,10 @@ destroy_token_by_username(Username, Token) ->
hash(Password) -> hash(Password) ->
SaltBin = emqx_dashboard_token:salt(), SaltBin = emqx_dashboard_token:salt(),
<<SaltBin/binary, (sha3_hash(SaltBin, Password))/binary>>. <<SaltBin/binary, (sha256(SaltBin, Password))/binary>>.
sha3_hash(SaltBin, Password) -> sha256(SaltBin, Password) ->
crypto:hash('sha3_256', <<SaltBin/binary, Password/binary>>). crypto:hash('sha256', <<SaltBin/binary, Password/binary>>).
add_default_user() -> add_default_user() ->
add_default_user(binenv(default_username), binenv(default_password)). add_default_user(binenv(default_username), binenv(default_password)).

15
apps/emqx_dashboard/src/emqx_dashboard_api.erl
View File

@ -16,16 +16,6 @@
-module(emqx_dashboard_api). -module(emqx_dashboard_api).
-ifndef(EMQX_ENTERPRISE).
-define(RELEASE, community).
-else.
-define(VERSION, enterprise).
-endif.
-behaviour(minirest_api). -behaviour(minirest_api).
-include("emqx_dashboard.hrl"). -include("emqx_dashboard.hrl").
@ -200,7 +190,10 @@ login(post, #{body := Params}) ->
case emqx_dashboard_admin:sign_token(Username, Password) of case emqx_dashboard_admin:sign_token(Username, Password) of
{ok, Token} -> {ok, Token} ->
Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())), Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())),
{200, #{token => Token, version => Version, license => #{edition => ?RELEASE}}}; {200, #{token => Token,
version => Version,
license => #{edition => emqx_release:edition()}
}};
{error, _} -> {error, _} ->
{401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}} {401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}}
end. end.

4
apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
View File

@ -150,11 +150,11 @@ t_cli(_Config) ->
emqx_dashboard_cli:admins(["add", "username", "password"]), emqx_dashboard_cli:admins(["add", "username", "password"]),
[#?ADMIN{ username = <<"username">>, pwdhash = <<Salt:4/binary, Hash/binary>>}] = [#?ADMIN{ username = <<"username">>, pwdhash = <<Salt:4/binary, Hash/binary>>}] =
emqx_dashboard_admin:lookup_user(<<"username">>), emqx_dashboard_admin:lookup_user(<<"username">>),
?assertEqual(Hash, crypto:hash(sha3_256, <<Salt/binary, <<"password">>/binary>>)), ?assertEqual(Hash, crypto:hash(sha256, <<Salt/binary, <<"password">>/binary>>)),
emqx_dashboard_cli:admins(["passwd", "username", "newpassword"]), emqx_dashboard_cli:admins(["passwd", "username", "newpassword"]),
[#?ADMIN{username = <<"username">>, pwdhash = <<Salt1:4/binary, Hash1/binary>>}] = [#?ADMIN{username = <<"username">>, pwdhash = <<Salt1:4/binary, Hash1/binary>>}] =
emqx_dashboard_admin:lookup_user(<<"username">>), emqx_dashboard_admin:lookup_user(<<"username">>),
?assertEqual(Hash1, crypto:hash(sha3_256, <<Salt1/binary, <<"newpassword">>/binary>>)), ?assertEqual(Hash1, crypto:hash(sha256, <<Salt1/binary, <<"newpassword">>/binary>>)),
emqx_dashboard_cli:admins(["del", "username"]), emqx_dashboard_cli:admins(["del", "username"]),
[] = emqx_dashboard_admin:lookup_user(<<"username">>), [] = emqx_dashboard_admin:lookup_user(<<"username">>),
emqx_dashboard_cli:admins(["add", "admin1", "pass1"]), emqx_dashboard_cli:admins(["add", "admin1", "pass1"]),

30
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -884,29 +884,27 @@ time_unit(<<"nanosecond">>) -> nanosecond.
%% Here the emqx_rule_funcs module acts as a proxy, forwarding %% Here the emqx_rule_funcs module acts as a proxy, forwarding
%% the function handling to the worker module. %% the function handling to the worker module.
%% @end %% @end
-ifdef(EMQX_ENTERPRISE). % '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) ->
'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> % emqx_schema_parser:decode(SchemaId, Data, MoreArgs);
emqx_schema_parser:decode(SchemaId, Data, MoreArgs); % '$handle_undefined_function'(schema_decode, Args) ->
'$handle_undefined_function'(schema_decode, Args) -> % error({args_count_error, {schema_decode, Args}});
error({args_count_error, {schema_decode, Args}});
'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> % '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) ->
emqx_schema_parser:encode(SchemaId, Term, MoreArgs); % emqx_schema_parser:encode(SchemaId, Term, MoreArgs);
'$handle_undefined_function'(schema_encode, Args) -> % '$handle_undefined_function'(schema_encode, Args) ->
error({args_count_error, {schema_encode, Args}}); % error({args_count_error, {schema_encode, Args}});
% '$handle_undefined_function'(sprintf, [Format|Args]) ->
% erlang:apply(fun sprintf_s/2, [Format, Args]);
% '$handle_undefined_function'(Fun, Args) ->
% error({sql_function_not_supported, function_literal(Fun, Args)}).
'$handle_undefined_function'(sprintf, [Format|Args]) -> '$handle_undefined_function'(sprintf, [Format|Args]) ->
erlang:apply(fun sprintf_s/2, [Format, Args]); erlang:apply(fun sprintf_s/2, [Format, Args]);
'$handle_undefined_function'(Fun, Args) -> '$handle_undefined_function'(Fun, Args) ->
error({sql_function_not_supported, function_literal(Fun, Args)}). error({sql_function_not_supported, function_literal(Fun, Args)}).
-else.
'$handle_undefined_function'(sprintf, [Format|Args]) ->
erlang:apply(fun sprintf_s/2, [Format, Args]);
'$handle_undefined_function'(Fun, Args) ->
error({sql_function_not_supported, function_literal(Fun, Args)}).
-endif. % EMQX_ENTERPRISE
map_path(Key) -> map_path(Key) ->
{path, [{key, P} || P <- string:split(Key, ".", all)]}. {path, [{key, P} || P <- string:split(Key, ".", all)]}.

90
build
View File

@ -12,23 +12,10 @@ ARTIFACT="$2"
# ensure dir # ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
PKG_VSN="$(./pkg-vsn.sh)" PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
export PKG_VSN export PKG_VSN
if [ "$(uname -s)" = 'Darwin' ]; then SYSTEM="$(./scripts/get-distro.sh)"
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'centos' /etc/*-release; then
DIST='centos'
VERSION_ID="$(rpm --eval '%{centos_ver}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
fi
ARCH="$(uname -m)" ARCH="$(uname -m)"
case "$ARCH" in case "$ARCH" in
@ -48,8 +35,8 @@ export ARCH
## Support RPM and Debian based linux systems ## Support RPM and Debian based linux systems
## ##
if [ "$(uname -s)" = 'Linux' ]; then if [ "$(uname -s)" = 'Linux' ]; then
case "${DIST:-}" in case "${SYSTEM:-}" in
ubuntu|debian|raspbian) ubuntu*|debian*|raspbian*)
PKGERDIR='deb' PKGERDIR='deb'
;; ;;
*) *)
@ -109,7 +96,7 @@ make_relup() {
rm -rf "$tmp_dir" rm -rf "$tmp_dir"
fi fi
releases+=( "$base_vsn" ) releases+=( "$base_vsn" )
done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
fi fi
if [ ${#releases[@]} -eq 0 ]; then if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored" log "No upgrade base found, relup ignored"
@ -149,7 +136,7 @@ make_zip() {
log "ERROR: $tarball is not found" log "ERROR: $tarball is not found"
fi fi
local zipball local zipball
zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
tar zxf "${tarball}" -C "${tard}/emqx" tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages. ## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt ## for DEB and RPM packages the dependencies are resoved by yum and apt
@ -157,18 +144,60 @@ make_zip() {
(cd "${tard}" && zip -qr - emqx) > "${zipball}" (cd "${tard}" && zip -qr - emqx) > "${zipball}"
} }
## This function builds the default docker image based on alpine:3.14 (by default)
make_docker() { make_docker() {
## Build Docker image EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
echo "DOCKER BUILD: Build Docker image." EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
echo "DOCKER BUILD: build version -> $PKG_VSN." set -x
echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " docker build --no-cache --pull \
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
docker build --no-cache \ --build-arg RUN_FROM="${EMQX_RUNNER}" \
--build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/5.0:${OTP:-24.1.1-emqx-1}-alpine3.14" \
--build-arg RUN_FROM="alpine:3.14" \
--build-arg EMQX_NAME="$PROFILE" \ --build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:$PKG_VSN" \ --tag "emqx/$PROFILE:${PKG_VSN}" \
-f deploy/docker/Dockerfile . -f "${DOCKERFILE}" .
}
## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X
##
## Export below variables to quickly build an image
##
## Name Default Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
##
make_docker_testing() {
if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
case "$SYSTEM" in
ubuntu20*)
EMQX_BASE_IMAGE="ubuntu:20.04"
;;
centos8)
EMQX_BASE_IMAGE="centos:8"
;;
*)
echo "Unsupported testing base image for $SYSTEM"
exit 1
;;
esac
fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip
defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
if [ ! -f "$zip" ]; then
log "ERROR: $zip not built?"
exit 1
fi
set -x
docker build \
--build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
--build-arg EMQX_ZIP_PACKAGE="${zip}" \
--tag "$EMQX_IMAGE_TAG" \
-f "${DOCKERFILE_TESTING}" .
} }
log "building artifact=$ARTIFACT for profile=$PROFILE" log "building artifact=$ARTIFACT for profile=$PROFILE"
@ -194,6 +223,9 @@ case "$ARTIFACT" in
docker) docker)
make_docker make_docker
;; ;;
docker-testing)
make_docker_testing
;;
*) *)
log "Unknown artifact $ARTIFACT" log "Unknown artifact $ARTIFACT"
exit 1 exit 1

2
deploy/docker/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-alpine3.14 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14
ARG RUN_FROM=alpine:3.14 ARG RUN_FROM=alpine:3.14
FROM ${BUILD_FROM} AS builder FROM ${BUILD_FROM} AS builder

2
deploy/packages/deb/Makefile
View File

@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture)
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
.PHONY: all .PHONY: all
all: | $(BUILT) all: | $(BUILT)

2
deploy/packages/deb/debian/control
View File

@ -4,7 +4,7 @@ Priority: optional
Maintainer: emqx <contact@emqx.io> Maintainer: emqx <contact@emqx.io>
Build-Depends: debhelper (>=9) Build-Depends: debhelper (>=9)
Standards-Version: 3.9.6 Standards-Version: 3.9.6
Homepage: https://www.emqx.io Homepage: https://www.emqx.com
Package: emqx Package: emqx
Architecture: any Architecture: any

14
deploy/packages/rpm/Makefile
View File

@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT
dash := - dash := -
none := none :=
space := $(none) $(none) space := $(none) $(none)
RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") ## RPM does not allow '-' in version nubmer and release string, replace with '_'
RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") RPM_VSN := $(subst -,_,$(PKG_VSN))
RPM_REL := otp$(subst -,_,$(OTP_VSN))
ARCH ?= amd64 ARCH ?= amd64
ifeq ($(ARCH),mips64) ifeq ($(ARCH),mips64)
@ -16,12 +17,8 @@ endif
EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz
TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH)
ifeq ($(RPM_REL),) SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
# no tail
RPM_REL := 1
endif
SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi)
# Not $(PWD) as it does not work for make -C # Not $(PWD) as it does not work for make -C
@ -47,7 +44,6 @@ all: | $(BUILT)
--define "_service_dst $(SERVICE_DST)" \ --define "_service_dst $(SERVICE_DST)" \
--define "_post_addition $(POST_ADDITION)" \ --define "_post_addition $(POST_ADDITION)" \
--define "_preun_addition $(PREUN_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \
--define "_ostype -$(SYSTEM)" \
--define "_sharedstatedir /var/lib" \ --define "_sharedstatedir /var/lib" \
emqx.spec emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)

2
deploy/packages/rpm/emqx.spec
View File

@ -5,7 +5,7 @@
%define _log_dir %{_var}/log/%{_name} %define _log_dir %{_var}/log/%{_name}
%define _lib_home /usr/lib/%{_name} %define _lib_home /usr/lib/%{_name}
%define _var_home %{_sharedstatedir}/%{_name} %define _var_home %{_sharedstatedir}/%{_name}
%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm %define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm
%define _build_id_links none %define _build_id_links none
Name: %{_package_name} Name: %{_package_name}

0
lib-ee/.gitkeep Normal file
View File

8
pkg-vsn.sh
View File

@ -6,14 +6,8 @@ set -euo pipefail
# ensure dir # ensure dir
cd -P -- "$(dirname -- "$0")" cd -P -- "$(dirname -- "$0")"
if [ -f EMQX_ENTERPRISE ]; then
EDITION='enterprise'
else
EDITION='opensource'
fi
## emqx_release.hrl is the single source of truth for release version ## emqx_release.hrl is the single source of truth for release version
RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" RELEASE="$(grep -E "define.+EMQX_RELEASE" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)"
git_exact_vsn() { git_exact_vsn() {
local tag local tag

79
rebar.config.erl
View File

@ -130,18 +130,17 @@ test_deps() ->
, {proper, "1.4.0"} , {proper, "1.4.0"}
]. ].
common_compile_opts(Edition) -> common_compile_opts() ->
[ debug_info % alwyas include debug_info [ debug_info % alwyas include debug_info
, {compile_info, [{emqx_vsn, get_vsn()}]} , {compile_info, [{emqx_vsn, get_vsn()}]}
] ++ ] ++
[{d, 'EMQX_ENTERPRISE'} || is_enterprise(Edition)] ++
[{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ].
prod_compile_opts(Edition) -> prod_compile_opts() ->
[ compressed [ compressed
, deterministic , deterministic
, warnings_as_errors , warnings_as_errors
| common_compile_opts(Edition) | common_compile_opts()
]. ].
prod_overrides() -> prod_overrides() ->
@ -149,34 +148,41 @@ prod_overrides() ->
profiles() -> profiles() ->
Vsn = get_vsn(), Vsn = get_vsn(),
ce_profiles(Vsn) ++ ee_profiles(Vsn). [ {'emqx', [ {erl_opts, prod_compile_opts()}
ce_profiles(Vsn) ->
[ {'emqx', [ {erl_opts, prod_compile_opts(ce)}
, {relx, relx(Vsn, cloud, bin, ce)} , {relx, relx(Vsn, cloud, bin, ce)}
, {overrides, prod_overrides()} , {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
, {'emqx-pkg', [ {erl_opts, prod_compile_opts(ce)} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, pkg, ce)} , {relx, relx(Vsn, cloud, pkg, ce)}
, {overrides, prod_overrides()} , {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
, {'emqx-edge', [ {erl_opts, prod_compile_opts(ce)} , {'emqx-ee', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, bin, ee)}
, {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ee)}
]}
, {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, cloud, pkg, ee)}
, {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ee)}
]}
, {'emqx-edge', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, bin, ce)} , {relx, relx(Vsn, edge, bin, ce)}
, {overrides, prod_overrides()} , {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
, {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(ce)} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()}
, {relx, relx(Vsn, edge, pkg, ce)} , {relx, relx(Vsn, edge, pkg, ce)}
, {overrides, prod_overrides()} , {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
, {check, [ {erl_opts, common_compile_opts(ce)} , {check, [ {erl_opts, common_compile_opts()}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
, {test, [ {deps, test_deps()} , {test, [ {deps, test_deps()}
, {erl_opts, common_compile_opts(ce) ++ erl_opts_i(ce) } , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) }
, {extra_src_dirs, [{"test", [{recursive, true}]}]} , {extra_src_dirs, [{"test", [{recursive, true}]}]}
, {project_app_dirs, project_app_dirs(ce)} , {project_app_dirs, project_app_dirs(ce)}
]} ]}
@ -198,13 +204,11 @@ relx(Vsn, RelType, PkgType, Edition) ->
| overlay_vars(RelType, PkgType, Edition)]} | overlay_vars(RelType, PkgType, Edition)]}
]. ].
emqx_description(cloud, ee) -> "EMQ X Enterprise"; emqx_description(cloud, ee) -> "EMQ X Enterprise Edition";
emqx_description(cloud, ce) -> "EMQ X Broker"; emqx_description(cloud, ce) -> "EMQ X Community Edition";
emqx_description(edge, ce) -> "EMQ X Edge". emqx_description(edge, ce) -> "EMQ X Edge Edition".
overlay_vars(_RelType, PkgType, ee) -> overlay_vars(RelType, PkgType, _Edition) ->
ee_overlay_vars(PkgType);
overlay_vars(RelType, PkgType, ce) ->
overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType). overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType).
%% vars per release type, cloud or edge %% vars per release type, cloud or edge
@ -289,7 +293,7 @@ relx_apps(ReleaseType, Edition) ->
, emqx_limiter , emqx_limiter
] ]
++ [quicer || is_quicer_supported()] ++ [quicer || is_quicer_supported()]
++ [emqx_license || is_enterprise(Edition)] %++ [emqx_license || is_enterprise(Edition)]
++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ [bcrypt || provide_bcrypt_release(ReleaseType)]
++ relx_apps_per_rel(ReleaseType) ++ relx_apps_per_rel(ReleaseType)
%% NOTE: applications below are only loaded after node start/restart %% NOTE: applications below are only loaded after node start/restart
@ -472,38 +476,3 @@ list_dir(Dir) ->
false -> false ->
[] []
end. end.
%% ==== Enterprise supports below ==================================================================
ee_profiles(Vsn) ->
[ {'emqx-ee', [ {erl_opts, prod_compile_opts(ee)}
, {relx, relx(Vsn, cloud, bin, ee)}
, {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ee)}
]}
, {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts(ee)}
, {relx, relx(Vsn, cloud, pkg, ee)}
, {overrides, prod_overrides()}
, {project_app_dirs, project_app_dirs(ee)}
]}
, {'check-ee', [ {erl_opts, common_compile_opts(ee)}
, {project_app_dirs, project_app_dirs(ee)}
]}
, {'test-ee', [ {deps, test_deps()}
, {erl_opts, common_compile_opts(ee) ++ erl_opts_i(ee) }
, {extra_src_dirs, [{"test", [{recursive, true}]}]}
, {project_app_dirs, project_app_dirs(ee)}
]}
].
ee_overlay_vars(PkgType) ->
Common = [],
Common ++ ee_overlay_vars_pkg(PkgType).
%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
ee_overlay_vars_pkg(bin) ->
[
];
ee_overlay_vars_pkg(pkg) ->
[
].

7
scripts/check-deps-integrity.escript
View File

@ -7,12 +7,7 @@
main([]) -> main([]) ->
Files = ["rebar.config"] ++ Files = ["rebar.config"] ++
apps_rebar_config("apps") ++ apps_rebar_config("apps") ++
case filelib:is_file("EMQX_ENTERPRISE") of apps_rebar_config("lib-ee"),
true ->
true = filelib:is_dir("lib-ee"),
apps_rebar_config("lib-ee");
false -> []
end,
Deps = collect_deps(Files, #{}), Deps = collect_deps(Files, #{}),
case count_bad_deps(Deps) of case count_bad_deps(Deps) of
0 -> 0 ->

2
scripts/find-apps.sh
View File

@ -11,9 +11,7 @@ find_app() {
} }
find_app 'apps' find_app 'apps'
if [ -f 'EMQX_ENTERPRISE' ]; then
find_app 'lib-ee' find_app 'lib-ee'
fi
## find directories in lib-extra ## find directories in lib-extra
find_app 'lib-extra' find_app 'lib-extra'

22
scripts/get-distro.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
## This script prints Linux distro name and its version number
## e.g. macos, centos8, ubuntu20.04
set -euo pipefail
if [ "$(uname -s)" = 'Darwin' ]; then
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'centos' /etc/*-release; then
DIST='centos'
VERSION_ID="$(rpm --eval '%{centos_ver}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
fi
echo "$SYSTEM"

5
scripts/get-otp-vsn.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
set -euo pipefail
erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().'

2
scripts/git-hook-pre-push.sh
View File

@ -4,6 +4,8 @@ set -euo pipefail
url="$2" url="$2"
# we keep this to secure OLD private repo
# even after we have disclosed new code under EMQ BSL 1.0
if [ -f 'EMQX_ENTERPRISE' ]; then if [ -f 'EMQX_ENTERPRISE' ]; then
if [[ "$url" != *emqx-enterprise* ]]; then if [[ "$url" != *emqx-enterprise* ]]; then
echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo" echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo"