From 3d8f79c4ac9250df430e191adfaf9e470dafee62 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 9 Aug 2024 09:15:01 +0200 Subject: [PATCH] ci: automatically create PRs to emqx/emqx-i18n and emqx/emqx-docs --- .github/workflows/_pr_entrypoint.yaml | 42 ++++++++++- .github/workflows/_push-entrypoint.yaml | 31 +++++++- .github/workflows/build_slim_packages.yaml | 7 -- .github/workflows/run_conf_tests.yaml | 2 +- .github/workflows/spellcheck.yaml | 3 +- .github/workflows/static_checks.yaml | 2 +- .github/workflows/update-emqx-docs.yaml | 84 ++++++++++++++++++++++ .github/workflows/update-emqx-i18n.yaml | 50 +++++++++++++ 8 files changed, 204 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/update-emqx-docs.yaml create mode 100644 .github/workflows/update-emqx-i18n.yaml diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 78206836e..dd0c28289 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -149,14 +149,25 @@ jobs: run: | make ensure-rebar3 make ${PROFILE}-compile test-compile - echo "PROFILE=${PROFILE}" | tee -a .env - echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env + echo "PROFILE=${PROFILE}" | tee -a .github_env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .github_env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . + make ${PROFILE}-rel - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 with: name: ${{ matrix.profile }} path: ${{ matrix.profile }}.zip retention-days: 7 + - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + with: + name: "${{ matrix.profile }}-schema-dump" + path: | + .github_env + env.sh + scripts/spellcheck + _build/docgen/${{ matrix.profile }}/*.json + _build/docgen/${{ matrix.profile }}/*.hocon + retention-days: 7 run_emqx_app_tests: needs: @@ -205,7 +216,7 @@ jobs: spellcheck: needs: - sanity-checks - - build_slim_packages + - compile uses: ./.github/workflows/spellcheck.yaml run_conf_tests: @@ -242,3 +253,28 @@ jobs: - sanity-checks - build_docker_for_test uses: ./.github/workflows/run_helm_tests.yaml + + update_i18n: + needs: + - compile + uses: ./.github/workflows/update-emqx-i18n.yaml + + build_and_push_docker_images: + needs: + - sanity-checks + uses: ./.github/workflows/build_and_push_docker_images.yaml + with: + profile: emqx-enterprise + publish: false + latest: ${{ format('{0}', 'false') }} + build_from: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12 + run_from: public.ecr.aws/debian/debian:stable-20240612-slim + secrets: inherit + + update_emqx_docs: + needs: + - compile + - build_and_push_docker_images + uses: ./.github/workflows/update-emqx-docs.yaml + with: + profile: emqx-enterprise diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 9fd4789c9..6c522bf20 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -160,14 +160,22 @@ jobs: ENABLE_COVER_COMPILE: 1 run: | make $PROFILE - echo "PROFILE=${PROFILE}" | tee -a .env - echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env + echo "PROFILE=${PROFILE}" | tee -a .github_env + echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .github_env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . + make ${PROFILE}-rel - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 with: name: ${{ matrix.profile }} path: ${{ matrix.profile }}.zip - retention-days: 1 + retention-days: 7 + - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + with: + name: "${{ matrix.profile }}-schema-dump" + path: | + _build/docgen/${{ matrix.profile }}/*.json + _build/docgen/${{ matrix.profile }}/*.hocon + retention-days: 7 run_emqx_app_tests: needs: @@ -212,3 +220,20 @@ jobs: with: builder: ${{ needs.init.outputs.BUILDER }} ct-matrix: ${{ needs.prepare.outputs.ct-matrix }} + + update_i18n: + if: needs.prepare.outputs.release == 'true' + needs: + - prepare + - compile + uses: ./.github/workflows/update-emqx-i18n.yaml + + update_emqx_docs: + if: needs.prepare.outputs.release == 'true' + needs: + - prepare + - compile + - build_and_push_docker_images + uses: ./.github/workflows/update-emqx-docs.yaml + with: + profile: ${{ needs.prepare.outputs.profile }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 3526a4857..df6c910a5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -47,13 +47,6 @@ jobs: path: _packages/${{ matrix.profile[0] }}/* retention-days: 7 compression-level: 0 - - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 - with: - name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" - path: | - scripts/spellcheck - _build/docgen/${{ matrix.profile[0] }}/schema-en.json - retention-days: 7 mac: strategy: diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index b02aec5e8..883955fa6 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -32,7 +32,7 @@ jobs: run: | unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" - - run: cat .env | tee -a $GITHUB_ENV + - run: cat .github_env | tee -a $GITHUB_ENV - run: make ${{ matrix.profile }} - run: ./scripts/test/check-example-configs.sh - run: ./scripts/conf-test/run.sh diff --git a/.github/workflows/spellcheck.yaml b/.github/workflows/spellcheck.yaml index a6857ff21..497e3d012 100644 --- a/.github/workflows/spellcheck.yaml +++ b/.github/workflows/spellcheck.yaml @@ -21,8 +21,7 @@ jobs: steps: - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - pattern: "${{ matrix.profile }}-schema-dump-*-x64" - merge-multiple: true + name: "${{ matrix.profile }}-schema-dump" - name: Run spellcheck run: | bash scripts/spellcheck/spellcheck.sh _build/docgen/${{ matrix.profile }}/schema-en.json diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index 4f89fb259..40054958f 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -43,6 +43,6 @@ jobs: key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }} restore-keys: | rebar3-dialyzer-plt-${{ matrix.profile }}- - - run: cat .env | tee -a $GITHUB_ENV + - run: cat .github_env | tee -a $GITHUB_ENV - name: run static checks run: make static_checks diff --git a/.github/workflows/update-emqx-docs.yaml b/.github/workflows/update-emqx-docs.yaml new file mode 100644 index 000000000..8db3d016b --- /dev/null +++ b/.github/workflows/update-emqx-docs.yaml @@ -0,0 +1,84 @@ +name: Update i18n + +concurrency: + group: update-i18n-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + inputs: + profile: + required: true + type: string + +permissions: + contents: read + +jobs: + emqx-docs: + runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} + + permissions: + pull-requests: write + + env: + PROFILE: ${{ inputs.profile }} + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: emqx/emqx-docs + + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: "emqx-*-schema-dump" + merge-multiple: true + + - name: Create PR to update api docs and config manual in emqx-docs + env: + GITHUB_TOKEN: ${{ secrets.CI_GIT_TOKEN }} + run: | + set -euxo pipefail + source .github_env + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + BASE_BRANCH="$(echo release-$(echo $PKG_VSN | cut -d '.' -f 1-2))" + git fetch origin + git checkout ${BASE_BRANCH} + NEW_BRANCH="update-api-cfg-${PKG_VSN}" + git checkout -b ${NEW_BRANCH} + + case "${PROFILE}" in + emqx) + EDITION=ce + EDITION_UPPER=CE + ;; + emqx-enterprise) + EDITION=ee + EDITION_UPPER=EE + ;; + esac + + VERSION=$(echo "${PKG_VSN}" | cut -d '-' -f 1) + MINOR_VERSION=$(echo "${PKG_VSN}" | cut -d '.' -f 1-2) + sed -i "s/${EDITION_UPPER}_VERSION=.*/${EDITION_UPPER}_VERSION=${VERSION}/" ./current-version.env + sed -i "s/${EDITION_UPPER}_MINOR_VERSION=.*/${EDITION_UPPER}_MINOR_VERSION=${MINOR_VERSION}/" ./current-version.env + git add current-version.env + + for lang in en zh; do + docker run -d --name emqx -p 18083:18083 -e EMQX_dashboard__i18n_lang=${lang} "emqx/${PROFILE}:${PKG_VSN}" + curl -fsS -m 10 --retry 10 --retry-all-errors -o /dev/null http://localhost:18083/api-docs/swagger.json + ./rewrite-swagger.sh ${EDITION} ${lang} + docker logs emqx + docker rm -f emqx + cp "_build/docgen/${PROFILE}/schema-v2-${lang}.json" "hocon/hocon-${EDITION}-v${VERSION}-${lang}.json" + git add "redocly/${EDITION}-${lang}.json" + git add "hocon/hocon-${EDITION}-v${VERSION}-${lang}.json" + done + + git commit -m "chore($VERSION): update api and cfg manual" + git push origin ${NEW_BRANCH}:${NEW_BRANCH} + for pr in $(gh pr list --state open --base ${BASE_BRANCH} --label update-api-cfg --search "update api and cfg manual ${VERSION} in:title" --repo emqx/emqx-docs --json number --jq '.[] | .number'); do + gh pr close $pr --repo emqx/emqx-docs --delete-branch || true + done + gh pr create --title "update api and cfg manual ${VERSION}" --body '' --base ${BASE_BRANCH} --head ${NEW_BRANCH} --label update-api-cfg --repo emqx/emqx-docs diff --git a/.github/workflows/update-emqx-i18n.yaml b/.github/workflows/update-emqx-i18n.yaml new file mode 100644 index 000000000..fbebd17f0 --- /dev/null +++ b/.github/workflows/update-emqx-i18n.yaml @@ -0,0 +1,50 @@ +name: Update emqx-i18n + +concurrency: + group: update-i18n-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_call: + +permissions: + contents: read + +jobs: + update-i18n: + runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }} + + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + repository: emqx/emqx-i18n + + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: "emqx-enterprise-schema-dump" + + - name: PR to update desc.en.hocon in emqx-i18n + env: + GITHUB_TOKEN: ${{ secrets.CI_GIT_TOKEN }} + run: | + set -euxo pipefail + source .github_env + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + BASE_BRANCH="$(echo v$(echo $PKG_VSN | awk -F. '{print $1$2}'))" + git fetch origin + git checkout ${BASE_BRANCH} + NEW_BRANCH="sync-${PKG_VSN}" + git checkout -b ${NEW_BRANCH} + cp _build/docgen/emqx-enterprise/desc.en.hocon ./ + git add desc.en.hocon + git commit -m "chore: update desc.en.hocon for ${PKG_VSN}" + git push origin ${NEW_BRANCH}:${NEW_BRANCH} + VERSION=$(echo "${PKG_VSN}" | cut -d '-' -f 1) + for pr in $(gh pr list --state open --base ${BASE_BRANCH} --label sync-en-doc --search "sync en doc for ${VERSION} in:title" --repo emqx/emqx-i18n --json number --jq '.[] | .number'); do + gh pr close $pr --repo emqx/emqx-i18n --delete-branch || true + done + gh pr create --title "sync en doc for ${VERSION}" --base ${BASE_BRANCH} --head ${NEW_BRANCH} --body '' --label sync-en-doc --repo emqx/emqx-i18n