wehub-resource-sync
511 lines
19 KiB
YAML
511 lines
19 KiB
YAML
name: E2E matrix
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 5 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
debug_enabled:
|
|
type: boolean
|
|
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
|
|
required: false
|
|
default: false
|
|
|
|
env:
|
|
CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress
|
|
# Pin corepack to the pnpm version from packageManager. Without this, corepack
|
|
# falls back to "latest" in directories that have no packageManager field
|
|
# (e.g. e2e temp dirs), pulling pnpm 11 and breaking install.
|
|
COREPACK_DEFAULT_TO_LATEST: '0'
|
|
|
|
permissions: {}
|
|
jobs:
|
|
preinstall:
|
|
if: ${{ github.repository_owner == 'nrwl' }}
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 20
|
|
env:
|
|
NODE_VERSION: ${{ matrix.node_version }}
|
|
strategy:
|
|
matrix:
|
|
os:
|
|
- ubuntu-latest
|
|
- macos-latest
|
|
# - windows-latest Windows fails to build gradle wrapper which always runs when we build nx.
|
|
## https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment
|
|
node_version:
|
|
- 22
|
|
- 24
|
|
- 26
|
|
exclude:
|
|
# macos skips the oldest node to keep the macos matrix slim
|
|
- os: macos-latest
|
|
node_version: 22
|
|
# - os: windows-latest TODO(Jack): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
|
|
# node_version: 22
|
|
|
|
name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
filter: tree:0
|
|
|
|
- name: Setup dev tools with mise
|
|
uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3
|
|
|
|
- name: Enable corepack and install pnpm
|
|
run: |
|
|
npm install -g corepack@latest
|
|
corepack enable
|
|
corepack prepare --activate
|
|
|
|
- name: Get pnpm store directory
|
|
id: pnpm-cache
|
|
run: echo "path=$(pnpm store path)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Cache pnpm store
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
path: ${{ steps.pnpm-cache.outputs.path }}
|
|
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
|
|
restore-keys: |
|
|
${{ runner.os }}-pnpm-store-
|
|
|
|
- name: Ensure Python setuptools Installed on Macos
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
id: brew-install-python-setuptools
|
|
run: brew install python-setuptools
|
|
|
|
- name: Install pnpm packages
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Install Playwright
|
|
run: pnpm playwright install --with-deps
|
|
|
|
- name: Restore .NET packages
|
|
run: dotnet restore nx.sln
|
|
|
|
- name: Homebrew cache directory path
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
id: homebrew-cache-dir-path
|
|
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Cache Homebrew
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
lookup-only: true
|
|
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
|
|
key: brew-${{ matrix.node_version }}
|
|
restore-keys: |
|
|
brew-
|
|
|
|
- name: Cache Cypress
|
|
id: cache-cypress
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
lookup-only: true
|
|
path: '${{ github.workspace }}/.cypress'
|
|
key: ${{ runner.os }}-cypress
|
|
|
|
- name: Install Cypress
|
|
if: steps.cache-cypress.outputs.cache-hit != 'true'
|
|
run: npx cypress install
|
|
|
|
prepare-matrix:
|
|
name: Prepare matrix combinations
|
|
if: ${{ github.repository_owner == 'nrwl' }}
|
|
timeout-minutes: 5
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
matrix: ${{ steps.process-json.outputs.MATRIX }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
filter: tree:0
|
|
|
|
- name: Process matrix data
|
|
id: process-json
|
|
run: echo "MATRIX=$(npx tsx .github/workflows/nightly/process-matrix.ts | jq -c .)" >> $GITHUB_OUTPUT
|
|
|
|
e2e:
|
|
if: ${{ github.repository_owner == 'nrwl' }}
|
|
needs:
|
|
- preinstall
|
|
- prepare-matrix
|
|
permissions:
|
|
contents: read
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 200 # <- cap each job to 200 minutes
|
|
env:
|
|
NODE_VERSION: ${{ matrix.node_version }}
|
|
strategy:
|
|
matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job
|
|
fail-fast: false
|
|
|
|
name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
filter: tree:0
|
|
|
|
- name: Prepare dir for output
|
|
run: mkdir -p outputs
|
|
|
|
- name: Setup dev tools with mise
|
|
uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3
|
|
|
|
- name: Enable corepack and install pnpm
|
|
run: |
|
|
npm install -g corepack@latest
|
|
corepack enable
|
|
corepack prepare --activate
|
|
|
|
- name: Install pnpm packages
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Install Playwright
|
|
run: pnpm playwright install --with-deps
|
|
|
|
- name: Restore .NET packages
|
|
run: dotnet restore nx.sln
|
|
|
|
- name: Cleanup
|
|
if: ${{ matrix.os == 'ubuntu-latest' }}
|
|
run: |
|
|
# Workaround to provide additional free space for testing.
|
|
# https://github.com/actions/virtual-environments/issues/2840
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf '/usr/local/share/boost'
|
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
sudo apt-get install lsof
|
|
echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
|
|
|
|
- name: Homebrew cache directory path
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
id: homebrew-cache-dir-path
|
|
run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Cache Homebrew
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
|
|
key: brew-${{ matrix.node_version }}
|
|
restore-keys: |
|
|
brew-
|
|
|
|
- name: Cache Cypress
|
|
id: cache-cypress
|
|
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
|
|
with:
|
|
path: '${{ github.workspace }}/.cypress'
|
|
key: ${{ runner.os }}-cypress
|
|
|
|
- name: Install Cypress
|
|
if: steps.cache-cypress.outputs.cache-hit != 'true'
|
|
run: npx cypress install
|
|
|
|
- name: Configure Detox Environment, Install applesimutils
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
run: |
|
|
# Ensure Xcode command line tools are installed and configured
|
|
xcode-select --print-path || sudo xcode-select --reset
|
|
sudo xcode-select -s /Applications/Xcode.app
|
|
|
|
# Install or update applesimutils with error handling
|
|
if ! brew list applesimutils &>/dev/null; then
|
|
echo 'Installing applesimutils...'
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
|
|
# Homebrew now refuses to load formulae from third-party taps unless trusted
|
|
brew trust wix/brew
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null || {
|
|
echo 'Failed to install applesimutils, retrying with update...'
|
|
brew update
|
|
brew trust wix/brew
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils
|
|
}
|
|
else
|
|
echo 'Updating applesimutils...'
|
|
HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade applesimutils || true
|
|
fi
|
|
|
|
# Verify applesimutils installation
|
|
applesimutils --version || (echo 'applesimutils installation failed' && exit 1)
|
|
|
|
# Configure environment for M-series Mac
|
|
echo 'DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer' >> $GITHUB_ENV
|
|
echo 'PLATFORM_NAME=iOS Simulator' >> $GITHUB_ENV
|
|
|
|
# Set additional environment variables for better debugging
|
|
echo 'DETOX_DISABLE_TELEMETRY=1' >> $GITHUB_ENV
|
|
echo 'DETOX_LOG_LEVEL=trace' >> $GITHUB_ENV
|
|
|
|
# Verify Xcode installation
|
|
xcodebuild -version
|
|
|
|
timeout-minutes: 10
|
|
continue-on-error: false
|
|
|
|
- name: Reset iOS Simulators
|
|
if: ${{ matrix.os == 'macos-latest' }}
|
|
id: reset-simulators
|
|
run: |
|
|
echo 'Resetting iOS Simulators...'
|
|
|
|
# Kill simulator processes
|
|
sudo killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
|
|
killall 'Simulator' 2>/dev/null || true
|
|
killall 'iOS Simulator' 2>/dev/null || true
|
|
|
|
# Wait for processes to terminate
|
|
sleep 3
|
|
|
|
# Shutdown and erase all simulators (ignore failures)
|
|
xcrun simctl shutdown all 2>/dev/null || true
|
|
sleep 5
|
|
xcrun simctl erase all 2>/dev/null || true
|
|
|
|
# If erase failed, try the nuclear option
|
|
if xcrun simctl list devices | grep -q 'Booted' 2>/dev/null; then
|
|
echo 'Standard reset failed, using nuclear option...'
|
|
rm -rf ~/Library/Developer/CoreSimulator/Devices/* 2>/dev/null || true
|
|
launchctl remove com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
|
|
sleep 3
|
|
fi
|
|
|
|
# Clean up additional directories
|
|
rm -rf ~/Library/Developer/CoreSimulator/Caches/* 2>/dev/null || true
|
|
rm -rf ~/Library/Logs/CoreSimulator/* 2>/dev/null || true
|
|
rm -rf ~/Library/Developer/Xcode/DerivedData/* 2>/dev/null || true
|
|
|
|
echo 'Simulator reset completed'
|
|
timeout-minutes: 5
|
|
continue-on-error: true
|
|
|
|
- name: Verify Simulator Reset
|
|
if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
|
|
run: |
|
|
# Verify CoreSimulator service restarted
|
|
pgrep -fl 'CoreSimulator' || (echo 'CoreSimulator service not running' && exit 1)
|
|
|
|
# Verify simulator runtime paths exist and are writable
|
|
test -d ~/Library/Developer/CoreSimulator/Devices || (echo 'Simulator devices directory missing' && exit 1)
|
|
touch ~/Library/Developer/CoreSimulator/Devices/test || (echo 'Simulator devices directory not writable' && exit 1)
|
|
rm ~/Library/Developer/CoreSimulator/Devices/test
|
|
timeout-minutes: 5
|
|
|
|
- name: Diagnose Simulator Reset Failure
|
|
if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'failure' }}
|
|
run: |
|
|
echo 'Simulator reset failed. Collecting diagnostic information...'
|
|
xcrun simctl list
|
|
echo 'Checking simulator logs...'
|
|
ls -la ~/Library/Logs/CoreSimulator/ || echo 'No simulator logs found'
|
|
|
|
- name: Configure git metadata (needed for lerna smoke tests)
|
|
if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
|
|
run: |
|
|
git config --global user.email test@test.com
|
|
git config --global user.name 'Test Test'
|
|
|
|
- name: Set starting timestamp
|
|
if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
|
|
id: before-e2e
|
|
shell: bash
|
|
run: |
|
|
echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Run e2e tests with pnpm (Linux/Windows)
|
|
id: e2e-run-pnpm
|
|
if: ${{ matrix.os != 'macos-latest' }}
|
|
run: pnpm nx run ${{ matrix.project }}:e2e-local
|
|
shell: bash
|
|
timeout-minutes: ${{ matrix.os_timeout }}
|
|
env:
|
|
NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
|
|
NX_DAEMON: 'true'
|
|
NX_PERF_LOGGING: 'false'
|
|
NX_E2E_VERBOSE_LOGGING: 'true'
|
|
NX_NATIVE_LOGGING: 'false'
|
|
NX_E2E_RUN_E2E: 'true'
|
|
NX_CLOUD_NO_TIMEOUTS: 'true'
|
|
NX_E2E_SKIP_GLOBAL_CLEANUP: 'true'
|
|
NODE_OPTIONS: --max_old_space_size=8192
|
|
SELECTED_PM: ${{ matrix.package_manager }}
|
|
npm_config_registry: http://localhost:4872
|
|
YARN_REGISTRY: http://localhost:4872
|
|
CI: true
|
|
|
|
- name: Run e2e tests with npm (macOS)
|
|
id: e2e-run-npm
|
|
if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
|
|
run: |
|
|
# Run the tests
|
|
if [[ '${{ matrix.project }}' == 'e2e-detox' ]] || [[ '${{ matrix.project }}' == 'e2e-react-native' ]] || [[ '${{ matrix.project }}' == 'e2e-expo' ]]; then
|
|
NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-macos-local
|
|
else
|
|
NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-local
|
|
fi
|
|
|
|
env:
|
|
NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
|
|
NX_PERF_LOGGING: 'false'
|
|
NX_CI_EXECUTION_ENV: 'macos'
|
|
NX_E2E_VERBOSE_LOGGING: 'true'
|
|
NX_NATIVE_LOGGING: 'false'
|
|
NX_E2E_RUN_E2E: 'true'
|
|
NX_E2E_SKIP_GLOBAL_CLEANUP: 'true'
|
|
NODE_OPTIONS: --max_old_space_size=8192
|
|
SELECTED_PM: 'npm'
|
|
npm_config_registry: http://localhost:4872
|
|
YARN_REGISTRY: http://localhost:4872
|
|
DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer'
|
|
CI: true
|
|
|
|
- name: Save matrix config in file
|
|
if: ${{ always() }}
|
|
id: save-matrix
|
|
shell: bash
|
|
run: |
|
|
before=${{ steps.before-e2e.outputs.timestamp }}
|
|
now=$(date +%s)
|
|
delta=$(($now - $before))
|
|
|
|
# Determine the outcome based on which step ran
|
|
outcome='${{ matrix.os == 'macos-latest' && steps.e2e-run-npm.outcome || steps.e2e-run-pnpm.outcome }}'
|
|
|
|
matrix=$((
|
|
echo '${{ toJSON(matrix) }}'
|
|
) | jq --argjson delta $delta -c '. + { "status": "'"$outcome"'", "duration": $delta }')
|
|
echo "$matrix" > 'outputs/matrix.json'
|
|
|
|
- name: Upload matrix config
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
if: ${{ always() }}
|
|
with:
|
|
name: ${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
|
|
overwrite: true
|
|
if-no-files-found: 'ignore'
|
|
path: 'outputs/matrix.json'
|
|
|
|
- name: Setup tmate session
|
|
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
|
|
uses: mxschmitt/action-tmate@1fb8b1023602bf1fd0e2994d7f1e93015cb5bbec # v3.22
|
|
timeout-minutes: 15
|
|
with:
|
|
sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging
|
|
|
|
process-result:
|
|
if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
|
|
runs-on: ubuntu-latest
|
|
needs: e2e
|
|
timeout-minutes: 15
|
|
outputs:
|
|
message: ${{ steps.process-json.outputs.slack_message }}
|
|
proj_duration: ${{ steps.process-json.outputs.slack_proj_duration }}
|
|
pm_duration: ${{ steps.process-json.outputs.slack_pm_duration }}
|
|
codeowners: ${{ steps.process-json.outputs.codeowners }}
|
|
has_golden_failures: ${{ steps.process-json.outputs.has_golden_failures }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
filter: tree:0
|
|
|
|
- name: Prepare dir for output
|
|
run: mkdir -p outputs
|
|
|
|
- name: Load outputs
|
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
with:
|
|
path: outputs
|
|
|
|
- name: Join and stringify matrix configs
|
|
id: combine-json
|
|
run: |
|
|
combined=$(jq -sc . outputs/*/matrix.json)
|
|
echo "combined=$combined" >> $GITHUB_OUTPUT
|
|
|
|
- name: Process results and collect failure details
|
|
id: process-json
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
echo '${{ steps.combine-json.outputs.combined }}' | npx tsx .github/workflows/nightly/process-result.ts
|
|
|
|
report-failure:
|
|
if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'true' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
|
|
needs: process-result
|
|
runs-on: ubuntu-latest
|
|
name: Report failure
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Send notification
|
|
uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
|
|
with:
|
|
status: 'failure'
|
|
message_format: '${{ needs.process-result.outputs.message }}'
|
|
notification_title: 'Golden Test Failure'
|
|
footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
|
|
mention_groups: ${{ needs.process-result.outputs.codeowners }}
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
|
|
|
|
report-success:
|
|
if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'false' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
|
|
needs: process-result
|
|
runs-on: ubuntu-latest
|
|
name: Report status
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Send notification
|
|
uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
|
|
with:
|
|
status: 'success'
|
|
message_format: '${{ needs.process-result.outputs.message }}'
|
|
notification_title: '✅ Golden Tests: All Passed!'
|
|
footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
|
|
|
|
report-pm-time:
|
|
if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
|
|
needs: process-result
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
name: Report duration per package manager
|
|
steps:
|
|
- name: Send notification
|
|
uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
|
|
with:
|
|
status: 'skipped'
|
|
message_format: '${{ needs.process-result.outputs.pm_duration }}'
|
|
notification_title: '⌛ Total duration per package manager (ubuntu only)'
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
|
|
|
|
report-proj-time:
|
|
if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
|
|
needs: process-result
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
name: Report duration per project
|
|
steps:
|
|
- name: Send notification
|
|
uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
|
|
with:
|
|
status: 'skipped'
|
|
message_format: '${{ needs.process-result.outputs.proj_duration }}'
|
|
notification_title: '⌛ E2E Project duration stats'
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
|