name: E2E matrix

on:
  schedule:
    - cron: '0 5 * * *'
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

env:
  CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress
  # Pin corepack to the pnpm version from packageManager. Without this, corepack
  # falls back to "latest" in directories that have no packageManager field
  # (e.g. e2e temp dirs), pulling pnpm 11 and breaking install.
  COREPACK_DEFAULT_TO_LATEST: '0'

permissions: {}
jobs:
  preinstall:
    if: ${{ github.repository_owner == 'nrwl' }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 20
    env:
      NODE_VERSION: ${{ matrix.node_version }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
          # - windows-latest Windows fails to build gradle wrapper which always runs when we build nx.
          ## https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment
        node_version:
          - 22
          - 24
          - 26
        exclude:
          # macos skips the oldest node to keep the macos matrix slim
          - os: macos-latest
            node_version: 22
          # - os: windows-latest TODO(Jack): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
          #   node_version: 22

    name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
    steps:
      - name: Checkout
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Setup dev tools with mise
        uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3

      - name: Enable corepack and install pnpm
        run: |
          npm install -g corepack@latest
          corepack enable
          corepack prepare --activate

      - name: Get pnpm store directory
        id: pnpm-cache
        run: echo "path=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Cache pnpm store
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
        with:
          path: ${{ steps.pnpm-cache.outputs.path }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - name: Ensure Python setuptools Installed on Macos
        if: ${{ matrix.os == 'macos-latest' }}
        id: brew-install-python-setuptools
        run: brew install python-setuptools

      - name: Install pnpm packages
        run: pnpm install --frozen-lockfile

      - name: Install Playwright
        run: pnpm playwright install --with-deps

      - name: Restore .NET packages
        run: dotnet restore nx.sln

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
        with:
          lookup-only: true
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
        with:
          lookup-only: true
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

  prepare-matrix:
    name: Prepare matrix combinations
    if: ${{ github.repository_owner == 'nrwl' }}
    timeout-minutes: 5
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.process-json.outputs.MATRIX }}
    steps:
      - name: Checkout
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Process matrix data
        id: process-json
        run: echo "MATRIX=$(npx tsx .github/workflows/nightly/process-matrix.ts | jq -c .)" >> $GITHUB_OUTPUT

  e2e:
    if: ${{ github.repository_owner == 'nrwl' }}
    needs:
      - preinstall
      - prepare-matrix
    permissions:
      contents: read
    runs-on: ${{ matrix.os }}
    timeout-minutes: 200 # <- cap each job to 200 minutes
    env:
      NODE_VERSION: ${{ matrix.node_version }}
    strategy:
      matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job
      fail-fast: false

    name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
    steps:
      - name: Checkout
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Prepare dir for output
        run: mkdir -p outputs

      - name: Setup dev tools with mise
        uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3

      - name: Enable corepack and install pnpm
        run: |
          npm install -g corepack@latest
          corepack enable
          corepack prepare --activate

      - name: Install pnpm packages
        run: pnpm install --frozen-lockfile

      - name: Install Playwright
        run: pnpm playwright install --with-deps

      - name: Restore .NET packages
        run: dotnet restore nx.sln

      - name: Cleanup
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # Workaround to provide additional free space for testing.
          # https://github.com/actions/virtual-environments/issues/2840
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf '/usr/local/share/boost'
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo apt-get install lsof
          echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
        with:
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
        with:
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

      - name: Configure Detox Environment, Install applesimutils
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          # Ensure Xcode command line tools are installed and configured
          xcode-select --print-path || sudo xcode-select --reset
          sudo xcode-select -s /Applications/Xcode.app

          # Install or update applesimutils with error handling
          if ! brew list applesimutils &>/dev/null; then
            echo 'Installing applesimutils...'
            HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
            # Homebrew now refuses to load formulae from third-party taps unless trusted
            brew trust wix/brew
            HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null || {
              echo 'Failed to install applesimutils, retrying with update...'
              brew update
              brew trust wix/brew
              HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils
            }
          else
            echo 'Updating applesimutils...'
            HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade applesimutils || true
          fi

          # Verify applesimutils installation
          applesimutils --version || (echo 'applesimutils installation failed' && exit 1)

          # Configure environment for M-series Mac
          echo 'DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer' >> $GITHUB_ENV
          echo 'PLATFORM_NAME=iOS Simulator' >> $GITHUB_ENV

          # Set additional environment variables for better debugging
          echo 'DETOX_DISABLE_TELEMETRY=1' >> $GITHUB_ENV
          echo 'DETOX_LOG_LEVEL=trace' >> $GITHUB_ENV

          # Verify Xcode installation
          xcodebuild -version

        timeout-minutes: 10
        continue-on-error: false

      - name: Reset iOS Simulators
        if: ${{ matrix.os == 'macos-latest' }}
        id: reset-simulators
        run: |
          echo 'Resetting iOS Simulators...'

          # Kill simulator processes
          sudo killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
          killall 'Simulator' 2>/dev/null || true
          killall 'iOS Simulator' 2>/dev/null || true

          # Wait for processes to terminate
          sleep 3

          # Shutdown and erase all simulators (ignore failures)
          xcrun simctl shutdown all 2>/dev/null || true
          sleep 5
          xcrun simctl erase all 2>/dev/null || true

          # If erase failed, try the nuclear option
          if xcrun simctl list devices | grep -q 'Booted' 2>/dev/null; then
            echo 'Standard reset failed, using nuclear option...'
            rm -rf ~/Library/Developer/CoreSimulator/Devices/* 2>/dev/null || true
            launchctl remove com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
            sleep 3
          fi

          # Clean up additional directories
          rm -rf ~/Library/Developer/CoreSimulator/Caches/* 2>/dev/null || true
          rm -rf ~/Library/Logs/CoreSimulator/* 2>/dev/null || true
          rm -rf ~/Library/Developer/Xcode/DerivedData/* 2>/dev/null || true

          echo 'Simulator reset completed'
        timeout-minutes: 5
        continue-on-error: true

      - name: Verify Simulator Reset
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
        run: |
          # Verify CoreSimulator service restarted
          pgrep -fl 'CoreSimulator' || (echo 'CoreSimulator service not running' && exit 1)

          # Verify simulator runtime paths exist and are writable
          test -d ~/Library/Developer/CoreSimulator/Devices || (echo 'Simulator devices directory missing' && exit 1)
          touch ~/Library/Developer/CoreSimulator/Devices/test || (echo 'Simulator devices directory not writable' && exit 1)
          rm ~/Library/Developer/CoreSimulator/Devices/test
        timeout-minutes: 5

      - name: Diagnose Simulator Reset Failure
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'failure' }}
        run: |
          echo 'Simulator reset failed. Collecting diagnostic information...'
          xcrun simctl list
          echo 'Checking simulator logs...'
          ls -la ~/Library/Logs/CoreSimulator/ || echo 'No simulator logs found'

      - name: Configure git metadata (needed for lerna smoke tests)
        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
        run: |
          git config --global user.email test@test.com
          git config --global user.name 'Test Test'

      - name: Set starting timestamp
        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
        id: before-e2e
        shell: bash
        run: |
          echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Run e2e tests with pnpm (Linux/Windows)
        id: e2e-run-pnpm
        if: ${{ matrix.os != 'macos-latest' }}
        run: pnpm nx run ${{ matrix.project }}:e2e-local
        shell: bash
        timeout-minutes: ${{ matrix.os_timeout }}
        env:
          NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
          NX_DAEMON: 'true'
          NX_PERF_LOGGING: 'false'
          NX_E2E_VERBOSE_LOGGING: 'true'
          NX_NATIVE_LOGGING: 'false'
          NX_E2E_RUN_E2E: 'true'
          NX_CLOUD_NO_TIMEOUTS: 'true'
          NX_E2E_SKIP_GLOBAL_CLEANUP: 'true'
          NODE_OPTIONS: --max_old_space_size=8192
          SELECTED_PM: ${{ matrix.package_manager }}
          npm_config_registry: http://localhost:4872
          YARN_REGISTRY: http://localhost:4872
          CI: true

      - name: Run e2e tests with npm (macOS)
        id: e2e-run-npm
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
        run: |
          # Run the tests
          if [[ '${{ matrix.project }}' == 'e2e-detox' ]] || [[ '${{ matrix.project }}' == 'e2e-react-native' ]] || [[ '${{ matrix.project }}' == 'e2e-expo' ]]; then
            NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-macos-local
          else
            NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-local
          fi

        env:
          NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
          NX_PERF_LOGGING: 'false'
          NX_CI_EXECUTION_ENV: 'macos'
          NX_E2E_VERBOSE_LOGGING: 'true'
          NX_NATIVE_LOGGING: 'false'
          NX_E2E_RUN_E2E: 'true'
          NX_E2E_SKIP_GLOBAL_CLEANUP: 'true'
          NODE_OPTIONS: --max_old_space_size=8192
          SELECTED_PM: 'npm'
          npm_config_registry: http://localhost:4872
          YARN_REGISTRY: http://localhost:4872
          DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer'
          CI: true

      - name: Save matrix config in file
        if: ${{ always() }}
        id: save-matrix
        shell: bash
        run: |
          before=${{ steps.before-e2e.outputs.timestamp }}
          now=$(date +%s)
          delta=$(($now - $before))

          # Determine the outcome based on which step ran
          outcome='${{ matrix.os == 'macos-latest' && steps.e2e-run-npm.outcome || steps.e2e-run-pnpm.outcome }}'

          matrix=$((
            echo '${{ toJSON(matrix) }}'
          ) | jq --argjson delta $delta -c '. + { "status": "'"$outcome"'", "duration": $delta }')
          echo "$matrix" > 'outputs/matrix.json'

      - name: Upload matrix config
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        if: ${{ always() }}
        with:
          name: ${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
          overwrite: true
          if-no-files-found: 'ignore'
          path: 'outputs/matrix.json'

      - name: Setup tmate session
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
        uses: mxschmitt/action-tmate@1fb8b1023602bf1fd0e2994d7f1e93015cb5bbec # v3.22
        timeout-minutes: 15
        with:
          sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging

  process-result:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    needs: e2e
    timeout-minutes: 15
    outputs:
      message: ${{ steps.process-json.outputs.slack_message }}
      proj_duration: ${{ steps.process-json.outputs.slack_proj_duration }}
      pm_duration: ${{ steps.process-json.outputs.slack_pm_duration }}
      codeowners: ${{ steps.process-json.outputs.codeowners }}
      has_golden_failures: ${{ steps.process-json.outputs.has_golden_failures }}
    steps:
      - name: Checkout
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Prepare dir for output
        run: mkdir -p outputs

      - name: Load outputs
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          path: outputs

      - name: Join and stringify matrix configs
        id: combine-json
        run: |
          combined=$(jq -sc . outputs/*/matrix.json)
          echo "combined=$combined" >> $GITHUB_OUTPUT

      - name: Process results and collect failure details
        id: process-json
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          echo '${{ steps.combine-json.outputs.combined }}' | npx tsx .github/workflows/nightly/process-result.ts

  report-failure:
    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'true' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report failure
    timeout-minutes: 10
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
        with:
          status: 'failure'
          message_format: '${{ needs.process-result.outputs.message }}'
          notification_title: 'Golden Test Failure'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
          mention_groups: ${{ needs.process-result.outputs.codeowners }}
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-success:
    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'false' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report status
    timeout-minutes: 10
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
        with:
          status: 'success'
          message_format: '${{ needs.process-result.outputs.message }}'
          notification_title: '✅ Golden Tests: All Passed!'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-pm-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    timeout-minutes: 10
    name: Report duration per package manager
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.pm_duration }}'
          notification_title: '⌛ Total duration per package manager (ubuntu only)'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-proj-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    timeout-minutes: 10
    name: Report duration per project
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.proj_duration }}'
          notification_title: '⌛ E2E Project duration stats'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}