chore: import upstream snapshot with attribution

.github/workflows/security-scan.yml

@@ -0,0 +1,427 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: security-scan
+on:
+  schedule:
+    - cron: "0 0 */2 * *"
+  workflow_dispatch:
+    inputs:
+      SEND_SLACK_NOTIFICATION:
+        description: "Post results to Slack (uncheck for test runs)"
+        type: boolean
+        required: false
+        default: true
+
+env:
+  # Manual runs may opt out of Slack; schedule (where inputs are unset) always sends.
+  # Explicit empty-check, since `||` would treat the string 'false' inconsistently when the
+  # input is typed `boolean` — only an empty string should fall back to 'true'.
+  SEND_SLACK_NOTIFICATION: ${{ github.event.inputs.SEND_SLACK_NOTIFICATION == '' && 'true' || github.event.inputs.SEND_SLACK_NOTIFICATION }}
+
+jobs:
+  vulnerability-scan:
+    runs-on: ubuntu-latest
+    environment: security-scan
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
+
+      - name: Enable yarn
+        run: corepack enable
+
+      - name: Install UI dependencies
+        working-directory: openmetadata-ui/src/main/resources/ui
+        run: yarn install --frozen-lockfile --ignore-scripts
+
+      - name: Run Retire.js scan
+        id: retire-scan
+        continue-on-error: true
+        working-directory: openmetadata-ui/src/main/resources/ui
+        run: |
+          npx retire@5 \
+            --path node_modules/ \
+            --severity high \
+            --outputformat json \
+            --outputpath retire-report.json
+
+      - name: Verify report was generated
+        working-directory: openmetadata-ui/src/main/resources/ui
+        run: |
+          if [ ! -f retire-report.json ]; then
+            echo '::error::retire-report.json was not generated — retire scan may have crashed'
+            exit 1
+          fi
+
+      - name: Upload Retire.js Report
+        if: success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: retire-js-report
+          path: openmetadata-ui/src/main/resources/ui/retire-report.json
+          retention-days: 30
+
+      - name: Generate Retire.js Slack summary
+        if: always()
+        run: |
+          mkdir -p retire-report
+          python3 scripts/retire_slack_summary.py \
+            openmetadata-ui/src/main/resources/ui/retire-report.json \
+            --counts-file retire-report/_retire_counts.json \
+            --slack-file retire-report/_retire_slack.txt \
+            > /dev/null
+
+      - name: Upload Retire.js Slack artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: retire-slack
+          path: |
+            retire-report/_retire_slack.txt
+            retire-report/_retire_counts.json
+          retention-days: 30
+
+      - name: Publish Retire.js Summary
+        if: success()
+        working-directory: openmetadata-ui/src/main/resources/ui
+        run: |
+          python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY
+          import json
+
+          SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
+          SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+          NM = "node_modules/"
+
+          def escape(text):
+              return str(text).replace('|', '\\|').replace('`', "'")
+
+          try:
+              with open("retire-report.json") as f:
+                  data = json.load(f)
+          except FileNotFoundError:
+              print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.")
+              raise SystemExit(0)
+
+          findings = data.get("data", [])
+          libs = {}
+          for item in findings:
+              filepath = item.get("file", "")
+              short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath
+              for result in item.get("results", []):
+                  key = (result.get("component", ""), result.get("version", ""))
+                  if key not in libs:
+                      libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])}
+                  if short not in libs[key]["files"]:
+                      libs[key]["files"].append(short)
+
+          print("## Retire.js Scan Results\n")
+
+          if not libs:
+              print("✅ No vulnerable libraries found.")
+          else:
+              total_vulns = sum(len(v["vulns"]) for v in libs.values())
+              print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n")
+
+              for (component, version), info in sorted(libs.items(), key=lambda x: min(
+                      (SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)):
+                  top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3))
+                  icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪")
+                  print(f"### {icon} {component} {version}\n")
+                  print("| Severity | CVE | Summary |")
+                  print("|---|---|---|")
+                  for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)):
+                      sev = vuln.get("severity", "")
+                      ids = vuln.get("identifiers", {})
+                      cves = ids.get("CVE", [])
+                      summary = ids.get("summary", "").split("\n")[0][:120]
+                      cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—")
+                      print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |")
+                  print("\n**Bundled in:**")
+                  for f in info["files"]:
+                      print(f"- `{f}`")
+                  print()
+          EOF
+
+      - name: Force failure on vulnerabilities found
+        if: steps.retire-scan.outcome == 'failure'
+        run: exit 1
+
+  security-scan:
+    runs-on: ubuntu-latest
+    environment: security-scan
+    env:
+      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      SNYK_ORGANIZATION: ${{ secrets.SNYK_ORGANIZATION_ID }}
+
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: false
+          docker-images: true
+          swap-storage: true
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: "21"
+          distribution: "temurin"
+
+      - name: Install Ubuntu dependencies
+        run: |
+          # stop relying on apt cache of GitHub runners
+          sudo apt-get update
+          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
+          librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev
+
+      # Install and Authenticate to Snyk
+      - name: Install Snyk & Authenticate
+        run: |
+          sudo make install_antlr_cli
+          sudo npm install -g snyk
+          snyk auth ${SNYK_TOKEN}
+          snyk config set org=${SNYK_ORGANIZATION}
+
+      - name: Install Python dependencies
+        run: |
+          python3 -m venv env
+          source env/bin/activate
+          make install_all install_apis
+
+      - name: Maven build
+        id: maven-build
+        continue-on-error: true
+        run: mvn -DskipTests clean install
+
+      - name: Run Scan
+        id: security-report
+        if: steps.maven-build.outcome == 'success'
+        continue-on-error: true
+        run: |
+          source env/bin/activate
+          rm -rf security-report
+          mkdir -p security-report
+          # Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion.
+          make snyk-ingestion-report || true
+          make snyk-ingestion-base-slim-report || true
+          make snyk-airflow-apis-report || true
+          make snyk-server-report || true
+          make snyk-ui-report || true
+
+      - name: Publish Snyk Summary
+        id: snyk-summary
+        if: always() && steps.maven-build.outcome == 'success'
+        run: |
+          python3 scripts/snyk_summary.py security-report \
+            --counts-file security-report/_counts.json \
+            --slack-file security-report/_slack.txt \
+            >> $GITHUB_STEP_SUMMARY
+          # Expose counts as step output for downstream gating.
+          counts=$(cat security-report/_counts.json)
+          echo "counts=$counts" >> $GITHUB_OUTPUT
+          high=$(jq '.high + .critical' security-report/_counts.json)
+          echo "high_critical=$high" >> $GITHUB_OUTPUT
+
+      - name: Fail on high/critical Snyk findings
+        if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0'
+        run: |
+          echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)"
+          exit 1
+
+      - name: Generate Snyk HTML/PDF
+        if: always() && steps.maven-build.outcome == 'success'
+        run: |
+          # Back up JSONs because html_to_pdf.py deletes them after PDF conversion.
+          mkdir -p /tmp/snyk-json-backup
+          cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true
+          make export-snyk-pdf-report || true
+          # Restore JSONs alongside generated PDFs/HTMLs.
+          cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true
+
+      - name: Upload Snyk Reports
+        if: always() && steps.maven-build.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: security-report
+          path: security-report
+          retention-days: 30
+
+      - name: Force failure
+        if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success'
+        run: |
+          exit 1
+
+  notify:
+    runs-on: ubuntu-latest
+    environment: security-scan
+    needs: [vulnerability-scan, security-scan]
+    if: always()
+    steps:
+      - name: Download Snyk artifact
+        if: needs.security-scan.result != 'skipped'
+        uses: actions/download-artifact@v4
+        with:
+          name: security-report
+          path: security-report
+        continue-on-error: true
+
+      - name: Download Retire.js Slack artifact
+        if: needs.vulnerability-scan.result != 'skipped'
+        uses: actions/download-artifact@v4
+        with:
+          name: retire-slack
+          path: retire-report
+        continue-on-error: true
+
+      - name: Build Slack header payload
+        id: build-header
+        env:
+          RETIRE_RESULT: ${{ needs.vulnerability-scan.result }}
+          SNYK_RESULT: ${{ needs.security-scan.result }}
+          REF_NAME: ${{ github.ref_name }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          python3 - <<'PY' > header.json
+          import json, os, pathlib
+
+          retire = os.environ["RETIRE_RESULT"]
+          snyk = os.environ["SNYK_RESULT"]
+          ref_name = os.environ["REF_NAME"]
+          run_url = os.environ["RUN_URL"]
+
+          def status_icon(s):
+              return {"success": "✅", "cancelled": "⚠️ (cancelled)", "skipped": "⚠️ (skipped)"}.get(s, "❌")
+
+          def load_counts(path):
+              p = pathlib.Path(path)
+              if not p.exists():
+                  return None
+              try:
+                  return json.loads(p.read_text())
+              except Exception:
+                  return None
+
+          def totals_line(counts):
+              if not counts:
+                  return ""
+              return (
+                  f"\n🚨 {counts.get('critical', 0)} critical  ·  "
+                  f"🔴 {counts.get('high', 0)} high  ·  "
+                  f"🟠 {counts.get('medium', 0)} medium  ·  "
+                  f"🟡 {counts.get('low', 0)} low"
+              )
+
+          retire_counts = load_counts("retire-report/_retire_counts.json")
+          snyk_counts = load_counts("security-report/_counts.json")
+
+          if retire == "success" and snyk == "success":
+              overall = "🟢"
+          elif retire == "failure" or snyk == "failure":
+              overall = "🚨"
+          else:
+              overall = "⚠️"
+
+          header = (
+              f"{overall}  *Security scan* — *OpenMetadata Repo*\n"
+              f"_branch_ `{ref_name}`  ·  <{run_url}|Open run details>\n"
+              f"• Vulnerability scan (Retire.js): {status_icon(retire)}"
+              f"{totals_line(retire_counts)}\n"
+              f"• Security scan (Snyk): {status_icon(snyk)}"
+              f"{totals_line(snyk_counts)}"
+          )
+          fallback = f"{overall} Security scan — OpenMetadata Repo on {ref_name}. {run_url}"
+          payload = {
+              "text": fallback,
+              "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": header}}],
+          }
+          print(json.dumps(payload))
+          PY
+
+      - name: Send Slack — header
+        id: send-header
+        if: env.SEND_SLACK_NOTIFICATION == 'true'
+        uses: slackapi/slack-github-action@v1.27.1
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
+          payload-file-path: header.json
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+      - name: Build Slack thread payload
+        id: build-thread
+        if: always() && steps.send-header.outputs.ts != ''
+        env:
+          THREAD_TS: ${{ steps.send-header.outputs.ts }}
+        run: |
+          python3 - <<'PY' > thread.json
+          import json, os, pathlib
+
+          thread_ts = os.environ["THREAD_TS"]
+
+          def read_sections(path):
+              p = pathlib.Path(path)
+              if not p.exists():
+                  return []
+              text = p.read_text().strip()
+              return [s.strip() for s in text.split("\n\n") if s.strip()]
+
+          retire_sections = read_sections("retire-report/_retire_slack.txt")
+          snyk_sections = read_sections("security-report/_slack.txt")
+
+          blocks = []
+          MAX_TEXT = 2850
+
+          def push(section):
+              text = section if len(section) <= MAX_TEXT else section[:MAX_TEXT].rstrip() + "\n_…truncated, see Job Summary_"
+              blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})
+
+          for section in retire_sections[:24]:
+              push(section)
+          if retire_sections and snyk_sections:
+              blocks.append({"type": "divider"})
+          for section in snyk_sections[:23]:
+              push(section)
+
+          if not blocks:
+              push("_No detailed scan output was produced._")
+
+          payload = {
+              "thread_ts": thread_ts,
+              "text": "Security scan details (thread reply)",
+              "blocks": blocks,
+          }
+          print(json.dumps(payload))
+          PY
+
+      - name: Send Slack — thread reply
+        if: always() && env.SEND_SLACK_NOTIFICATION == 'true' && steps.send-header.outputs.ts != '' && steps.build-thread.outcome == 'success'
+        uses: slackapi/slack-github-action@v1.27.1
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
+          payload-file-path: thread.json
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}