chore: import upstream snapshot with attribution

scripts/check_prerequisites.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+set -eu
+
+set +e
+declare -A test_map
+res=$?
+if [[ $res -ne 0 ]]; then
+  echo "✗ ERROR: declare -A is not supported. Do you have bash version 4.0 or higher installed?"
+  exit 2
+fi
+set -e
+
+
+declare -A python
+python["name"]="Python"
+python["version_command"]="python --version 2>&1 | awk '{print \$2}'"
+python["required_version"]="3.11 3.12 3.13"
+
+declare -A docker
+docker["name"]="Docker"
+docker["version_command"]="docker --version | awk '{print \$3}' | sed 's/,//'"
+docker["required_version"]="20 21 22 23 24 25 26 27 28 29"
+
+declare -A maven
+maven["name"]="Maven"
+maven["version_command"]="mvn --version | head -n1 | awk '{print \$3}'"
+maven["required_version"]="3.6 3.7 3.8 3.9"
+
+declare -A java
+java["name"]="Java"
+java["version_command"]="java -version 2>&1 | awk -F'\"' '/version/ {print \$2}'"
+java["required_version"]="21"
+
+declare -A jq
+jq["name"]="jq"
+jq["version_command"]="jq --version | awk -F- '{print \$2}'"
+jq["required_version"]="any"
+
+declare -A node
+node["name"]="Node"
+node["version_command"]="node --version"
+node["required_version"]="22"
+
+declare -A yarn
+yarn["name"]="Yarn"
+yarn["version_command"]="yarn --version"
+yarn["required_version"]="1.22 1.23 1.24"
+
+declare -A antlr
+antlr["name"]="ANTLR"
+antlr["version_command"]="antlr4 | head -n1 | awk 'NF>1{print \$NF}'"
+antlr["required_version"]="4.9"
+
+
+code=0
+
+function print_error() {
+    >&2 echo "✗ ERROR: $1"
+}
+
+check_command_existence() {
+    which "$1" >/dev/null 2>&1
+    res=$?
+    if [[ $res -ne 0 ]]; then
+      print_error "$command is not installed."
+      code=2
+    fi
+    echo $res
+}
+
+check_version() {
+    local tool_name=$1
+    local current=$2
+    local required=$3
+    IFS=' ' read -r -a required_versions <<< "$required"
+    if [[ "$required" == "any" ]]; then
+        echo "✓ $tool_name version $current is supported."
+        return
+    fi
+    for v in "${required_versions[@]}"; do
+        if [[ "$current" =~ $v.* ]]; then
+            echo "✓ $tool_name version $version is supported."
+            return
+        fi
+    done
+    print_error "$tool_name version $version is not supported. Supported versions are: $required"
+    code=1
+}
+
+declare -n dependency
+for dependency in python docker java maven jq node yarn antlr; do
+    command=$(echo "${dependency["version_command"]}" | awk '{print $1}')
+    if [[ $(check_command_existence "$command") -ne 0 ]]; then
+        continue
+    fi
+    tool_name=${dependency["name"]}
+    version=$(eval ${dependency["version_command"]})
+    required_version=${dependency["required_version"]}
+    check_version $tool_name "$version" "$required_version"
+done
+if [[ $code -eq 0 ]]; then
+    echo "✓ All prerequisites are met."
+else
+    print_error "Some prerequisites are not met."
+fi
+exit $code

scripts/datamodel_generation.py

@@ -0,0 +1,100 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+This script generates the Python models from the JSON Schemas definition. Additionally, it replaces the `SecretStr`
+pydantic class used for the password fields with the `CustomSecretStr` pydantic class which retrieves the secrets
+from a configured secrets' manager.
+"""
+import datamodel_code_generator.model.pydantic
+from datamodel_code_generator.imports import Import
+import os
+import re
+
+
+datamodel_code_generator.model.pydantic.types.IMPORT_SECRET_STR = Import.from_full_path(
+    "metadata.ingestion.models.custom_pydantic.CustomSecretStr"
+)
+
+from datamodel_code_generator.__main__ import main
+
+current_directory = os.getcwd()
+ingestion_path = "./" if current_directory.endswith("/ingestion") else "ingestion/"
+directory_root = "../" if current_directory.endswith("/ingestion") else "./"
+
+UTF_8 = "UTF-8"
+UNICODE_REGEX_REPLACEMENT_FILE_PATHS = [
+    f"{ingestion_path}src/metadata/generated/schema/entity/classification/tag.py",
+    f"{ingestion_path}src/metadata/generated/schema/entity/events/webhook.py",
+    f"{ingestion_path}src/metadata/generated/schema/entity/teams/user.py",
+    f"{ingestion_path}src/metadata/generated/schema/entity/type.py",
+    f"{ingestion_path}src/metadata/generated/schema/type/basic.py",
+]
+
+args = f"--input {directory_root}openmetadata-spec/src/main/resources/json/schema --output-model-type pydantic_v2.BaseModel --use-annotated --base-class metadata.ingestion.models.custom_pydantic.BaseModel --input-file-type jsonschema --output {ingestion_path}src/metadata/generated/schema --set-default-enum-member".split(" ")
+
+main(args)
+
+for file_path in UNICODE_REGEX_REPLACEMENT_FILE_PATHS:
+    with open(file_path, "r", encoding=UTF_8) as file_:
+        content = file_.read()
+        # Python now requires to move the global flags at the very start of the expression
+        content = content.replace("(?U)", "(?u)")
+    with open(file_path, "w", encoding=UTF_8) as file_:
+        file_.write(content)
+
+# Until https://github.com/koxudaxi/datamodel-code-generator/issues/1895
+# TODO: This has been merged but `Union` is still not there. We'll need to validate
+MISSING_IMPORTS = [f"{ingestion_path}src/metadata/generated/schema/entity/applications/app.py",]
+WRITE_AFTER = "from __future__ import annotations"
+
+for file_path in MISSING_IMPORTS:
+    with open(file_path, "r", encoding=UTF_8) as file_:
+        lines = file_.readlines()
+    with open(file_path, "w", encoding=UTF_8) as file_:
+        for line in lines:
+            file_.write(line)
+            if line.strip() == WRITE_AFTER:
+                file_.write("from typing import Union  # custom generate import\n\n")
+
+
+# unsupported rust regex pattern for pydantic v2
+# https://docs.pydantic.dev/2.7/api/config/#pydantic.config.ConfigDict.regex_engine
+# We'll remove validation from the client and let it fail on the server, rather than on the model generation
+UNSUPPORTED_REGEX_PATTERN_FILE_PATHS = [
+    f"{ingestion_path}src/metadata/generated/schema/type/basic.py",
+    f"{ingestion_path}src/metadata/generated/schema/entity/data/searchIndex.py",
+    f"{ingestion_path}src/metadata/generated/schema/entity/data/table.py",
+]
+
+for file_path in UNSUPPORTED_REGEX_PATTERN_FILE_PATHS:
+    with open(file_path, "r", encoding=UTF_8) as file_:
+        content = file_.read()
+        content = content.replace("pattern='^((?!::).)*$',", "")
+    with open(file_path, "w", encoding=UTF_8) as file_:
+        file_.write(content)
+
+# Until https://github.com/koxudaxi/datamodel-code-generator/issues/1996
+# Supporting timezone aware datetime is too complex for the profiler
+DATETIME_AWARE_FILE_PATHS = [
+    f"{ingestion_path}src/metadata/generated/schema/type/basic.py",
+]
+
+for file_path in DATETIME_AWARE_FILE_PATHS:
+    with open(file_path, "r", encoding=UTF_8) as file_:
+        content = file_.read()
+        content = content.replace(
+            "from pydantic import AnyUrl, AwareDatetime, ConfigDict, EmailStr, Field, RootModel",
+            "from pydantic import AnyUrl, ConfigDict, EmailStr, Field, RootModel"
+        )
+        content = content.replace("from datetime import date, time", "from datetime import date, time, datetime")
+        content = content.replace("AwareDatetime", "datetime")
+    with open(file_path, "w", encoding=UTF_8) as file_:
+        file_.write(content)

scripts/deploy-pipelines.py

@@ -0,0 +1,263 @@
+"""
+Deploy Ingestion Pipelines Script
+
+This script uses the OpenMetadata client to:
+1. Paginate over ingestion pipelines in groups of 20
+2. Fetch the IDs of those IngestionPipelines  
+3. Send bulk deploy requests to api/v1/services/ingestionPipelines/bulk/deploy
+4. Track responses to monitor deployment success/failure
+"""
+
+import argparse
+import json
+import logging
+import sys
+from typing import Dict, List
+from uuid import UUID
+
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
+    IngestionPipeline,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.pipelineServiceClientResponse import (
+    PipelineServiceClientResponse,
+)
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.ometa.utils import model_str
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class PipelineDeployer:
+    """Class to handle bulk deployment of ingestion pipelines"""
+    
+    def __init__(self, server_url: str, jwt_token: str):
+        """
+        Initialize the PipelineDeployer with OpenMetadata connection
+        
+        Args:
+            server_url: OpenMetadata server URL
+            jwt_token: JWT token for authentication
+        """
+        # Remove trailing slash if present
+        self.server_url = server_url.rstrip('/')
+        
+        # Configure OpenMetadata connection
+        server_config = OpenMetadataConnection(
+            hostPort=self.server_url,
+            authProvider="openmetadata",
+            securityConfig=OpenMetadataJWTClientConfig(jwtToken=jwt_token),
+        )
+        
+        self.metadata = OpenMetadata(server_config)
+        logger.info(f"Connected to OpenMetadata server: {server_url}")
+    
+    def bulk_deploy_pipelines(self, pipeline_ids: List[str]) -> List[PipelineServiceClientResponse]:
+        """
+        Send bulk deploy request to OpenMetadata API
+        
+        Args:
+            pipeline_ids: List of pipeline UUIDs to deploy
+            
+        Returns:
+            List of PipelineServiceClientResponse objects
+        """
+        if not pipeline_ids:
+            logger.warning("No pipeline IDs provided for deployment")
+            return []
+        
+        logger.info(f"Deploying {len(pipeline_ids)} pipelines...")
+        
+        try:
+            # Make POST request to bulk deploy endpoint
+            response = self.metadata.client.post(
+                "/services/ingestionPipelines/bulk/deploy",
+                data=json.dumps(pipeline_ids),
+            )
+            
+            if response:
+                # Parse response data into Pydantic models
+                parsed_responses = [
+                    PipelineServiceClientResponse.model_validate(item)
+                    for item in response
+                ]
+                logger.info(f"Bulk deploy request completed with {len(parsed_responses)} responses")
+                return parsed_responses
+            else:
+                logger.error("No response from bulk deploy API")
+                return []
+                
+        except Exception as e:
+            logger.error(f"Error during bulk deployment: {e}")
+            raise
+    
+    def analyze_deployment_results(
+        self, 
+        responses: List[PipelineServiceClientResponse],
+        pipeline_ids: List[UUID]
+    ) -> Dict[str, int]:
+        """
+        Analyze deployment responses to track success/failure
+        
+        Args:
+            responses: List of PipelineServiceClientResponse objects
+            pipeline_ids: List of pipeline IDs that were deployed (for correlation)
+            
+        Returns:
+            Dictionary with deployment statistics
+        """
+        stats = {
+            "total": len(responses),
+            "success": 0,
+            "failed": 0,
+            "unknown": 0
+        }
+        
+        success_codes = {200, 201}  # HTTP success codes
+        
+        # Correlate responses with pipeline IDs (assuming same order)
+        for i, response in enumerate(responses):
+            pipeline_id = pipeline_ids[i] if i < len(pipeline_ids) else "unknown"
+            code = response.code
+            reason = response.reason or ""
+            platform = response.platform
+            version = response.version or "unknown"
+            
+            if code in success_codes:
+                stats["success"] += 1
+                logger.info(f"✓ Pipeline {pipeline_id} deployed successfully on {platform} v{version} (code: {code})")
+            elif code:
+                stats["failed"] += 1
+                logger.warning(f"✗ Pipeline {pipeline_id} deployment failed on {platform} v{version} (code: {code}): {reason}")
+            else:
+                stats["unknown"] += 1
+                logger.warning(f"? Pipeline {pipeline_id} unknown deployment status on {platform}: {response.model_dump()}")
+        
+        return stats
+    
+    def deploy_all_pipelines(self, batch_size: int = 20) -> Dict[str, int]:
+        """
+        Main method to deploy all ingestion pipelines
+        
+        Args:
+            batch_size: Size of batches for pagination and deployment
+            
+        Returns:
+            Dictionary with overall deployment statistics
+        """
+        try:
+            # Fetch all pipelines
+            pipelines = list(self.metadata.list_all_entities(entity=IngestionPipeline, skip_on_failure=True))
+
+            if not pipelines:
+                logger.warning("No pipelines found to deploy")
+                return {"total": 0, "success": 0, "failed": 0, "unknown": 0}
+
+            logger.info("Found %d pipelines to deploy", len(pipelines))
+            
+            # Extract pipeline IDs
+            pipeline_ids = [model_str(pipeline.id) for pipeline in pipelines]
+            
+            if not pipeline_ids:
+                logger.error("No valid pipeline IDs found")
+                return {"total": 0, "success": 0, "failed": 0, "unknown": 0}
+            
+            # Deploy pipelines in batches
+            all_responses = []
+            total_batches = (len(pipeline_ids) + batch_size - 1) // batch_size
+            
+            for i in range(0, len(pipeline_ids), batch_size):
+                batch = pipeline_ids[i:i+batch_size]
+                batch_num = i//batch_size + 1
+                remaining_batches = total_batches - batch_num
+                
+                logger.info(f"Deploying batch {batch_num}/{total_batches} with {len(batch)} pipelines... ({remaining_batches} chunks remaining)")
+                
+                batch_responses = self.bulk_deploy_pipelines(batch)
+                all_responses.extend(batch_responses)
+                
+                # Log parsed responses for this batch
+                logger.info(f"Batch {batch_num} responses:")
+                for j, response in enumerate(batch_responses):
+                    pipeline_id = batch[j] if j < len(batch) else "unknown"
+                    logger.info(f"  Pipeline {pipeline_id}: code={response.code}, platform={response.platform}, reason={response.reason or 'N/A'}")
+                
+                logger.info(f"Batch {batch_num} completed. Progress: {batch_num}/{total_batches} batches processed")
+            
+            # Analyze results - correlate responses with the pipeline IDs we sent
+            stats = self.analyze_deployment_results(all_responses, pipeline_ids)
+            
+            logger.info("=== Deployment Summary ===")
+            logger.info(f"Total pipelines: {stats['total']}")
+            logger.info(f"Successfully deployed: {stats['success']}")
+            logger.info(f"Failed deployments: {stats['failed']}")
+            logger.info(f"Unknown status: {stats['unknown']}")
+            
+            return stats
+            
+        except Exception as e:
+            logger.error(f"Deployment process failed: {e}")
+            raise
+
+
+def main():
+    """Main function to run the pipeline deployment script"""
+    parser = argparse.ArgumentParser(
+        description="Deploy all ingestion pipelines using OpenMetadata bulk deploy API"
+    )
+    parser.add_argument(
+        "--server-url",
+        required=True,
+        help="OpenMetadata server URL (e.g., http://localhost:8585/api/)"
+    )
+    parser.add_argument(
+        "--jwt-token", 
+        required=True,
+        help="JWT token for authentication"
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=20,
+        help="Batch size for pagination and deployment (default: 20)"
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Enable verbose logging"
+    )
+    
+    args = parser.parse_args()
+    logging.getLogger().setLevel(logging.INFO)
+    
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+    
+    try:
+        deployer = PipelineDeployer(args.server_url, args.jwt_token)
+        stats = deployer.deploy_all_pipelines(batch_size=args.batch_size)
+        
+        # Exit with error code if any deployments failed
+        if stats["failed"] > 0:
+            sys.exit(1)
+        else:
+            sys.exit(0)
+            
+    except Exception as e:
+        logger.error(f"Script failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/format-code.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+mvn spotless:apply

scripts/generate-rdf-models.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Script to generate RDF models from JSON schemas
+# This creates JSON-LD contexts and OWL ontology from OpenMetadata schemas
+#
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$SCRIPT_DIR/.."
+
+SCHEMA_PATH="$ROOT_DIR/openmetadata-spec/src/main/resources/json/schema"
+OUTPUT_PATH="$ROOT_DIR/openmetadata-spec/src/main/resources/rdf"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo -e "${GREEN}OpenMetadata RDF Model Generator${NC}"
+echo "=================================="
+
+# Check if schema directory exists
+if [ ! -d "$SCHEMA_PATH" ]; then
+    echo -e "${RED}Error: Schema directory not found at $SCHEMA_PATH${NC}"
+    exit 1
+fi
+
+# Create output directory if it doesn't exist
+mkdir -p "$OUTPUT_PATH"
+
+# Compile the generator if needed
+echo -e "${YELLOW}Compiling RDF generator...${NC}"
+cd "$ROOT_DIR"
+mvn compile -pl openmetadata-service -DskipTests
+
+# Run the generator
+echo -e "${YELLOW}Generating RDF models...${NC}"
+mvn exec:java \
+    -pl openmetadata-service \
+    -Dexec.mainClass="org.openmetadata.service.rdf.generator.RdfModelGenerator" \
+    -Dexec.args="$SCHEMA_PATH $OUTPUT_PATH"
+
+if [ $? -eq 0 ]; then
+    echo -e "${GREEN}RDF model generation completed successfully!${NC}"
+    echo ""
+    echo "Generated files:"
+    echo "- JSON-LD contexts: $OUTPUT_PATH/contexts/"
+    echo "- OWL ontology: $OUTPUT_PATH/ontology/openmetadata-generated.ttl"
+else
+    echo -e "${RED}RDF model generation failed!${NC}"
+    exit 1
+fi

scripts/html_to_pdf.py

@@ -0,0 +1,47 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+This script generates a PDF from all HTML files contained in a INPUT_FOLDER into a OUTPUT_FOLDER with a PDF_FILE_NAME.
+It removes all the html files during the generation of the PDF file.
+"""
+import glob
+import os
+
+import pdfkit
+from PyPDF2 import PdfMerger
+
+INPUT_FOLDER = "security-report"
+
+OUTPUT_FOLDER = "security-report"
+
+PDF_FILE_NAME = "security-report"
+
+merger = PdfMerger()
+
+for file in glob.glob(f"{INPUT_FOLDER}/*.html"):
+    file_name, _ = os.path.splitext(file)
+    pdf_file = f"{file_name}.pdf"
+    print(f"Generating PDF file '{pdf_file}'")
+    pdfkit.from_file(file, pdf_file)
+    merger.append(pdf_file)
+    try:
+        print(f"Removing file '{file}'")
+        os.remove(file)
+        print(f"Removing file '{file_name}'")
+        os.remove(file_name)
+    except OSError as err:
+        pass
+
+print("Generating PDF report...")
+merger.write(f"{OUTPUT_FOLDER}/{PDF_FILE_NAME}.pdf")
+merger.close()
+print("Process done!")

scripts/ingest_100k_containers.py

@@ -0,0 +1,434 @@
+#!/usr/bin/env python3
+"""
+Generate ~100k Container entities in OpenMetadata for reindex performance testing.
+
+Builds a Storage service plus a tree of Containers (depth 1-10) where every Container
+has tags and a dataModel of 5-10 columns with column-level tags. Mirrors the structure
+of {@code ingest_100k_tables.py} but for the Container entity, which is the path that
+exposed the 580k-record reindex slowness this PR addresses.
+
+Layout:
+  - 5 classifications, 10 tags each = 50 tags
+  - 1 storage service
+  - ~100k containers distributed across depths 1-10 (root-heavy bell curve)
+  - Each container: 1-3 tags + dataModel(5-10 columns, each with 0-2 tags)
+
+Run after the local docker stack is up:
+
+    python scripts/ingest_100k_containers.py \\
+        --server http://localhost:8585/api --token "$OM_JWT_TOKEN"
+
+Use --containers / --workers / --batch-size to tune. Default 100000 / 10 / 50.
+"""
+
+import argparse
+import concurrent.futures
+import random
+import string
+import sys
+import time
+from typing import Dict, List, Tuple
+
+# Force unbuffered output so progress shows up under tee/redirect.
+sys.stdout.reconfigure(line_buffering=True)
+
+# Skip the strict server/client version match. The repo venv may carry an older
+# client SDK against a freshly built 1.13 server (or vice versa) and the resulting
+# VersionMismatchException is the difference between "this script runs" and
+# "this script doesn't". Nothing this script does is version-sensitive — it's just
+# create_or_update on standard entity types.
+import metadata.ingestion.ometa.mixins.server_mixin as _server_mixin  # noqa: E402
+
+_server_mixin.OMetaServerMixin.validate_versions = lambda self: None  # type: ignore[assignment]
+
+from metadata.generated.schema.api.classification.createClassification import (
+    CreateClassificationRequest,
+)
+from metadata.generated.schema.api.classification.createTag import CreateTagRequest
+from metadata.generated.schema.api.data.createContainer import CreateContainerRequest
+from metadata.generated.schema.api.services.createStorageService import (
+    CreateStorageServiceRequest,
+)
+from metadata.generated.schema.entity.classification.classification import (
+    Classification,
+)
+from metadata.generated.schema.entity.classification.tag import Tag
+from metadata.generated.schema.entity.data.container import (
+    Container,
+    ContainerDataModel,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+from metadata.generated.schema.entity.services.storageService import (
+    StorageService,
+    StorageServiceType,
+)
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.tagLabel import (
+    LabelType,
+    State,
+    TagLabel,
+    TagSource,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+
+
+# Distribution of containers across depths 1..10. Root-heavy bell curve so the tree
+# looks realistic (lots of mid-depth nodes, fewer at the extremes). Sum = 100000.
+DEPTH_DISTRIBUTION = {
+    1: 1000,
+    2: 5000,
+    3: 12000,
+    4: 20000,
+    5: 22000,
+    6: 18000,
+    7: 12000,
+    8: 6000,
+    9: 3000,
+    10: 1000,
+}
+
+
+def scale_depth_counts(distribution: Dict[int, int], target: int) -> Dict[int, int]:
+    """Scale a depth distribution to a requested total, preserving the exact total.
+
+    Floor each weighted share, then hand out the leftover one-by-one to the depths with the
+    largest fractional remainder. Honors --containers exactly (10 → 10 containers, not 10 *
+    len(distribution)) and produces a stable distribution for the same input. Depth 1 must
+    always have at least one container so subsequent depths have a parent to attach to; the
+    floor would otherwise zero it out at very small targets.
+    """
+    if target <= 0:
+        return {d: 0 for d in distribution}
+    base_total = sum(distribution.values())
+    raw = {d: c * target / base_total for d, c in distribution.items()}
+    floored = {d: int(v) for d, v in raw.items()}
+    # Reserve one slot at depth 1 so the tree has a root layer.
+    if floored.get(1, 0) == 0 and target >= 1:
+        floored[1] = 1
+    deficit = target - sum(floored.values())
+    if deficit > 0:
+        remainders = sorted(
+            ((d, raw[d] - floored[d]) for d in raw),
+            key=lambda x: (-x[1], x[0]),
+        )
+        for d, _ in remainders[:deficit]:
+            floored[d] += 1
+    elif deficit < 0:
+        # Overshoot from the depth=1 reservation: take back from depths with the largest counts.
+        excess = -deficit
+        biggest = sorted(floored.items(), key=lambda kv: (-kv[1], kv[0]))
+        for d, _ in biggest:
+            if excess == 0:
+                break
+            if d == 1:
+                continue
+            take = min(excess, floored[d])
+            floored[d] -= take
+            excess -= take
+    return floored
+
+CLASSIFICATIONS = [
+    ("PIIClass", ["Email", "Phone", "Address", "SSN", "DOB",
+                   "CreditCard", "BankAccount", "MedicalRecord", "Biometric", "Geolocation"]),
+    ("Sensitivity", ["Public", "Internal", "Confidential", "Restricted", "Secret",
+                     "TopSecret", "PartnerOnly", "EmployeeOnly", "ExecutiveOnly", "Auditor"]),
+    ("Quality", ["Gold", "Silver", "Bronze", "Raw", "Curated",
+                 "Validated", "Deprecated", "Experimental", "Production", "Staging"]),
+    ("Domain", ["Sales", "Marketing", "Engineering", "Finance", "HR",
+                "Operations", "Legal", "Support", "Product", "Research"]),
+    ("Compliance", ["GDPR", "HIPAA", "SOX", "CCPA", "PCI",
+                    "SOC2", "ISO27001", "NIST", "FedRAMP", "FERPA"]),
+]
+
+DATA_TYPES = [
+    DataType.BIGINT, DataType.INT, DataType.VARCHAR, DataType.TEXT,
+    DataType.TIMESTAMP, DataType.DATE, DataType.BOOLEAN, DataType.JSON,
+    DataType.DOUBLE, DataType.DECIMAL,
+]
+
+# Cached after setup; workers pick from this list to attach tags to containers/columns.
+ALL_TAG_FQNS: List[str] = []
+
+
+def random_word(length: int = 8) -> str:
+    return "".join(random.choices(string.ascii_lowercase, k=length))
+
+
+def make_metadata_client(server_url: str, token: str) -> OpenMetadata:
+    return OpenMetadata(
+        OpenMetadataConnection(
+            hostPort=server_url,
+            securityConfig=OpenMetadataJWTClientConfig(jwtToken=token),
+        )
+    )
+
+
+def ensure_classifications_and_tags(metadata: OpenMetadata) -> List[str]:
+    """Idempotent: create classifications and tags if missing, return all tag FQNs."""
+    fqns: List[str] = []
+    for class_name, tag_names in CLASSIFICATIONS:
+        if not metadata.get_by_name(entity=Classification, fqn=class_name):
+            metadata.create_or_update(
+                CreateClassificationRequest(
+                    name=class_name,
+                    description=f"Reindex perf test classification: {class_name}",
+                )
+            )
+            print(f"Created classification: {class_name}", flush=True)
+        for tag_name in tag_names:
+            tag_fqn = f"{class_name}.{tag_name}"
+            if not metadata.get_by_name(entity=Tag, fqn=tag_fqn):
+                metadata.create_or_update(
+                    CreateTagRequest(
+                        name=tag_name,
+                        classification=class_name,
+                        description=f"Reindex perf test tag: {tag_fqn}",
+                    )
+                )
+            fqns.append(tag_fqn)
+    print(f"Total tags available: {len(fqns)}", flush=True)
+    return fqns
+
+
+def ensure_storage_service(metadata: OpenMetadata, name: str) -> str:
+    """Returns the service name (used as FQN for sub-entities)."""
+    if metadata.get_by_name(entity=StorageService, fqn=name):
+        print(f"Using existing storage service: {name}", flush=True)
+        return name
+    # CustomStorage with empty connection — sufficient for entity creation; reindex
+    # only cares about the entity rows, not whether the connection works.
+    from metadata.generated.schema.entity.services.connections.storage.customStorageConnection import (
+        CustomStorageConnection,
+        CustomStorageType,
+    )
+    from metadata.generated.schema.entity.services.storageService import (
+        StorageConnection,
+    )
+
+    metadata.create_or_update(
+        CreateStorageServiceRequest(
+            name=name,
+            serviceType=StorageServiceType.CustomStorage,
+            connection=StorageConnection(
+                config=CustomStorageConnection(
+                    type=CustomStorageType.CustomStorage,
+                    sourcePythonClass="x.y.z",
+                )
+            ),
+        )
+    )
+    print(f"Created storage service: {name}", flush=True)
+    return name
+
+
+def random_tag_labels(count_min: int, count_max: int) -> List[TagLabel]:
+    n = random.randint(count_min, count_max)
+    if n == 0 or not ALL_TAG_FQNS:
+        return []
+    chosen = random.sample(ALL_TAG_FQNS, min(n, len(ALL_TAG_FQNS)))
+    return [
+        TagLabel(
+            tagFQN=fqn,
+            source=TagSource.Classification,
+            labelType=LabelType.Manual,
+            state=State.Confirmed,
+        )
+        for fqn in chosen
+    ]
+
+
+def random_columns() -> List[Column]:
+    n = random.randint(5, 10)
+    cols: List[Column] = []
+    for i in range(n):
+        dt = random.choice(DATA_TYPES)
+        col = Column(
+            name=f"col_{i}_{random_word(4)}",
+            dataType=dt,
+            tags=random_tag_labels(0, 2),
+        )
+        if dt == DataType.VARCHAR:
+            col.dataLength = 255
+        elif dt == DataType.DECIMAL:
+            col.dataLength = 10
+        cols.append(col)
+    return cols
+
+
+def build_parent_ref(parent_id: str, parent_fqn: str) -> EntityReference:
+    return EntityReference(id=parent_id, type="container", fullyQualifiedName=parent_fqn)
+
+
+def create_depth_layer(
+    server_url: str,
+    token: str,
+    service_name: str,
+    depth: int,
+    count: int,
+    parents: List[Tuple[str, str]],  # list of (id, fqn) from previous layer
+    workers: int,
+    batch_size: int,
+) -> List[Tuple[str, str]]:
+    """
+    Create {@code count} containers at the given depth, each picking a random parent
+    from {@code parents}. Returns the (id, fqn) of every created container — feeds the
+    next deeper layer.
+    """
+    print(f"\n=== Depth {depth}: creating {count} containers ===", flush=True)
+    start = time.time()
+    created: List[Tuple[str, str]] = []
+    failed = 0
+
+    def make_batch(batch_start: int, batch_count: int) -> List[Tuple[str, str]]:
+        worker_metadata = make_metadata_client(server_url, token)
+        batch_results: List[Tuple[str, str]] = []
+        for i in range(batch_start, batch_start + batch_count):
+            name = f"c_d{depth}_{i:06d}_{random_word(4)}"
+            try:
+                if depth == 1:
+                    # Depth-1 containers attach directly to the service (no parent ref).
+                    req = CreateContainerRequest(
+                        name=name,
+                        service=service_name,
+                        description=f"Reindex perf test container {name}",
+                        tags=random_tag_labels(1, 3),
+                        dataModel=ContainerDataModel(
+                            isPartitioned=False, columns=random_columns()
+                        ),
+                    )
+                else:
+                    parent_id, parent_fqn = random.choice(parents)
+                    req = CreateContainerRequest(
+                        name=name,
+                        service=service_name,
+                        parent=build_parent_ref(parent_id, parent_fqn),
+                        description=f"Reindex perf test container {name}",
+                        tags=random_tag_labels(1, 3),
+                        dataModel=ContainerDataModel(
+                            isPartitioned=False, columns=random_columns()
+                        ),
+                    )
+                obj = worker_metadata.create_or_update(req)
+                fqn = (
+                    obj.fullyQualifiedName.root
+                    if hasattr(obj.fullyQualifiedName, "root")
+                    else str(obj.fullyQualifiedName)
+                )
+                batch_results.append((str(obj.id.root) if hasattr(obj.id, "root") else str(obj.id), fqn))
+            except Exception as e:  # noqa: BLE001
+                # Don't let one bad row sink the whole layer.
+                print(f"  [d{depth} #{i}] {name}: {e}", flush=True)
+        return batch_results
+
+    batches = [
+        (start_idx, min(batch_size, count - start_idx))
+        for start_idx in range(0, count, batch_size)
+    ]
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(make_batch, s, c): (s, c) for s, c in batches}
+        completed_batches = 0
+        for fut in concurrent.futures.as_completed(futures):
+            try:
+                results = fut.result()
+                created.extend(results)
+                # batch_size minus actual results = failures we logged
+                expected = futures[fut][1]
+                failed += expected - len(results)
+            except Exception as e:  # noqa: BLE001
+                print(f"  Batch {futures[fut]} hard-failed: {e}", flush=True)
+                failed += futures[fut][1]
+            completed_batches += 1
+            if completed_batches % 20 == 0 or completed_batches == len(batches):
+                elapsed = time.time() - start
+                rate = len(created) / elapsed if elapsed > 0 else 0
+                print(
+                    f"  depth={depth} progress: {len(created)}/{count} "
+                    f"({100 * len(created) / count:.1f}%) - {rate:.1f}/s",
+                    flush=True,
+                )
+
+    elapsed = time.time() - start
+    rate = len(created) / elapsed if elapsed > 0 else 0
+    print(
+        f"=== Depth {depth} done: {len(created)} created, {failed} failed, "
+        f"{elapsed:.1f}s, {rate:.1f}/s ===",
+        flush=True,
+    )
+    return created
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--server", default="http://localhost:8585/api")
+    parser.add_argument("--token", required=True, help="OpenMetadata JWT")
+    parser.add_argument("--service", default="reindex_perf_storage")
+    parser.add_argument("--containers", type=int, default=100000)
+    parser.add_argument("--workers", type=int, default=10)
+    parser.add_argument("--batch-size", type=int, default=50)
+    parser.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args()
+
+    # The seeded RNG is consumed concurrently by worker threads, so seed determinism is
+    # best-effort: the *layout* (depth distribution, target totals) is reproducible, the
+    # specific tag/name/column choices interleave non-deterministically across threads. Good
+    # enough for perf reproducibility; if you need exact byte-for-byte output, drop
+    # --workers to 1.
+    random.seed(args.seed)
+    print(f"Server: {args.server}", flush=True)
+    print(f"Target: {args.containers} containers, {args.workers} workers, batch {args.batch_size}", flush=True)
+    print("-" * 60, flush=True)
+
+    metadata = make_metadata_client(args.server, args.token)
+
+    global ALL_TAG_FQNS
+    ALL_TAG_FQNS = ensure_classifications_and_tags(metadata)
+    service_name = ensure_storage_service(metadata, args.service)
+
+    # Scale the depth distribution to honor --containers exactly. Floor-and-distribute the
+    # remainder largest-first so the totals add up to the requested target without overshoot
+    # (the previous max(1, round(...)) form clamped tiny inputs to >= len(distribution)).
+    depth_counts = scale_depth_counts(DEPTH_DISTRIBUTION, args.containers)
+    print(f"Depth counts: {depth_counts}", flush=True)
+    print(f"Sum: {sum(depth_counts.values())}", flush=True)
+
+    overall_start = time.time()
+    parents: List[Tuple[str, str]] = []
+    for depth in sorted(depth_counts.keys()):
+        count = depth_counts[depth]
+        if depth > 1 and not parents:
+            print(f"No parents available at depth {depth}; aborting.", flush=True)
+            break
+        layer = create_depth_layer(
+            server_url=args.server,
+            token=args.token,
+            service_name=service_name,
+            depth=depth,
+            count=count,
+            parents=parents,
+            workers=args.workers,
+            batch_size=args.batch_size,
+        )
+        # Keep a bounded sample of this depth's containers as parents for the next
+        # depth — picking from a 10k subset is plenty of branching variety and keeps
+        # memory bounded if the user runs at 1M+ scale.
+        parents = layer if len(layer) <= 10000 else random.sample(layer, 10000)
+
+    total_elapsed = time.time() - overall_start
+    print("\n" + "=" * 60, flush=True)
+    print(
+        f"Done: ~{args.containers} containers across "
+        f"{len(depth_counts)} depths in {total_elapsed:.1f}s "
+        f"({args.containers / total_elapsed:.1f}/s overall)",
+        flush=True,
+    )
+    print("=" * 60, flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/ingest_100k_tables.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""
+Script to ingest 100k tables into OpenMetadata for testing distributed indexing.
+"""
+
+import argparse
+import concurrent.futures
+import sys
+import time
+from datetime import datetime
+
+# Force unbuffered output
+sys.stdout.reconfigure(line_buffering=True)
+
+from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
+from metadata.generated.schema.api.data.createDatabaseSchema import (
+    CreateDatabaseSchemaRequest,
+)
+from metadata.generated.schema.api.data.createTable import CreateTableRequest
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceRequest,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.entity.services.connections.database.common.basicAuth import (
+    BasicAuth,
+)
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+)
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseConnection,
+    DatabaseService,
+    DatabaseServiceType,
+)
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+
+
+def create_metadata_client(server_url: str, token: str) -> OpenMetadata:
+    """Create OpenMetadata client."""
+    server_config = OpenMetadataConnection(
+        hostPort=server_url,
+        securityConfig=OpenMetadataJWTClientConfig(jwtToken=token),
+    )
+    return OpenMetadata(server_config)
+
+
+def create_service(metadata: OpenMetadata, service_name: str) -> DatabaseService:
+    """Create or get database service."""
+    # Check if service exists
+    existing = metadata.get_by_name(entity=DatabaseService, fqn=service_name)
+    if existing:
+        print(f"Using existing service: {service_name}")
+        return existing
+
+    # Create new service
+    service = CreateDatabaseServiceRequest(
+        name=service_name,
+        serviceType=DatabaseServiceType.Mysql,
+        connection=DatabaseConnection(
+            config=MysqlConnection(
+                username="test",
+                authType=BasicAuth(password="test"),
+                hostPort="localhost:3306",
+            )
+        ),
+    )
+    created = metadata.create_or_update(service)
+    print(f"Created service: {service_name}")
+    return created
+
+
+def create_database(metadata: OpenMetadata, service_fqn: str, db_name: str):
+    """Create or get database."""
+    fqn = f"{service_fqn}.{db_name}"
+    from metadata.generated.schema.entity.data.database import Database
+
+    existing = metadata.get_by_name(entity=Database, fqn=fqn)
+    if existing:
+        print(f"Using existing database: {fqn}")
+        return existing
+
+    db = CreateDatabaseRequest(name=db_name, service=service_fqn)
+    created = metadata.create_or_update(db)
+    print(f"Created database: {fqn}")
+    return created
+
+
+def create_schema(metadata: OpenMetadata, database_fqn: str, schema_name: str):
+    """Create or get schema."""
+    fqn = f"{database_fqn}.{schema_name}"
+    from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+
+    existing = metadata.get_by_name(entity=DatabaseSchema, fqn=fqn)
+    if existing:
+        print(f"Using existing schema: {fqn}")
+        return existing
+
+    schema = CreateDatabaseSchemaRequest(name=schema_name, database=database_fqn)
+    created = metadata.create_or_update(schema)
+    print(f"Created schema: {fqn}")
+    return created
+
+
+def create_tables_batch(
+    metadata: OpenMetadata, schema_fqn: str, start_idx: int, count: int
+) -> int:
+    """Create a batch of tables."""
+    created_count = 0
+    columns = [
+        Column(name="id", dataType=DataType.BIGINT, description="Primary key"),
+        Column(name="name", dataType=DataType.VARCHAR, dataLength=255),
+        Column(name="description", dataType=DataType.TEXT),
+        Column(name="created_at", dataType=DataType.TIMESTAMP),
+        Column(name="updated_at", dataType=DataType.TIMESTAMP),
+        Column(name="status", dataType=DataType.VARCHAR, dataLength=50),
+        Column(name="metadata", dataType=DataType.JSON),
+    ]
+
+    for i in range(start_idx, start_idx + count):
+        table_name = f"test_table_{i:06d}"
+        try:
+            table = CreateTableRequest(
+                name=table_name,
+                databaseSchema=schema_fqn,
+                columns=columns,
+                description=f"Test table {i} for distributed indexing benchmark",
+            )
+            metadata.create_or_update(table)
+            created_count += 1
+        except Exception as e:
+            print(f"Error creating table {table_name}: {e}")
+
+    return created_count
+
+
+def ingest_tables(
+    server_url: str,
+    token: str,
+    total_tables: int = 100000,
+    batch_size: int = 100,
+    workers: int = 10,
+):
+    """Ingest tables into OpenMetadata."""
+    print(f"Starting ingestion of {total_tables} tables...", flush=True)
+    print(f"Server: {server_url}", flush=True)
+    print(f"Batch size: {batch_size}, Workers: {workers}", flush=True)
+    print("-" * 60, flush=True)
+
+    # Create main client for setup
+    print("Creating metadata client...", flush=True)
+    metadata = create_metadata_client(server_url, token)
+    print("Client created!", flush=True)
+
+    # Create service, database, and schema
+    service_name = "scale_test_service"
+    db_name = "scale_test_db"
+    schema_name = "scale_test_schema"
+
+    service = create_service(metadata, service_name)
+    database = create_database(metadata, service_name, db_name)
+    schema = create_schema(metadata, f"{service_name}.{db_name}", schema_name)
+    schema_fqn = f"{service_name}.{db_name}.{schema_name}"
+
+    print("-" * 60)
+    print(f"Creating {total_tables} tables in {schema_fqn}")
+    print("-" * 60)
+
+    start_time = time.time()
+    total_created = 0
+
+    # Create batches
+    batches = []
+    for start_idx in range(0, total_tables, batch_size):
+        count = min(batch_size, total_tables - start_idx)
+        batches.append((start_idx, count))
+
+    # Process batches with thread pool
+    def process_batch(batch_info):
+        start_idx, count = batch_info
+        # Each worker needs its own client
+        worker_metadata = create_metadata_client(server_url, token)
+        return create_tables_batch(worker_metadata, schema_fqn, start_idx, count)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {executor.submit(process_batch, batch): batch for batch in batches}
+
+        completed = 0
+        for future in concurrent.futures.as_completed(futures):
+            batch = futures[future]
+            try:
+                created = future.result()
+                total_created += created
+                completed += 1
+
+                # Progress update every 10 batches
+                if completed % 10 == 0:
+                    elapsed = time.time() - start_time
+                    rate = total_created / elapsed if elapsed > 0 else 0
+                    print(
+                        f"Progress: {total_created}/{total_tables} tables "
+                        f"({100*total_created/total_tables:.1f}%) - "
+                        f"{rate:.1f} tables/sec"
+                    )
+            except Exception as e:
+                print(f"Batch {batch} failed: {e}")
+
+    elapsed = time.time() - start_time
+    rate = total_created / elapsed if elapsed > 0 else 0
+
+    print("-" * 60)
+    print(f"Ingestion complete!")
+    print(f"Total tables created: {total_created}")
+    print(f"Time elapsed: {elapsed:.1f} seconds")
+    print(f"Average rate: {rate:.1f} tables/sec")
+    print("-" * 60)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Ingest tables into OpenMetadata for scale testing"
+    )
+    parser.add_argument(
+        "--server",
+        default="http://localhost:8585/api",
+        help="OpenMetadata server URL",
+    )
+    parser.add_argument(
+        "--token",
+        required=True,
+        help="JWT token for authentication",
+    )
+    parser.add_argument(
+        "--tables",
+        type=int,
+        default=100000,
+        help="Number of tables to create (default: 100000)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=100,
+        help="Batch size for table creation (default: 100)",
+    )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=10,
+        help="Number of parallel workers (default: 10)",
+    )
+
+    args = parser.parse_args()
+
+    ingest_tables(
+        server_url=args.server,
+        token=args.token,
+        total_tables=args.tables,
+        batch_size=args.batch_size,
+        workers=args.workers,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/jacoco_diff_coverage.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+
+HUNK_PATTERN = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
+
+
+@dataclass
+class FileCoverage:
+    path: str
+    executable_lines: list[int]
+    covered_lines: list[int]
+    missed_lines: list[int]
+    non_executable_lines: list[int]
+
+    @property
+    def executable_count(self) -> int:
+        return len(self.executable_lines)
+
+    @property
+    def covered_count(self) -> int:
+        return len(self.covered_lines)
+
+    @property
+    def missed_count(self) -> int:
+        return len(self.missed_lines)
+
+    @property
+    def non_executable_count(self) -> int:
+        return len(self.non_executable_lines)
+
+    @property
+    def coverage_pct(self) -> float | None:
+        if not self.executable_count:
+            return None
+        return (self.covered_count / self.executable_count) * 100.0
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Compute JaCoCo coverage for changed production lines in a PR diff."
+    )
+    parser.add_argument("--report", required=True, help="Path to jacoco.xml report")
+    parser.add_argument(
+        "--source-root",
+        default="openmetadata-service/src/main/java",
+        help="Source root to evaluate for changed production code",
+    )
+    parser.add_argument(
+        "--base-ref",
+        required=True,
+        help="Git base ref/SHA. If --head-ref is omitted, compare working tree against this ref.",
+    )
+    parser.add_argument(
+        "--head-ref",
+        help="Git head ref/SHA. If set, diff is computed with base...head.",
+    )
+    parser.add_argument(
+        "--minimum-coverage",
+        type=float,
+        default=90.0,
+        help="Minimum required changed-line coverage percentage",
+    )
+    parser.add_argument(
+        "--markdown-output",
+        required=True,
+        help="File to write the Markdown summary to",
+    )
+    return parser.parse_args()
+
+
+def run_git_diff(base_ref: str, head_ref: str | None, source_root: str) -> str:
+    cmd = ["git", "diff", "--unified=0", "--no-color"]
+    if head_ref:
+        cmd.append(f"{base_ref}...{head_ref}")
+    else:
+        cmd.append(base_ref)
+    cmd.extend(["--", source_root])
+    result = subprocess.run(cmd, check=True, capture_output=True, text=True)
+    return result.stdout
+
+
+def parse_changed_lines(diff_text: str) -> dict[str, set[int]]:
+    changed_lines: dict[str, set[int]] = defaultdict(set)
+    current_file: str | None = None
+    current_line: int | None = None
+
+    for line in diff_text.splitlines():
+        if line.startswith("+++ "):
+            file_path = line[4:]
+            if file_path == "/dev/null":
+                current_file = None
+            elif file_path.startswith("b/"):
+                current_file = file_path[2:]
+            else:
+                current_file = file_path
+            current_line = None
+            continue
+
+        if line.startswith("@@ "):
+            match = HUNK_PATTERN.match(line)
+            current_line = int(match.group(1)) if match else None
+            continue
+
+        if current_file is None or current_line is None:
+            continue
+
+        if line.startswith("+") and not line.startswith("+++"):
+            changed_lines[current_file].add(current_line)
+            current_line += 1
+        elif line.startswith("-") and not line.startswith("---"):
+            continue
+        elif line.startswith(" "):
+            current_line += 1
+
+    return changed_lines
+
+
+def parse_jacoco_report(report_path: str, source_root: str) -> dict[str, dict[int, bool]]:
+    root = ET.parse(report_path).getroot()
+    normalized_root = PurePosixPath(source_root)
+    coverage: dict[str, dict[int, bool]] = {}
+
+    for package in root.findall("package"):
+        package_name = package.attrib.get("name", "")
+        package_root = normalized_root / package_name if package_name else normalized_root
+
+        for sourcefile in package.findall("sourcefile"):
+            file_path = (package_root / sourcefile.attrib["name"]).as_posix()
+            line_coverage: dict[int, bool] = {}
+            for line in sourcefile.findall("line"):
+                line_number = int(line.attrib["nr"])
+                line_coverage[line_number] = int(line.attrib["ci"]) > 0
+            coverage[file_path] = line_coverage
+
+    return coverage
+
+
+def build_file_coverage(
+    changed_lines: dict[str, set[int]], jacoco_coverage: dict[str, dict[int, bool]]
+) -> list[FileCoverage]:
+    files: list[FileCoverage] = []
+
+    for file_path in sorted(changed_lines):
+        line_map = jacoco_coverage.get(file_path, {})
+        executable_lines = sorted(line for line in changed_lines[file_path] if line in line_map)
+        covered_lines = sorted(line for line in executable_lines if line_map.get(line, False))
+        missed_lines = sorted(line for line in executable_lines if not line_map.get(line, False))
+        non_executable_lines = sorted(line for line in changed_lines[file_path] if line not in line_map)
+        files.append(
+            FileCoverage(
+                path=file_path,
+                executable_lines=executable_lines,
+                covered_lines=covered_lines,
+                missed_lines=missed_lines,
+                non_executable_lines=non_executable_lines,
+            )
+        )
+
+    files.sort(
+        key=lambda item: (
+            item.coverage_pct if item.coverage_pct is not None else 101.0,
+            -item.missed_count,
+            item.path,
+        )
+    )
+    return files
+
+
+def format_line_list(lines: list[int], limit: int = 12) -> str:
+    if not lines:
+        return "-"
+    if len(lines) <= limit:
+        return ", ".join(str(line) for line in lines)
+    visible = ", ".join(str(line) for line in lines[:limit])
+    return f"{visible}, +{len(lines) - limit} more"
+
+
+def render_markdown(
+    files: list[FileCoverage], minimum_coverage: float, source_root: str
+) -> tuple[str, bool]:
+    changed_files = len(files)
+    executable_total = sum(item.executable_count for item in files)
+    covered_total = sum(item.covered_count for item in files)
+    missed_total = sum(item.missed_count for item in files)
+    non_executable_total = sum(item.non_executable_count for item in files)
+    overall_pct = (covered_total / executable_total * 100.0) if executable_total else 100.0
+
+    failing_files = [
+        item for item in files if item.coverage_pct is not None and item.coverage_pct < minimum_coverage
+    ]
+    should_fail = executable_total > 0 and (
+        overall_pct < minimum_coverage or bool(failing_files)
+    )
+    status = "FAIL" if should_fail else "PASS"
+    status_icon = "❌" if should_fail else "✅"
+
+    lines: list[str] = []
+    lines.append("## OpenMetadata Service New-Code Coverage")
+    lines.append("")
+
+    if changed_files == 0:
+        lines.append(
+            f"{status_icon} No changed production Java files under `{source_root}`. Coverage gate skipped."
+        )
+        return "\n".join(lines) + "\n", False
+
+    lines.append(
+        f"{status_icon} **{status}**. Required changed-line coverage: `{minimum_coverage:.2f}%` overall and per touched production file."
+    )
+    lines.append("")
+    lines.append(
+        f"- Overall executable changed lines: `{covered_total}/{executable_total}` covered (`{overall_pct:.2f}%`)"
+    )
+    lines.append(f"- Missed executable changed lines: `{missed_total}`")
+    lines.append(f"- Non-executable changed lines ignored by JaCoCo: `{non_executable_total}`")
+    lines.append(f"- Changed production files: `{changed_files}`")
+    lines.append("")
+
+    if executable_total == 0:
+        lines.append(
+            "All changed production lines are non-executable from JaCoCo's perspective. Gate passed."
+        )
+        lines.append("")
+    elif failing_files:
+        lines.append("Files below threshold:")
+        for item in failing_files:
+            lines.append(
+                f"- `{item.path}`: `{item.covered_count}/{item.executable_count}` covered (`{item.coverage_pct:.2f}%`), uncovered lines `{format_line_list(item.missed_lines)}`"
+            )
+        lines.append("")
+
+    lines.append("| File | Covered | Missed | Executable | Non-exec | Coverage | Uncovered lines |")
+    lines.append("| --- | ---: | ---: | ---: | ---: | ---: | --- |")
+    for item in files:
+        coverage_display = (
+            f"{item.coverage_pct:.2f}%"
+            if item.coverage_pct is not None
+            else "N/A"
+        )
+        lines.append(
+            f"| `{item.path}` | {item.covered_count} | {item.missed_count} | {item.executable_count} | {item.non_executable_count} | {coverage_display} | {format_line_list(item.missed_lines)} |"
+        )
+
+    lines.append("")
+    lines.append(
+        f"Only changed executable lines under `{source_root}` are counted. Test files, comments, imports, and non-executable lines are excluded."
+    )
+    return "\n".join(lines) + "\n", should_fail
+
+
+def main() -> int:
+    args = parse_args()
+
+    diff_text = run_git_diff(args.base_ref, args.head_ref, args.source_root)
+    changed_lines = parse_changed_lines(diff_text)
+    jacoco_coverage = parse_jacoco_report(args.report, args.source_root)
+    files = build_file_coverage(changed_lines, jacoco_coverage)
+    markdown, should_fail = render_markdown(files, args.minimum_coverage, args.source_root)
+
+    Path(args.markdown_output).parent.mkdir(parents=True, exist_ok=True)
+    Path(args.markdown_output).write_text(markdown, encoding="utf-8")
+
+    if should_fail:
+        print(markdown)
+        return 1
+
+    print(markdown)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/reindex-perf-bootstrap.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+#
+# Reindex performance test bootstrap.
+#
+# Tears down any existing local OM stack, brings it up fresh against PostgreSQL,
+# waits for the server to be healthy, fetches the ingestion-bot JWT, and runs
+# scripts/ingest_100k_containers.py to populate ~100k containers (depth 1-10)
+# with column-level tags.
+#
+# Usage:
+#   ./scripts/reindex-perf-bootstrap.sh                # default 100k, skip-maven-build
+#   CONTAINERS=50000 ./scripts/reindex-perf-bootstrap.sh
+#   SKIP_BUILD=false ./scripts/reindex-perf-bootstrap.sh   # full mvn rebuild
+#   SKIP_INGEST=true ./scripts/reindex-perf-bootstrap.sh   # docker only, no data
+#   SKIP_DOCKER=true ./scripts/reindex-perf-bootstrap.sh   # data only, against running stack
+#
+# Requirements: docker, docker compose, python 3.10+, the `metadata` package
+# installed (cd ingestion && make install_dev_env from the repo's venv).
+
+set -euo pipefail
+
+# Resolve repo root regardless of CWD.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+cd "${REPO_ROOT}"
+
+# Tunables (override via env).
+CONTAINERS="${CONTAINERS:-100000}"
+WORKERS="${WORKERS:-10}"
+BATCH_SIZE="${BATCH_SIZE:-50}"
+SERVICE_NAME="${SERVICE_NAME:-reindex_perf_storage}"
+SERVER_URL="${SERVER_URL:-http://localhost:8585/api}"
+SKIP_BUILD="${SKIP_BUILD:-true}"      # default: skip mvn rebuild for fast iteration
+SKIP_DOCKER="${SKIP_DOCKER:-false}"
+SKIP_INGEST="${SKIP_INGEST:-false}"
+HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-600}"  # seconds to wait for OM to come up
+
+log() { printf "\n\033[1;36m== %s ==\033[0m\n" "$*" >&2; }
+warn() { printf "\033[1;33m!! %s\033[0m\n" "$*" >&2; }
+die() { printf "\033[1;31m++ %s\033[0m\n" "$*" >&2; exit 1; }
+
+ensure_python_sdk() {
+  python -c "import metadata" >/dev/null 2>&1 \
+    || die "Python 'metadata' package not importable. Activate the repo's venv and run 'cd ingestion && make install_dev_env'."
+}
+
+stop_stack() {
+  log "Tearing down any existing OM stack (postgres + mysql variants)"
+  docker compose -f docker/development/docker-compose-postgres.yml down -v --remove-orphans 2>/dev/null || true
+  docker compose -f docker/development/docker-compose.yml down -v --remove-orphans 2>/dev/null || true
+}
+
+start_stack() {
+  log "Starting OM stack (PostgreSQL backing store)"
+  local args=(-m ui -d postgresql)
+  if [[ "${SKIP_BUILD}" == "true" ]]; then
+    args+=(-s true)
+  fi
+  ./docker/run_local_docker.sh "${args[@]}"
+}
+
+wait_for_health() {
+  log "Waiting for OpenMetadata server (max ${HEALTH_TIMEOUT}s)"
+  local elapsed=0
+  while (( elapsed < HEALTH_TIMEOUT )); do
+    if curl -fsS "${SERVER_URL}/v1/system/version" >/dev/null 2>&1; then
+      log "Server is up after ${elapsed}s"
+      return 0
+    fi
+    sleep 5
+    elapsed=$((elapsed + 5))
+    if (( elapsed % 30 == 0 )); then
+      printf "  still waiting... (%ds)\n" "${elapsed}"
+    fi
+  done
+  die "Timed out waiting for OpenMetadata at ${SERVER_URL}"
+}
+
+fetch_token() {
+  # The local dev stack ships a fixed admin JWT (see run_local_docker_common.sh
+  # `authorizationToken`). It's bound to the built-in `admin` user and accepted by
+  # the dev profile's JwtFilter. We use it as the bearer to fetch the live
+  # ingestion-bot JWT; the bot's token is what we want for create_or_update calls
+  # because it has unrestricted entity write perms in the dev policy set.
+  log "Fetching ingestion-bot JWT"
+  local admin_jwt="${ADMIN_JWT:-eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg}"
+  local token
+  token="$(curl -fsS -H "Authorization: Bearer ${admin_jwt}" \
+    "${SERVER_URL}/v1/bots/name/ingestion-bot" \
+    | python -c 'import json,sys; d=json.load(sys.stdin); print((d.get("botUser") or {}).get("authenticationMechanism", {}).get("config", {}).get("JWTToken", ""))' 2>/dev/null || true)"
+  if [[ -z "${token}" ]]; then
+    # Fallback: just use the admin JWT directly. It has all permissions in the dev profile.
+    warn "Bot JWT lookup returned empty; falling back to the admin JWT (dev only)"
+    token="${admin_jwt}"
+  fi
+  printf '%s' "${token}"
+}
+
+run_ingest() {
+  local token="$1"
+  log "Generating ${CONTAINERS} containers via ingest_100k_containers.py"
+  python scripts/ingest_100k_containers.py \
+    --server "${SERVER_URL}" \
+    --token "${token}" \
+    --service "${SERVICE_NAME}" \
+    --containers "${CONTAINERS}" \
+    --workers "${WORKERS}" \
+    --batch-size "${BATCH_SIZE}"
+}
+
+main() {
+  log "Reindex perf bootstrap starting"
+  printf "  CONTAINERS=%s WORKERS=%s BATCH_SIZE=%s\n" "${CONTAINERS}" "${WORKERS}" "${BATCH_SIZE}"
+  printf "  SKIP_BUILD=%s SKIP_DOCKER=%s SKIP_INGEST=%s\n" \
+    "${SKIP_BUILD}" "${SKIP_DOCKER}" "${SKIP_INGEST}"
+
+  if [[ "${SKIP_INGEST}" != "true" ]]; then
+    ensure_python_sdk
+  fi
+
+  if [[ "${SKIP_DOCKER}" != "true" ]]; then
+    stop_stack
+    start_stack
+    wait_for_health
+  else
+    log "Skipping docker bring-up (SKIP_DOCKER=true)"
+    wait_for_health
+  fi
+
+  if [[ "${SKIP_INGEST}" == "true" ]]; then
+    log "Skipping data generation (SKIP_INGEST=true). Stack is up at ${SERVER_URL}."
+    exit 0
+  fi
+
+  local token
+  if [[ -n "${OM_JWT_TOKEN:-}" ]]; then
+    log "Using OM_JWT_TOKEN from env"
+    token="${OM_JWT_TOKEN}"
+  else
+    token="$(fetch_token)"
+  fi
+
+  run_ingest "${token}"
+
+  log "All done. Stack at ${SERVER_URL}, ~${CONTAINERS} containers under service '${SERVICE_NAME}'."
+}
+
+main "$@"

scripts/retire_slack_summary.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Render a Retire.js JSON report as Markdown + Slack-friendly summary.
+
+Usage:
+  python3 scripts/retire_slack_summary.py <retire-report.json | dir> \\
+      [--counts-file PATH] [--slack-file PATH] [--top N]
+
+Mirrors the surface of scripts/snyk_summary.py so the notify job can
+consume `_retire_counts.json` and `_retire_slack.txt` identically.
+"""
+import argparse
+import json
+import os
+import sys
+
+SEV_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
+SEV_RANK = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+NM = "node_modules/"
+
+
+def esc(s):
+    return str(s).replace("|", "\\|").replace("`", "'").replace("\n", " ")
+
+
+def sev_key(s):
+    return SEV_RANK.get((s or "low").lower(), 3)
+
+
+def resolve_report_path(src):
+    if os.path.isdir(src):
+        return os.path.join(src, "retire-report.json")
+    return src
+
+
+def short_path(filepath):
+    if NM in filepath:
+        return filepath[filepath.find(NM) + len(NM):]
+    return filepath
+
+
+def vuln_id(vuln):
+    """Stable identifier for dedup across files."""
+    ids = vuln.get("identifiers") or {}
+    return json.dumps(ids, sort_keys=True)
+
+
+def vuln_title(vuln):
+    ids = vuln.get("identifiers") or {}
+    cves = ids.get("CVE") or []
+    if cves:
+        return cves[0]
+    ghsa = ids.get("githubID")
+    if ghsa:
+        return ghsa
+    summary = ids.get("summary") or ""
+    return summary.split("\n")[0][:60] or "—"
+
+
+def vuln_summary(vuln):
+    ids = vuln.get("identifiers") or {}
+    summary = ids.get("summary") or ""
+    return summary.split("\n")[0][:80]
+
+
+def vuln_fix(vuln):
+    """Retire.js encodes fix as `below: <first-fixed-version>`."""
+    below = vuln.get("below")
+    if below:
+        return str(below)
+    return "no fix"
+
+
+def collect_libs(data):
+    """Aggregate vulnerabilities per (component, version), deduped."""
+    libs = {}
+    findings = data.get("data") or []
+    for item in findings:
+        filepath = item.get("file", "")
+        short = short_path(filepath)
+        for result in item.get("results") or []:
+            key = (result.get("component", "?"), result.get("version", "?"))
+            entry = libs.setdefault(key, {
+                "files": [],
+                "vulns": [],
+                "seen": set(),
+            })
+            if short and short not in entry["files"]:
+                entry["files"].append(short)
+            for v in result.get("vulnerabilities") or []:
+                vid = vuln_id(v)
+                if vid in entry["seen"]:
+                    continue
+                entry["seen"].add(vid)
+                entry["vulns"].append(v)
+    return libs
+
+
+def lib_top_severity(info):
+    if not info["vulns"]:
+        return "low"
+    return min(
+        (v.get("severity", "low") for v in info["vulns"]),
+        key=sev_key,
+    )
+
+
+def count_severities(libs):
+    counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+    for info in libs.values():
+        for v in info["vulns"]:
+            sev = (v.get("severity") or "low").lower()
+            if sev in counts:
+                counts[sev] += 1
+    return counts
+
+
+def render_md(libs):
+    out = ["## 🛡️ Vulnerability JS Scan\n"]
+    if not libs:
+        out.append("✅ No vulnerable libraries found.\n")
+        return "\n".join(out)
+    total_vulns = sum(len(info["vulns"]) for info in libs.values())
+    out.append(
+        f"> **{len(libs)} vulnerable librar"
+        f"{'y' if len(libs) == 1 else 'ies'} · {total_vulns} "
+        f"CVE{'s' if total_vulns != 1 else ''} found**\n"
+    )
+    ordered = sorted(libs.items(), key=lambda kv: sev_key(lib_top_severity(kv[1])))
+    for (component, version), info in ordered:
+        top_icon = SEV_ICON.get(lib_top_severity(info), "⚪")
+        out.append(f"### {top_icon} {component} {version}\n")
+        out.append("| Severity | CVE | Summary | Fix |")
+        out.append("|---|---|---|---|")
+        for v in sorted(info["vulns"], key=lambda x: sev_key(x.get("severity"))):
+            sev = v.get("severity", "")
+            icon = SEV_ICON.get(sev, "⚪")
+            ids = v.get("identifiers") or {}
+            cves = ids.get("CVE") or []
+            if cves:
+                cve_str = ", ".join(
+                    f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves
+                )
+            else:
+                cve_str = ids.get("githubID") or "—"
+            summary = vuln_summary(v) or "—"
+            fix = vuln_fix(v)
+            out.append(
+                f"| {icon} {sev} | {esc(cve_str)} | {esc(summary)} | {esc(fix)} |"
+            )
+        if info["files"]:
+            out.append("\n**Bundled in:**")
+            for f in info["files"]:
+                out.append(f"- `{f}`")
+        out.append("")
+    return "\n".join(out)
+
+
+def render_lib_slack(component, version, info, top):
+    head = f"📦 *{component}@{version}* — {len(info['vulns'])} CVE{'s' if len(info['vulns']) != 1 else ''}"
+    rows = []
+    ordered = sorted(info["vulns"], key=lambda v: sev_key(v.get("severity")))
+    for v in ordered[:top]:
+        icon = SEV_ICON.get(v.get("severity", "low"), "⚪")
+        title = vuln_title(v)
+        fix = vuln_fix(v)
+        fix_label = "no fix" if fix == "no fix" else f"fix in {fix}"
+        rows.append(f"  {icon} {title} · {fix_label}")
+    extra = len(info["vulns"]) - top
+    if extra > 0:
+        rows.append(f"  … +{extra} more")
+    return head + "\n" + "\n".join(rows) if rows else head
+
+
+def render_slack(libs, totals, top):
+    header = (
+        f"*🛡️ Vulnerability JS Scan*\n"
+        f"🚨 {totals['critical']} critical  ·  🔴 {totals['high']} high  ·  "
+        f"🟠 {totals['medium']} medium  ·  🟡 {totals['low']} low"
+    )
+    if not libs:
+        return f"{header}\n> ✅ No vulnerable libraries found."
+    ordered = sorted(libs.items(), key=lambda kv: sev_key(lib_top_severity(kv[1])))
+    sections = [
+        render_lib_slack(component, version, info, top)
+        for (component, version), info in ordered
+    ]
+    body = "\n\n".join([header] + sections)
+    if len(body) > 2800:
+        cut = body.rfind("\n", 0, 2750)
+        if cut < 0:
+            cut = 2750
+        body = body[:cut].rstrip() + "\n…truncated. See Job Summary for full report."
+    return body
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("src", nargs="?", default="retire-report.json")
+    ap.add_argument("--counts-file")
+    ap.add_argument("--slack-file")
+    ap.add_argument("--top", type=int, default=5)
+    args = ap.parse_args()
+
+    path = resolve_report_path(args.src)
+    totals = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+
+    if not os.path.exists(path):
+        md = f"## 🛡️ Vulnerability JS Scan\n\n> Report file not found at `{path}`.\n"
+        sys.stdout.write(md)
+        if args.counts_file:
+            with open(args.counts_file, "w") as f:
+                json.dump({**totals, "total": 0}, f)
+        if args.slack_file:
+            with open(args.slack_file, "w") as f:
+                f.write("*🛡️ Vulnerability JS Scan*\n> Report file not found.")
+        return
+
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except Exception as e:
+        md = f"## 🛡️ Vulnerability JS Scan\n\n> Failed to parse `{path}`: {e}\n"
+        sys.stdout.write(md)
+        if args.counts_file:
+            with open(args.counts_file, "w") as f:
+                json.dump({**totals, "total": 0}, f)
+        if args.slack_file:
+            with open(args.slack_file, "w") as f:
+                f.write(f"*🛡️ Vulnerability JS Scan*\n> Parse error: {e}")
+        return
+
+    libs = collect_libs(data)
+    totals.update(count_severities(libs))
+
+    sys.stdout.write(render_md(libs) + "\n")
+
+    if args.counts_file:
+        with open(args.counts_file, "w") as f:
+            json.dump({**totals, "total": sum(totals.values())}, f)
+
+    if args.slack_file:
+        with open(args.slack_file, "w") as f:
+            f.write(render_slack(libs, totals, args.top))
+
+
+if __name__ == "__main__":
+    main()

scripts/scaffold_connector.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Thin wrapper to run the scaffold-connector command.
+
+Preferred usage:
+    metadata scaffold-connector              # Interactive mode
+    metadata scaffold-connector --name X ... # Non-interactive mode
+
+This script is provided for convenience when the `metadata` CLI is not
+installed:
+    python scripts/scaffold_connector.py     # Interactive mode
+"""
+import sys
+from pathlib import Path
+
+# Ensure the ingestion source is on the path
+ingestion_src = Path(__file__).resolve().parent.parent / "ingestion" / "src"
+if str(ingestion_src) not in sys.path:
+    sys.path.insert(0, str(ingestion_src))
+
+from metadata.cmd import metadata  # noqa: E402
+
+if __name__ == "__main__":
+    metadata(["scaffold-connector"] + sys.argv[1:])

scripts/slack-link-monitor.py

@@ -0,0 +1,100 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Slack Link Monitor - validates Slack invite links are still active.
+Uses Playwright to render the page and check for expiration messages.
+"""
+
+import logging
+import os
+from typing import Optional
+
+from playwright.sync_api import sync_playwright
+from slack_sdk.webhook import WebhookClient
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def get_required_env_var(var_name: str) -> str:
+    value: Optional[str] = os.getenv(var_name)
+    if value is None:
+        raise RuntimeError(f"Required environment variable '{var_name}' is not set.")
+    return value
+
+
+slack_webhook_url: str = get_required_env_var("SLACK_WEBHOOK_URL")
+github_server_url: str = get_required_env_var("GITHUB_SERVER_URL")
+github_repository: str = get_required_env_var("GITHUB_REPOSITORY")
+github_run_id: str = get_required_env_var("GITHUB_RUN_ID")
+
+logger.info("Successfully loaded all required environment variables.")
+logger.info(f"Repo: {github_repository}, Run ID: {github_run_id}")
+
+
+def send_slack_alert(message: str) -> None:
+    slack_client = WebhookClient(url=slack_webhook_url)
+    full_message = f"{message}\nWorkflow run: {github_server_url}/{github_repository}/actions/runs/{github_run_id}"
+    slack_client.send(text=full_message)
+
+
+def validate_slack_link(url: str) -> bool:
+    """
+    Returns True if the Slack invite link is active, False if expired.
+    """
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        try:
+            page = browser.new_page()
+            page.goto(url, wait_until="networkidle", timeout=30000)
+            content = page.content()
+            return "This link is no longer active" not in content
+        finally:
+            browser.close()
+
+
+def main():
+    slack_url_map = {
+        "open-metadata": "https://slack.open-metadata.org",
+        "free-tier-support": "https://free-tier-support.getcollate.io/",
+    }
+
+    errors = []
+
+    for product_type, slack_url in slack_url_map.items():
+        try:
+            logger.info(f"Checking {product_type}: {slack_url}")
+            is_active = validate_slack_link(slack_url)
+
+            if is_active:
+                logger.info(f"{product_type} slack link is active")
+            else:
+                error_msg = f"🔥 {product_type} slack link is EXPIRED 🔥"
+                logger.error(error_msg)
+                send_slack_alert(error_msg)
+                errors.append(error_msg)
+
+        except Exception as err:
+            error_msg = f"🔥 {product_type} monitoring error: {err} 🔥"
+            logger.error(error_msg)
+            send_slack_alert(error_msg)
+            errors.append(error_msg)
+
+    if errors:
+        raise RuntimeError(f"Monitoring failed with {len(errors)} error(s)")
+
+
+if __name__ == "__main__":
+    main()

scripts/snyk_summary.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""Render Snyk JSON reports as Markdown.
+
+Usage:
+  python3 scripts/snyk_summary.py [dir] \\
+      [--counts-file PATH] [--slack-file PATH] [--top N]
+"""
+import argparse
+import glob
+import json
+import os
+import sys
+
+SEV_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
+SEV_RANK = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+
+
+def esc(s):
+    return str(s).replace("|", "\\|").replace("`", "'").replace("\n", " ")
+
+
+def sev_key(s):
+    return SEV_RANK.get((s or "low").lower(), 3)
+
+
+def load(path):
+    try:
+        with open(path) as f:
+            return json.load(f), None
+    except Exception as e:
+        return None, str(e)
+
+
+def iter_projects(data):
+    if isinstance(data, list):
+        for d in data:
+            yield d
+    elif isinstance(data, dict):
+        yield data
+
+
+def collect_deps(data):
+    """Aggregate vulnerabilities by (package, version). Return (libs_dict, total_findings)."""
+    libs = {}
+    total = 0
+    for proj in iter_projects(data):
+        if not isinstance(proj, dict):
+            continue
+        for v in proj.get("vulnerabilities", []) or []:
+            key = (v.get("packageName", "?"), v.get("version", "?"))
+            entry = libs.setdefault(key, {
+                "sev": "low", "cves": set(), "titles": set(),
+                "fixedIn": set(), "paths": 0, "ids": set(),
+            })
+            if sev_key(v.get("severity")) < sev_key(entry["sev"]):
+                entry["sev"] = v.get("severity", "low")
+            for cve in (v.get("identifiers", {}) or {}).get("CVE", []) or []:
+                entry["cves"].add(cve)
+            entry["titles"].add(v.get("title", ""))
+            for fx in v.get("fixedIn", []) or []:
+                entry["fixedIn"].add(fx)
+            entry["ids"].add(v.get("id", ""))
+            entry["paths"] += 1
+            total += 1
+    return libs, total
+
+
+def collect_code(data):
+    """Aggregate Snyk Code SARIF results. Return (by_rule_dict, rules_desc_dict, total)."""
+    runs = data.get("runs", []) if isinstance(data, dict) else []
+    findings = {}
+    rules = {}
+    for run in runs:
+        for rule in (run.get("tool", {}).get("driver", {}).get("rules", []) or []):
+            rules[rule.get("id")] = rule.get("shortDescription", {}).get("text", rule.get("name", ""))
+        for r in run.get("results", []) or []:
+            rid = r.get("ruleId", "?")
+            level = r.get("level", "warning")
+            for loc in r.get("locations", []) or []:
+                phys = loc.get("physicalLocation", {})
+                uri = phys.get("artifactLocation", {}).get("uri", "?")
+                line = phys.get("region", {}).get("startLine", "?")
+                findings.setdefault((rid, uri, level), []).append(line)
+    by_rule = {}
+    for (rid, uri, level), lines in findings.items():
+        by_rule.setdefault(rid, []).append((uri, level, lines))
+    total = sum(len(v) for v in by_rule.values())
+    return by_rule, rules, total
+
+
+def render_deps_md(name, libs, total):
+    out = [f"\n### 📦 {name}\n"]
+    if not libs:
+        out.append("✅ No vulnerabilities.\n")
+        return "\n".join(out)
+    out.append(f"> **{len(libs)} vulnerable librar{'y' if len(libs)==1 else 'ies'} · {total} finding{'s' if total!=1 else ''}**\n")
+    out.append("| Sev | Package | Version | CVE / ID | Title | Fix in | Paths |")
+    out.append("|---|---|---|---|---|---|---|")
+    for (pkg, ver), info in sorted(libs.items(), key=lambda kv: sev_key(kv[1]["sev"])):
+        sev = info["sev"]
+        ids = sorted(info["cves"]) or sorted(i for i in info["ids"] if i)[:1]
+        ids_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" if c.startswith("CVE") else c for c in ids) or "—"
+        title = sorted(info["titles"])[0][:80] if info["titles"] else ""
+        fix = ", ".join(sorted(info["fixedIn"])[:2]) or "—"
+        out.append(f"| {SEV_ICON.get(sev,'⚪')} {sev} | `{esc(pkg)}` | {esc(ver)} | {esc(ids_str)} | {esc(title)} | {esc(fix)} | {info['paths']} |")
+    out.append("")
+    return "\n".join(out)
+
+
+def render_code_md(name, by_rule, rules, total):
+    out = [f"\n### 🔎 {name} (Code)\n"]
+    if not by_rule:
+        out.append("✅ No code findings.\n")
+        return "\n".join(out)
+    out.append(f"> **{total} location{'s' if total!=1 else ''} across {len(by_rule)} rule{'s' if len(by_rule)!=1 else ''}**\n")
+    for rid, items in sorted(by_rule.items()):
+        desc = rules.get(rid, rid)
+        out.append(f"<details><summary><b>{esc(rid)}</b> — {esc(desc)} ({len(items)})</summary>\n")
+        out.append("| Level | File | Lines |")
+        out.append("|---|---|---|")
+        for uri, level, lines in sorted(items):
+            ln = ", ".join(str(x) for x in sorted(set(lines))[:10])
+            icon = "🔴" if level == "error" else "🟠"
+            out.append(f"| {icon} {level} | `{esc(uri)}` | {esc(ln)} |")
+        out.append("\n</details>\n")
+    return "\n".join(out)
+
+
+def render_deps_slack(name, libs, total, top):
+    if not libs:
+        return f"📦 *{name}*: ✅ clean"
+    head = f"📦 *{name}*: {len(libs)} libs · {total} findings"
+    rows = []
+    ordered = sorted(libs.items(), key=lambda kv: sev_key(kv[1]["sev"]))
+    for (pkg, ver), info in ordered[:top]:
+        icon = SEV_ICON.get(info["sev"], "⚪")
+        title = (sorted(info["titles"])[0] if info["titles"] else "")[:60]
+        fix = sorted(info["fixedIn"])[0] if info["fixedIn"] else "no fix"
+        rows.append(f"  {icon} `{pkg}` {ver} — {title} (fix: {fix})")
+    extra = len(libs) - top
+    if extra > 0:
+        rows.append(f"  … +{extra} more")
+    return head + "\n" + "\n".join(rows)
+
+
+def render_code_slack(name, by_rule, rules, total, top):
+    if not by_rule:
+        return f"🔎 *{name}*: ✅ clean"
+    head = f"🔎 *{name}*: {total} findings · {len(by_rule)} rules"
+    ordered = sorted(by_rule.items(), key=lambda kv: -len(kv[1]))
+    rows = [f"  • `{rid}` ({len(items)})" for rid, items in ordered[:top]]
+    extra = len(by_rule) - top
+    if extra > 0:
+        rows.append(f"  … +{extra} more")
+    return head + "\n" + "\n".join(rows)
+
+
+def count_severities(libs, by_rule):
+    """Snyk Code SARIF uses level (error/warning/note); treat error=high, warning=medium, note=low.
+    Dep libs already have explicit severity."""
+    counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+    for info in libs.values():
+        sev = (info["sev"] or "low").lower()
+        if sev in counts:
+            counts[sev] += info["paths"]
+    for rid, items in by_rule.items():
+        for uri, level, lines in items:
+            mapped = {"error": "high", "warning": "medium", "note": "low"}.get(level, "medium")
+            counts[mapped] += 1  # count locations to match render_code_md total
+    return counts
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("src", nargs="?", default="security-report")
+    ap.add_argument("--counts-file")
+    ap.add_argument("--slack-file")
+    ap.add_argument("--top", type=int, default=5)
+    args = ap.parse_args()
+
+    md_parts = ["## 🛡️ Snyk Security Scan\n"]
+    slack_parts = ["*🛡️ Snyk Security Scan*"]
+    totals = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+
+    files = sorted(glob.glob(os.path.join(args.src, "*.json")))
+    # exclude our own output files if present
+    files = [f for f in files if not os.path.basename(f).startswith("_")]
+
+    if not files:
+        md_parts.append(f"> No JSON reports found in `{args.src}/`.")
+        sys.stdout.write("\n".join(md_parts) + "\n")
+        if args.counts_file:
+            with open(args.counts_file, "w") as f:
+                json.dump({**totals, "total": 0}, f)
+        if args.slack_file:
+            with open(args.slack_file, "w") as f:
+                f.write("*🛡️ Snyk Security Scan*\n> No JSON reports found.")
+        return
+
+    for path in files:
+        name = os.path.basename(path).replace(".json", "")
+        data, err = load(path)
+        if err is not None:
+            md_parts.append(f"\n### ⚠️ {name}\nFailed to parse: {err}\n")
+            slack_parts.append(f"⚠️ *{name}*: parse error — {err}")
+            continue
+        if isinstance(data, dict) and "runs" in data:
+            by_rule, rules, total = collect_code(data)
+            md_parts.append(render_code_md(name, by_rule, rules, total))
+            slack_parts.append(render_code_slack(name, by_rule, rules, total, args.top))
+            sub = count_severities({}, by_rule)
+        else:
+            libs, total = collect_deps(data)
+            md_parts.append(render_deps_md(name, libs, total))
+            slack_parts.append(render_deps_slack(name, libs, total, args.top))
+            sub = count_severities(libs, {})
+        for k in totals:
+            totals[k] += sub[k]
+
+    sys.stdout.write("\n".join(md_parts) + "\n")
+
+    if args.counts_file:
+        with open(args.counts_file, "w") as f:
+            json.dump({**totals, "total": sum(totals.values())}, f)
+
+    if args.slack_file:
+        header = (
+            f"*🛡️ Snyk Security Scan*\n"
+            f"🚨 {totals['critical']} critical  ·  🔴 {totals['high']} high  ·  "
+            f"🟠 {totals['medium']} medium  ·  🟡 {totals['low']} low"
+        )
+        body = "\n\n".join([header] + slack_parts[1:])
+        # Slack section block text limit 3000 chars; leave headroom for shell-added header lines.
+        # Truncate at the last newline before the limit so we don't cut a Slack mrkdwn link
+        # (`<url|text>`) or a multi-codepoint emoji sequence mid-token.
+        if len(body) > 2800:
+            cut = body.rfind("\n", 0, 2750)
+            if cut < 0:
+                cut = 2750
+            body = body[:cut].rstrip() + "\n…truncated. See Job Summary for full report."
+        with open(args.slack_file, "w") as f:
+            f.write(body)
+
+
+if __name__ == "__main__":
+    main()

scripts/test_connection.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Test connection to OpenMetadata."""
+import sys
+
+# Read token from file
+with open("/Users/harsha/Code/OpenMetadata/scripts/token.txt") as f:
+    token = f.read().strip()
+
+print(f"Token length: {len(token)}")
+print("Connecting to OpenMetadata...")
+
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+
+server_config = OpenMetadataConnection(
+    hostPort="http://localhost:8585/api",
+    securityConfig=OpenMetadataJWTClientConfig(jwtToken=token),
+)
+
+print("Created config, initializing client...")
+metadata = OpenMetadata(server_config)
+print("Client initialized!")
+
+# Test a simple API call
+print("Testing API call...")
+from metadata.generated.schema.entity.services.databaseService import DatabaseService
+services = metadata.list_all_entities(entity=DatabaseService, limit=5)
+count = 0
+for svc in services:
+    print(f"  Found service: {svc.name}")
+    count += 1
+    if count >= 3:
+        break
+
+print("Connection successful!")

scripts/update_version.py

@@ -0,0 +1,144 @@
+import argparse
+import re
+import logging
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger()
+
+
+def get_python_version(version: str) -> str:
+    "Get Python Package formatted version"
+
+    if "-rc" in version:
+        version_parts = version.split("-")
+        return f"{version_parts[0]}.0{version_parts[1]}"
+    if "-SNAPSHOT" in version:
+        version_parts = version.split("-")
+        return f"{version_parts[0]}.0.dev0"
+    return f"{version}.0"
+
+
+def regex_sub(file_path: str, pattern: str, substitution: str):
+    with open(file_path, "r") as f:
+        content = f.read()
+
+    updated_content = re.sub(pattern, substitution, content)
+
+    with open(file_path, "w") as f:
+        f.write(updated_content)
+
+
+def update_dockerfile_arg(arg, file_path, value):
+    """Updates a Dockerfile ARG."""
+
+    logger.info(f"Updating ARG {arg} in {file_path} to {value}\n")
+
+    regex_sub(
+        file_path,
+        rf"(ARG\s+{arg}=).+",
+        rf'\1"{value}"',
+    )
+
+
+def update_docker_tag(args):
+    """Updates the Docker Tag on docker-compose files."""
+
+    file_path = args.file_path
+    tag = args.tag
+
+    logger.info(f"Updating Docker Tag in {file_path} to {tag}\n")
+
+    regex_sub(
+        file_path,
+        r"(image: docker\.getcollate\.io/openmetadata/.*?):.+",
+        rf"\1:{tag}",
+    )
+
+
+def update_ri_version(args):
+    """Updates a Dockerfile RI_VERSION ARG."""
+
+    version = args.version
+    with_python_version = args.with_python_version
+    file_path = args.file_path
+
+    if with_python_version:
+        version = get_python_version(version)
+
+    update_dockerfile_arg(arg="RI_VERSION", file_path=file_path, value=version)
+
+
+def update_pyproject_version(args):
+    """Updates pyproject version."""
+
+    file_path = args.file_path
+    version = args.version
+
+    version = get_python_version(version)
+
+    logger.info(f"Updating {file_path} version to {version}\n")
+
+    regex_sub(
+        file_path,
+        r'(?m)^version\s*=\s*"[^"]+"',
+        f'version = "{version}"',
+    )
+
+
+def update_openapi_version(args):
+    """Updates OpenAPI version in OpenMetadataApplication.java."""
+
+    version = args.version
+    file_path = "openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java"
+
+    logger.info(f"Updating OpenAPI version in {file_path} to {version}\n")
+
+    regex_sub(
+        file_path,
+        r'(@Info\s*\(\s*title\s*=\s*"OpenMetadata APIs",\s*version\s*=\s*")\d+\.\d+\.\d+(")',
+        rf"\g<1>{version}\g<2>",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Update files for release.")
+    subparsers = parser.add_subparsers(required=True)
+
+    # Update Docker tag
+    parser_udt = subparsers.add_parser("update_docker_tag")
+    parser_udt.add_argument(
+        "--file-path", "-f", type=str, help="Docker compose file to update."
+    )
+    parser_udt.add_argument("--tag", "-t", type=str, help="Tag to update the file to.")
+    parser_udt.set_defaults(func=update_docker_tag)
+
+    # Update Dockerfile ARG
+    parser_urv = subparsers.add_parser("update_ri_version")
+    parser_urv.add_argument(
+        "--file-path", "-f", type=str, help="Dockerfile file to update."
+    )
+    parser_urv.add_argument(
+        "--version", "-v", type=str, help="Verision to set for the argument"
+    )
+    parser_urv.add_argument("--with-python-version", action="store_true")
+    parser_urv.set_defaults(func=update_ri_version)
+
+    # Update pyproject.toml Version
+    parser_upv = subparsers.add_parser("update_pyproject_version")
+    parser_upv.add_argument(
+        "--file-path", "-f", type=str, help="pyproject.toml file to update."
+    )
+    parser_upv.add_argument("--version", "-v", type=str, help="Version to update to")
+    parser_upv.set_defaults(func=update_pyproject_version)
+
+    # Update OpenAPI version in OpenMetadataApplication.java
+    parser_uoav = subparsers.add_parser("update_openapi_version")
+    parser_uoav.add_argument("--version", "-v", type=str, help="Version to update to")
+    parser_uoav.set_defaults(func=update_openapi_version)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/validate_change.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Fetch latest from remotes
+git fetch --quiet 2>/dev/null
+
+# Auto-detect base branch (upstream/main > origin/main)
+if git rev-parse --verify upstream/main &>/dev/null; then
+    BASE_BRANCH="upstream/main"
+elif git rev-parse --verify origin/main &>/dev/null; then
+    BASE_BRANCH="origin/main"
+  else
+      echo "❌ Could not find main branch on origin or upstream"
+      exit 1
+fi
+
+echo "📊 Comparing against: ${BASE_BRANCH}"
+
+
+# Get the diff (staged changes, or last commit, or working directory)
+if [ "$1" == "--staged" ]; then
+    DIFF=$(git diff --cached --ignore-submodules)
+    CONTEXT="staged changes"
+elif [ "$1" == "--last-commit" ]; then
+    DIFF=$(git diff HEAD~1 --ignore-submodules)
+    CONTEXT="last commit: $(git log -1 --pretty=%B)"
+elif [ "$1" == "--working" ]; then
+    DIFF=$(git diff --ignore-submodules)
+    CONTEXT="working directory changes"
+else
+    # Default: compare current branch to base
+    DIFF=$(git diff ${BASE_BRANCH}...HEAD --ignore-submodules --no-color -U3 --minimal)
+    CONTEXT="current branch vs ${BASE_BRANCH}"
+fi
+
+# Get list of changed files vs base branch
+CHANGED_FILES=$(git diff --name-only ${BASE_BRANCH}...HEAD --ignore-submodules 2>/dev/null || git diff --name-only HEAD)
+
+# Current branch name
+CURRENT_BRANCH=$(git branch --show-current)
+
+# Exit if no changes
+if [ -z "$DIFF" ]; then
+    echo "No changes to validate"
+    exit 0
+fi
+
+# Run Claude Code with the diff as context (-p for non-interactive mode)
+claude "
+## Task: Validate Code Changes
+
+### Context
+Branch: ${CURRENT_BRANCH}
+Comparing: ${CONTEXT}
+
+### Changed Files:
+$CHANGED_FILES
+
+### Diff:
+\`\`\`diff
+$DIFF
+\`\`\`
+
+### Instructions:
+1. Analyze which files changed and what functionality was affected
+2. Provide a throrough code review of the changes validating for:
+   - Code quality
+   - Adherence to project conventions
+   - Potential bugs or issues
+   - Missing tests or documentation
+   - Security implications
+   - Performance considerations
+3. Determine which tests are relevant:
+   - Unit tests for changed modules
+   - Integration tests if APIs/services changed (for python integration tests assume docker and test DB are available)
+   - E2E tests (via Playwright MCP) if UI/flows changed
+4. Run the relevant tests:
+   - For unit/integration: use the project's test runner (npm test, pytest, etc.)
+   - For E2E: use Playwright MCP to validate on http://localhost:8585
+5. Report results in this exact format:
+
+## Code Review
+[Detailed code review comments by files changed]
+
+## Validation Results
+
+| Test Type | Status | Details |
+|-----------|--------|---------|
+| Unit      | 🟢/🔴  | X passed, Y failed |
+| Integration | 🟢/🔴 | X passed, Y failed |
+| E2E       | 🟢/🔴  | Verified: [list] |
+
+### Overall: 🟢 PASS or 🔴 FAIL
+
+[If FAIL, list specific failures and suggested fixes]
+" --allowedTools "Bash,mcp__playwright__*" 2>&1
+
+# Capture exit code
+EXIT_CODE=$?
+
+exit $EXIT_CODE

scripts/validate_json_yaml.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# bash strict mode
+set -eup pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+EXCLUDED_DIRS=".vscode|great_expectations/resources|playwright/test-data"
+echo "Validating JSON files..."
+git ls-files | grep "\.json$" | grep -vE "/($EXCLUDED_DIRS)/" | while read file; do jq . "$file" >/dev/null 2>&1 || { echo "Invalid JSON in $file"; exit 1; }; done
+echo "Validating YAML files..."
+git ls-files | grep -E "\.ya?ml$" | grep -vE "/($EXCLUDED_DIRS)/" | while read file; do python ${SCRIPT_DIR}/validate_yaml.py "$file" >/dev/null 2>&1 || { echo "Invalid YAML in $file"; exit 1; }; done

scripts/validate_sample_data.py

@@ -0,0 +1,10 @@
+from metadata.generated.schema.entity.data.table import Table
+from _openmetadata_testutils.ometa import int_admin_ometa
+
+metadata = int_admin_ometa()
+entity = metadata.get_by_name(
+    entity=Table, fqn="sample_data.ecommerce_db.shopify.dim_address"
+)
+
+if not entity:
+    raise ValueError("Table not found")

scripts/validate_yaml.py

@@ -0,0 +1,12 @@
+import os
+import sys
+
+import yaml
+
+path = sys.argv[1]
+if os.path.islink(path):
+    exit()
+
+with open(path, "r") as f:
+    # safe_load_all works for both single and multi-document YAML
+    list(yaml.safe_load_all(f))

scripts/worktree_dev.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# ~/scripts/task.sh
+
+set -e
+
+ISSUE=$1
+
+if [ -z "$ISSUE" ]; then
+    echo "Usage: task <issue-number>"
+    exit 1
+fi
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+REPO_NAME=$(basename "$REPO_ROOT")
+WORKTREE_PATH="${REPO_ROOT}/../${REPO_NAME}-${ISSUE}"
+BRANCH_NAME="feature/${ISSUE}"
+LOG_FILE="${WORKTREE_PATH}/claude-${ISSUE}.log"
+
+# Create worktree (new branch or existing)
+if [ -d "$WORKTREE_PATH" ]; then
+    echo "🌳 Worktree already exists at $WORKTREE_PATH, reusing..."
+else
+    echo "🌳 Creating worktree..."
+    if git show-ref --verify --quiet "refs/heads/${BRANCH_NAME}"; then
+        git worktree add "$WORKTREE_PATH" "$BRANCH_NAME"
+    else
+        git worktree add "$WORKTREE_PATH" -b "$BRANCH_NAME"
+    fi
+fi
+
+# Kill existing session if any
+tmux kill-session -t "task-${ISSUE}" 2>/dev/null || true
+
+# Create tmux session
+tmux new-session -d -s "task-${ISSUE}" -c "$WORKTREE_PATH"
+tmux set -g mouse on
+
+# Main pane: Claude auto-accept, output to log
+tmux send-keys -t "task-${ISSUE}" "claude \"
+## Task
+Implement the requirements from GitHub issue #${ISSUE}.
+
+If you are working with python, make sure you are creating a virtual environment and installing dependencies (make install_dev && make generate).
+
+## Steps
+1. First, read the issue: gh issue view ${ISSUE}
+2. Understand the requirements and acceptance criteria
+3. Implement the solution
+4. Write/update tests if applicable
+5. Run tests to verify
+6. make sure linting and formatting checks pass (e.g. make py_format, mvn spotless:apply, etc.), for python linting make sure the virtual environment is activated before running linting commands
+7. Commit with message describing the changes referencing issue ${ISSUE}
+8. Push: git push -u origin HEAD
+9. Create PR with a thorough description of the changes: gh pr create --title '#${ISSUE}: \<title\>' --body 'Closes #${ISSUE}'
+10. Summarize what you accomplished
+\"; echo '✅ Claude finished. Closing in 5s...'; sleep 5; tmux kill-session -t task-${ISSUE}" Enter
+
+# Right pane: Live diff (colored)
+tmux split-window -h -t "task-${ISSUE}" -c "$WORKTREE_PATH"
+tmux send-keys -t "task-${ISSUE}" "watch -c -n 2 'echo \"📊 Changes vs HEAD\"; echo; git diff --color=always --stat; echo; echo \"📜 Commits\"; git log --oneline -5 2>/dev/null || echo \"None yet\"'" Enter
+
+# Bottom-left pane: Git status
+tmux select-pane -t "task-${ISSUE}.0"
+tmux split-window -v -t "task-${ISSUE}" -c "$WORKTREE_PATH"
+tmux send-keys -t "task-${ISSUE}" "watch -n 2 'echo \"📝 Status\" && git status -s'" Enter
+
+# Focus on log pane (easier to watch)
+tmux select-pane -t "task-${ISSUE}.1"
+
+echo "🚀 Launching tmux session: task-${ISSUE}"
+echo "   Worktree: $WORKTREE_PATH"
+echo "   Log file: $LOG_FILE"
+
+# Attach to session
+tmux attach -t "task-${ISSUE}"