# Copyright 2025 Collate # Licensed under the Collate Community License, Version 1.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ OpenMetadata base class for tests """ import uuid from datetime import datetime from textwrap import dedent from typing import TYPE_CHECKING, Any, List, Optional, Type # noqa: UP035 if TYPE_CHECKING: from airflow import DAG from airflow.operators.bash import BashOperator # noqa: F401 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest from metadata.generated.schema.api.data.createDashboardDataModel import ( CreateDashboardDataModelRequest, ) from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createDatabaseSchema import ( CreateDatabaseSchemaRequest, ) from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest from metadata.generated.schema.api.data.createTable import CreateTableRequest from metadata.generated.schema.api.services.createDashboardService import ( CreateDashboardServiceRequest, ) from metadata.generated.schema.api.services.createDatabaseService import ( CreateDatabaseServiceRequest, ) from metadata.generated.schema.api.services.createMessagingService import ( CreateMessagingServiceRequest, ) from metadata.generated.schema.api.services.createMlModelService import ( CreateMlModelServiceRequest, ) from metadata.generated.schema.api.services.createPipelineService import ( CreatePipelineServiceRequest, ) from metadata.generated.schema.api.services.createStorageService import ( CreateStorageServiceRequest, ) from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest from metadata.generated.schema.api.teams.createUser import CreateUserRequest from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest from metadata.generated.schema.api.tests.createTestDefinition import ( CreateTestDefinitionRequest, ) from metadata.generated.schema.api.tests.createTestSuite import ( CreateTestSuiteRequest, TestSuiteEntityName, ) from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.dashboardDataModel import ( DashboardDataModel, DataModelType, ) from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.pipeline import Pipeline, Task from metadata.generated.schema.entity.data.table import Column, DataType, Table from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import ( LookerConnection, ) from metadata.generated.schema.entity.services.connections.database.common.basicAuth import ( BasicAuth, ) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import ( KafkaConnection, ) from metadata.generated.schema.entity.services.connections.mlmodel.mlflowConnection import ( MlflowConnection, ) from metadata.generated.schema.entity.services.connections.pipeline.customPipelineConnection import ( CustomPipelineConnection, CustomPipelineType, ) from metadata.generated.schema.entity.services.connections.storage.s3Connection import ( S3Connection, ) from metadata.generated.schema.entity.services.dashboardService import ( DashboardConnection, DashboardService, DashboardServiceType, ) from metadata.generated.schema.entity.services.databaseService import ( DatabaseConnection, DatabaseService, DatabaseServiceType, ) from metadata.generated.schema.entity.services.messagingService import ( MessagingConnection, MessagingService, MessagingServiceType, ) from metadata.generated.schema.entity.services.mlmodelService import ( MlModelConnection, MlModelService, MlModelServiceType, ) from metadata.generated.schema.entity.services.pipelineService import ( PipelineConnection, PipelineService, PipelineServiceType, ) from metadata.generated.schema.entity.services.storageService import ( StorageConnection, StorageService, StorageServiceType, ) from metadata.generated.schema.entity.teams.team import TeamType from metadata.generated.schema.security.credentials.awsCredentials import AWSCredentials from metadata.generated.schema.tests.testCase import TestCaseParameterValue from metadata.generated.schema.tests.testDefinition import ( TestCaseParameterDefinition, TestPlatform, ) from metadata.generated.schema.type.basic import ( Email, EntityLink, EntityName, FullyQualifiedEntityName, Markdown, TestCaseEntityName, ) from metadata.generated.schema.type.tagLabel import ( LabelType, State, TagFQN, TagLabel, TagSource, ) from metadata.ingestion.ometa.ometa_api import C, T from metadata.utils.dispatch import class_register TIER1_TAG: TagLabel = TagLabel( tagFQN=TagFQN(f"Tier.Tier1"), # noqa: F541 name="Tier1", source=TagSource.Classification, labelType=LabelType.Automated, state=State.Suggested, ) COLUMNS = [ Column(name="id", dataType=DataType.BIGINT), Column(name="another", dataType=DataType.BIGINT), Column( name="struct", dataType=DataType.STRUCT, children=[ Column(name="id", dataType=DataType.INT), Column(name="name", dataType=DataType.STRING), ], ), ] METADATA_INGESTION_CONFIG_TEMPLATE = dedent( """{{ "source": {{ "type": "{type}", "serviceName": "{service_name}", "serviceConnection": {{ "config": {service_config} }}, "sourceConfig": {{"config": {source_config} }} }}, "sink": {{"type": "metadata-rest", "config": {{}}}}, "workflowConfig": {{ "loggerLevel": "DEBUG", "openMetadataServerConfig": {{ "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", "securityConfig": {{ "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" }} }} }} }}""" ) PROFILER_INGESTION_CONFIG_TEMPLATE = dedent( """{{ "source": {{ "type": "{type}", "serviceName": "{service_name}", "serviceConnection": {{ "config": {service_config} }}, "sourceConfig": {{"config": {{"type":"Profiler", "profileSampleConfig": {{"sampleConfigType": "STATIC", "config": {{"profileSample": 100, "profileSampleType": "PERCENTAGE"}}}}}}}} }}, "processor": {{"type": "orm-profiler", "config": {{}}}}, "sink": {{"type": "metadata-rest", "config": {{}}}}, "workflowConfig": {{ "loggerLevel": "DEBUG", "openMetadataServerConfig": {{ "hostPort": "http://localhost:8585/api", "authProvider": "openmetadata", "securityConfig": {{ "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" }} }} }} }}""" ) def generate_name() -> EntityName: """Generate a random for the asset""" return EntityName(str(uuid.uuid4())) create_service_registry = class_register() def get_create_service(entity: Type[T], name: Optional[EntityName] = None) -> C: # noqa: UP006, UP045 """Create a vanilla service based on the input type""" func = create_service_registry.registry.get(entity.__name__) if not func: raise ValueError( f"Create Service for type {entity.__name__} has not yet been implemented. Add it on `integration_base.py`" ) if not name: name = generate_name().root return func(name) @create_service_registry.add(PipelineService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreatePipelineServiceRequest( name=name, serviceType=PipelineServiceType.CustomPipeline, connection=PipelineConnection(config=CustomPipelineConnection(type=CustomPipelineType.CustomPipeline)), ) @create_service_registry.add(DatabaseService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreateDatabaseServiceRequest( name=name, serviceType=DatabaseServiceType.Mysql, connection=DatabaseConnection( config=MysqlConnection( username="username", authType=BasicAuth( password="password", ), hostPort="http://localhost:1234", ) ), ) @create_service_registry.add(DashboardService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreateDashboardServiceRequest( name=name, serviceType=DashboardServiceType.Looker, connection=DashboardConnection( config=LookerConnection(hostPort="http://hostPort", clientId="id", clientSecret="secret") ), ) @create_service_registry.add(MessagingService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreateMessagingServiceRequest( name=name, serviceType=MessagingServiceType.Kafka, connection=MessagingConnection(config=KafkaConnection(bootstrapServers="localhost:9092")), ) @create_service_registry.add(StorageService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreateStorageServiceRequest( name=name, serviceType=StorageServiceType.S3, connection=StorageConnection(config=S3Connection(awsConfig=AWSCredentials(awsRegion="us-east-2"))), ) @create_service_registry.add(MlModelService) def _(name: EntityName) -> C: """Prepare a Create service request""" return CreateMlModelServiceRequest( name=name, serviceType=MlModelServiceType.Mlflow, connection=MlModelConnection( config=MlflowConnection( trackingUri="http://localhost:1234", registryUri="http://localhost:4321", ) ), ) create_entity_registry = class_register() def get_create_entity( entity: Type[T], # noqa: UP006 reference: Any, name: Optional[EntityName] = None, # noqa: UP045 ) -> C: """Create a vanilla entity based on the input type""" func = create_entity_registry.registry.get(entity.__name__) if not func: raise ValueError( f"Create Service for type {entity.__name__} has not yet been implemented. Add it on `integration_base.py`" ) if not name: name = generate_name().root return func(reference, name) @create_entity_registry.add(Pipeline) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreatePipelineRequest( name=name, service=reference, tasks=[ Task(name="task1"), Task(name="task2", downstreamTasks=["task1"]), Task(name="task3", downstreamTasks=["task2"]), Task(name="task4", downstreamTasks=["task2"]), ], ) @create_entity_registry.add(Database) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreateDatabaseRequest( name=name, service=reference, ) @create_entity_registry.add(DatabaseSchema) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreateDatabaseSchemaRequest( name=name, database=reference, ) @create_entity_registry.add(Table) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreateTableRequest( name=name, databaseSchema=reference, columns=COLUMNS, ) @create_entity_registry.add(Dashboard) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreateDashboardRequest( name=name, service=reference, ) @create_entity_registry.add(DashboardDataModel) def _(reference: FullyQualifiedEntityName, name: EntityName) -> C: return CreateDashboardDataModelRequest( name=name, service=reference, dataModelType=DataModelType.LookMlExplore, columns=COLUMNS, ) def get_create_user_entity(name: Optional[EntityName] = None, email: Optional[str] = None): # noqa: UP045 if not name: name = generate_name().root if not email: email = f"{generate_name().root}@getcollate.io" return CreateUserRequest(name=name, email=Email(root=email)) def get_create_team_entity(name: Optional[EntityName] = None, users=List[str]): # noqa: UP006, UP045 if not name: name = generate_name().root return CreateTeamRequest(name=name, teamType=TeamType.Group, users=users) def get_create_test_definition( parameter_definition: List[TestCaseParameterDefinition], # noqa: UP006 entity_type: [T], name: Optional[EntityName] = None, # noqa: UP045 description: Optional[str] = None, # noqa: UP045 ): if not name: name = generate_name().root if not description: description = generate_name().root return CreateTestDefinitionRequest( name=TestCaseEntityName(name), description=Markdown(description), entityType=entity_type, testPlatforms=[TestPlatform.GreatExpectations], parameterDefinition=parameter_definition, ) def get_create_test_suite( executable_entity_reference: str, name: Optional[EntityName] = None, # noqa: UP045 description: Optional[str] = None, # noqa: UP045 ): if not name: name = generate_name().root if not description: description = generate_name().root return CreateTestSuiteRequest( name=TestSuiteEntityName(name), description=Markdown(description), basicEntityReference=FullyQualifiedEntityName(executable_entity_reference), ) def get_create_test_case( entity_link: str, test_definition: FullyQualifiedEntityName, parameter_values: List[TestCaseParameterValue], # noqa: UP006 name: Optional[EntityName] = None, # noqa: UP045 ): if not name: name = generate_name().root return CreateTestCaseRequest( name=TestCaseEntityName(name), entityLink=EntityLink(entity_link), testDefinition=test_definition, parameterValues=parameter_values, ) def get_test_dag(name: str) -> "DAG": """Get a DAG with the tasks created in the CreatePipelineRequest""" from airflow import DAG from airflow.operators.bash import BashOperator with DAG(name, start_date=datetime(2021, 1, 1)) as dag: tasks = [ BashOperator( task_id=task_id, bash_command="date", ) for task_id in ("task1", "task2", "task3", "task4") ] tasks[0] >> tasks[1] >> [tasks[2], tasks[3]] return dag