wehub/open-metadata--openmetadata
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Details
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Details
Java Checkstyle / java-checkstyle (push) Has been cancelled
Details
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Details
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
Details
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:35:45 +08:00
commit bf2343b7e4
16049 changed files with 3531137 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ingestion/tests/unit/metadata/data_quality/test_data_diff.py
+284
View File
@@ -0,0 +1,284 @@
from unittest.mock import Mock, patch
import pytest
from metadata.data_quality.validations.models import (
TableDiffRuntimeParameters,
TableParameter,
)
from metadata.data_quality.validations.table.sqlalchemy.tableDiff import (
TableDiffValidator,
compile_and_clauses,
)
from metadata.generated.schema.entity.data.table import (
Column,
DataType,
TableProfilerConfig,
)
from metadata.generated.schema.entity.services.databaseService import (
DatabaseServiceType,
)
from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
from metadata.generated.schema.type.basic import ProfileSampleType
from metadata.generated.schema.type.samplingConfig import ProfileSampleConfig
@pytest.mark.parametrize(
"elements, expected",
[
("a", "a"),
(["a", "b"], "a and b"),
(["a", ["b", "c"]], "a and (b and c)"),
(["a", ["b", ["c", "d"]]], "a and (b and (c and d))"),
(["a", ["b", "c"], "d"], "a and (b and c) and d"),
([], ""),
("", ""),
(["a"], "a"),
([["a"]], "a"),
([["a"]], "a"),
],
)
def test_compile_and_clauses(elements, expected):
assert compile_and_clauses(elements) == expected
@pytest.mark.parametrize(
"config,expected",
[
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"database_service_type": "BigQuery",
"table_profile_config": TableProfilerConfig(
profileSampleConfig=ProfileSampleConfig(
sampleConfigType="STATIC",
config={
"profileSample": 10,
"profileSampleType": "PERCENTAGE",
},
),
),
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"keyColumns": ["id"],
}
),
("SUBSTRING(MD5(id || 'a'), 1, 8) < '19999999'",) * 2,
),
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"database_service_type": "BigQuery",
"table_profile_config": TableProfilerConfig(
profileSampleConfig=ProfileSampleConfig(
sampleConfigType="STATIC",
config={
"profileSample": 20,
"profileSampleType": "PERCENTAGE",
},
),
),
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"keyColumns": ["id"],
}
),
("SUBSTRING(MD5(id || 'a'), 1, 8) < '33333333'",) * 2,
),
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"database_service_type": "BigQuery",
"table_profile_config": TableProfilerConfig(
profileSampleConfig=ProfileSampleConfig(
sampleConfigType="STATIC",
config={
"profileSample": 10,
"profileSampleType": "PERCENTAGE",
},
),
),
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id", "name"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id", "name"],
}
),
"keyColumns": ["id", "name"],
}
),
("SUBSTRING(MD5(id || name || 'a'), 1, 8) < '19999999'",) * 2,
),
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"database_service_type": "BigQuery",
"table_profile_config": TableProfilerConfig(
profileSampleConfig=ProfileSampleConfig(
sampleConfigType="STATIC",
config={
"profileSample": 20,
"profileSampleType": "ROWS",
},
),
),
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id", "name"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id", "name"],
},
),
"keyColumns": ["id", "name"],
}
),
("SUBSTRING(MD5(id || name || 'a'), 1, 8) < '0083126e'",) * 2,
),
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"table_profile_config": TableProfilerConfig(
profileSampleConfig=ProfileSampleConfig(
sampleConfigType="STATIC",
config={
"profileSample": 20,
"profileSampleType": "ROWS",
},
),
),
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="ID", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
},
),
"keyColumns": ["id"],
}
),
(
"SUBSTRING(MD5(id || 'a'), 1, 8) < '0083126e'",
"SUBSTRING(MD5(\"ID\" || 'a'), 1, 8) < '0083126e'",
),
),
(
TableDiffRuntimeParameters.model_construct(
**{ # noqa: PIE804
"table_profile_config": None,
"table1": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
}
),
"table2": TableParameter.model_construct(
**{ # noqa: PIE804
"database_service_type": DatabaseServiceType.Postgres,
"columns": [
Column(name="id", dataType=DataType.STRING),
Column(name="name", dataType=DataType.STRING),
],
"key_columns": ["id"],
},
),
"keyColumns": ["id"],
}
),
(None, None),
),
],
)
def test_sample_where_clauses(config, expected):
validator = TableDiffValidator(
None,
TestCase.model_construct(parameterValues=[TestCaseParameterValue(name="caseSensitiveColumns", value="false")]),
None,
)
validator.runtime_params = config
table_profile_config = config.table_profile_config if config else None
profile_sample_config = table_profile_config.profileSampleConfig.root if table_profile_config else None
sample_config = profile_sample_config.config if profile_sample_config else None
if sample_config and sample_config.profileSampleType == ProfileSampleType.ROWS:
validator.get_total_row_count = Mock(return_value=10_000)
with patch("random.choices", Mock(return_value=["a"])):
assert validator.sample_where_clause() == expected