wehub-resource-sync
bf2343b7e4
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
285 lines
12 KiB
Python
285 lines
12 KiB
Python
from unittest.mock import Mock, patch
|
|
|
|
import pytest
|
|
|
|
from metadata.data_quality.validations.models import (
|
|
TableDiffRuntimeParameters,
|
|
TableParameter,
|
|
)
|
|
from metadata.data_quality.validations.table.sqlalchemy.tableDiff import (
|
|
TableDiffValidator,
|
|
compile_and_clauses,
|
|
)
|
|
from metadata.generated.schema.entity.data.table import (
|
|
Column,
|
|
DataType,
|
|
TableProfilerConfig,
|
|
)
|
|
from metadata.generated.schema.entity.services.databaseService import (
|
|
DatabaseServiceType,
|
|
)
|
|
from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
|
|
from metadata.generated.schema.type.basic import ProfileSampleType
|
|
from metadata.generated.schema.type.samplingConfig import ProfileSampleConfig
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"elements, expected",
|
|
[
|
|
("a", "a"),
|
|
(["a", "b"], "a and b"),
|
|
(["a", ["b", "c"]], "a and (b and c)"),
|
|
(["a", ["b", ["c", "d"]]], "a and (b and (c and d))"),
|
|
(["a", ["b", "c"], "d"], "a and (b and c) and d"),
|
|
([], ""),
|
|
("", ""),
|
|
(["a"], "a"),
|
|
([["a"]], "a"),
|
|
([["a"]], "a"),
|
|
],
|
|
)
|
|
def test_compile_and_clauses(elements, expected):
|
|
assert compile_and_clauses(elements) == expected
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"config,expected",
|
|
[
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": "BigQuery",
|
|
"table_profile_config": TableProfilerConfig(
|
|
profileSampleConfig=ProfileSampleConfig(
|
|
sampleConfigType="STATIC",
|
|
config={
|
|
"profileSample": 10,
|
|
"profileSampleType": "PERCENTAGE",
|
|
},
|
|
),
|
|
),
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"keyColumns": ["id"],
|
|
}
|
|
),
|
|
("SUBSTRING(MD5(id || 'a'), 1, 8) < '19999999'",) * 2,
|
|
),
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": "BigQuery",
|
|
"table_profile_config": TableProfilerConfig(
|
|
profileSampleConfig=ProfileSampleConfig(
|
|
sampleConfigType="STATIC",
|
|
config={
|
|
"profileSample": 20,
|
|
"profileSampleType": "PERCENTAGE",
|
|
},
|
|
),
|
|
),
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"keyColumns": ["id"],
|
|
}
|
|
),
|
|
("SUBSTRING(MD5(id || 'a'), 1, 8) < '33333333'",) * 2,
|
|
),
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": "BigQuery",
|
|
"table_profile_config": TableProfilerConfig(
|
|
profileSampleConfig=ProfileSampleConfig(
|
|
sampleConfigType="STATIC",
|
|
config={
|
|
"profileSample": 10,
|
|
"profileSampleType": "PERCENTAGE",
|
|
},
|
|
),
|
|
),
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id", "name"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id", "name"],
|
|
}
|
|
),
|
|
"keyColumns": ["id", "name"],
|
|
}
|
|
),
|
|
("SUBSTRING(MD5(id || name || 'a'), 1, 8) < '19999999'",) * 2,
|
|
),
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": "BigQuery",
|
|
"table_profile_config": TableProfilerConfig(
|
|
profileSampleConfig=ProfileSampleConfig(
|
|
sampleConfigType="STATIC",
|
|
config={
|
|
"profileSample": 20,
|
|
"profileSampleType": "ROWS",
|
|
},
|
|
),
|
|
),
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id", "name"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id", "name"],
|
|
},
|
|
),
|
|
"keyColumns": ["id", "name"],
|
|
}
|
|
),
|
|
("SUBSTRING(MD5(id || name || 'a'), 1, 8) < '0083126e'",) * 2,
|
|
),
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"table_profile_config": TableProfilerConfig(
|
|
profileSampleConfig=ProfileSampleConfig(
|
|
sampleConfigType="STATIC",
|
|
config={
|
|
"profileSample": 20,
|
|
"profileSampleType": "ROWS",
|
|
},
|
|
),
|
|
),
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="ID", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
},
|
|
),
|
|
"keyColumns": ["id"],
|
|
}
|
|
),
|
|
(
|
|
"SUBSTRING(MD5(id || 'a'), 1, 8) < '0083126e'",
|
|
"SUBSTRING(MD5(\"ID\" || 'a'), 1, 8) < '0083126e'",
|
|
),
|
|
),
|
|
(
|
|
TableDiffRuntimeParameters.model_construct(
|
|
**{ # noqa: PIE804
|
|
"table_profile_config": None,
|
|
"table1": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
}
|
|
),
|
|
"table2": TableParameter.model_construct(
|
|
**{ # noqa: PIE804
|
|
"database_service_type": DatabaseServiceType.Postgres,
|
|
"columns": [
|
|
Column(name="id", dataType=DataType.STRING),
|
|
Column(name="name", dataType=DataType.STRING),
|
|
],
|
|
"key_columns": ["id"],
|
|
},
|
|
),
|
|
"keyColumns": ["id"],
|
|
}
|
|
),
|
|
(None, None),
|
|
),
|
|
],
|
|
)
|
|
def test_sample_where_clauses(config, expected):
|
|
validator = TableDiffValidator(
|
|
None,
|
|
TestCase.model_construct(parameterValues=[TestCaseParameterValue(name="caseSensitiveColumns", value="false")]),
|
|
None,
|
|
)
|
|
validator.runtime_params = config
|
|
table_profile_config = config.table_profile_config if config else None
|
|
profile_sample_config = table_profile_config.profileSampleConfig.root if table_profile_config else None
|
|
sample_config = profile_sample_config.config if profile_sample_config else None
|
|
if sample_config and sample_config.profileSampleType == ProfileSampleType.ROWS:
|
|
validator.get_total_row_count = Mock(return_value=10_000)
|
|
with patch("random.choices", Mock(return_value=["a"])):
|
|
assert validator.sample_where_clause() == expected
|