chore: import upstream snapshot with attribution

ingestion/tests/integration/sql_server/conftest.py

@@ -0,0 +1,168 @@
+import os
+import shutil
+import tempfile
+import uuid
+
+import pytest
+from sqlalchemy import create_engine, text
+from testcontainers.mssql import SqlServerContainer
+
+from _openmetadata_testutils.helpers.docker import copy_dir_to_container, try_bind
+from metadata.generated.schema.api.services.createDatabaseService import (
+    CreateDatabaseServiceRequest,
+)
+from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
+    MssqlConnection,
+    MssqlScheme,
+)
+from metadata.generated.schema.entity.services.databaseService import (
+    DatabaseConnection,
+    DatabaseService,
+    DatabaseServiceType,
+)
+
+from ..conftest import ingestion_config as base_ingestion_config  # noqa: F401, TID252
+
+
+@pytest.fixture(scope="package")
+def db_name():
+    return "AdventureWorksLT2022"
+
+
+class CustomSqlServerContainer(SqlServerContainer):
+    def start(self) -> "DbContainer":  # noqa: F821
+        dockerfile = f"""
+            FROM {self.image}
+            USER root
+            RUN mkdir -p /data
+            RUN chown mssql /data
+            USER mssql
+            """
+        temp_dir = os.path.join(tempfile.gettempdir(), "mssql")  # noqa: PTH118
+        os.makedirs(temp_dir, exist_ok=True)  # noqa: PTH103
+        temp_dockerfile_path = os.path.join(temp_dir, "Dockerfile")  # noqa: PTH118
+        with open(temp_dockerfile_path, "w") as temp_dockerfile:  # noqa: PTH123
+            temp_dockerfile.write(dockerfile)
+        self.get_docker_client().build(temp_dir, tag=self.image)
+        return super().start()
+
+    def _configure(self) -> None:
+        super()._configure()
+        self.with_env("SQL_SA_PASSWORD", self.password)
+
+
+@pytest.fixture(scope="package")
+def mssql_container(tmp_path_factory, db_name):
+    container = CustomSqlServerContainer("mcr.microsoft.com/mssql/server:2022-latest", dbname="master")
+    data_dir = tmp_path_factory.mktemp("data")
+    shutil.copy(
+        os.path.join(os.path.dirname(__file__), "data", f"{db_name}.bak"),  # noqa: PTH118, PTH120
+        str(data_dir),
+    )
+    with open(data_dir / "install.sql", "w") as f:  # noqa: PTH123
+        f.write(
+            f"""
+USE [master]
+RESTORE FILELISTONLY
+    FROM DISK = '/data/{db_name}.bak';
+GO
+
+RESTORE DATABASE [{db_name}]
+    FROM DISK = '/data/{db_name}.bak'
+    WITH MOVE '{db_name}_Data' TO '/var/opt/mssql/data/{db_name}.mdf',
+         MOVE '{db_name}_Log' TO '/var/opt/mssql/data/{db_name}.ldf';
+GO
+        """
+        )
+
+    with try_bind(container, 1433, 1433) as container:
+        docker_container = container.get_wrapped_container()
+        copy_dir_to_container(str(data_dir), docker_container, "/data")
+        res = docker_container.exec_run(
+            [
+                "bash",
+                "-c",
+                " ".join(
+                    [
+                        "/opt/mssql-tools*/bin/sqlcmd",
+                        "-U",
+                        container.username,
+                        "-P",
+                        f"'{container.password}'",
+                        "-d",
+                        "master",
+                        "-i",
+                        "/data/install.sql",
+                        "-C",
+                    ]
+                ),
+            ]
+        )
+        if res[0] != 0:
+            raise Exception("Failed to create mssql database:" + res[1].decode("utf-8"))  # noqa: TRY002
+        engine = create_engine(
+            "mssql+pytds://" + container.get_connection_url().split("://")[1],
+            connect_args={"autocommit": True},
+        )
+        with engine.connect() as conn:
+            transaciton = conn.begin()
+            conn.execute(text(f"SELECT * INTO {db_name}.SalesLT.CustomerCopy FROM {db_name}.SalesLT.Customer;"))
+            transaciton.commit()
+        yield container
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        MssqlScheme.mssql_pytds,
+        MssqlScheme.mssql_pyodbc,
+    ],
+)
+def scheme(request):
+    return request.param
+
+
+@pytest.fixture(scope="module")
+def create_service_request(mssql_container, scheme, db_name):
+    return CreateDatabaseServiceRequest(
+        name=f"docker_test_mssql_{uuid.uuid4().hex[:8]}_{scheme.name}",
+        serviceType=DatabaseServiceType.Mssql,
+        connection=DatabaseConnection(
+            config=MssqlConnection(
+                username=mssql_container.username,
+                password=mssql_container.password,
+                hostPort="localhost:" + mssql_container.get_exposed_port(mssql_container.port),
+                database=db_name,
+                scheme=scheme,
+                ingestAllDatabases=True,
+                connectionOptions={
+                    "TrustServerCertificate": "yes",
+                    "MARS_Connection": "yes",
+                },
+            )
+        ),
+    )
+
+
+@pytest.fixture(scope="module")
+def ingestion_config(
+    db_service,
+    tmp_path_factory,
+    workflow_config,
+    sink_config,
+    base_ingestion_config,  # noqa: F811
+    db_name,
+):
+    base_ingestion_config["source"]["sourceConfig"]["config"]["databaseFilterPattern"] = {
+        "includes": ["TestDB", db_name],
+    }
+    return base_ingestion_config
+
+
+@pytest.fixture(scope="module")
+def unmask_password(create_service_request):
+    def inner(service: DatabaseService):
+        service.connection.config.password = create_service_request.connection.config.password
+        return service
+
+    return inner

ingestion/tests/integration/sql_server/test_lineage.py

@@ -0,0 +1,61 @@
+import pytest
+from freezegun import freeze_time
+from sqlalchemy import create_engine, text
+
+from metadata.generated.schema.entity.data.table import Table
+from metadata.ingestion.lineage.sql_lineage import search_cache
+from metadata.workflow.metadata import MetadataWorkflow
+
+
+@pytest.fixture(
+    params=["german", "english"],  # test for both languages
+)
+def language_config(mssql_container, request):
+    language = request.param
+    engine = create_engine(
+        "mssql+pytds://" + mssql_container.get_connection_url().split("://")[1],
+        connect_args={"autocommit": True},
+    )
+    with engine.connect() as conn:
+        conn.execute(text(f"ALTER LOGIN {mssql_container.username} WITH DEFAULT_LANGUAGE={language};"))
+        conn.commit()
+
+
+@pytest.fixture()
+def lineage_config(language_config, db_service, workflow_config, sink_config, db_name):
+    return {
+        "source": {
+            "type": "mssql-lineage",
+            "serviceName": db_service.fullyQualifiedName.root,
+            "sourceConfig": {
+                "config": {
+                    "type": "DatabaseLineage",
+                    "databaseFilterPattern": {"includes": ["TestDB", db_name]},
+                },
+            },
+        },
+        "sink": sink_config,
+        "workflowConfig": workflow_config,
+    }
+
+
+@freeze_time("2024-01-30")  # to demonstrate the issue with german language
+def test_lineage(
+    patch_passwords_for_db_services,
+    run_workflow,
+    ingestion_config,
+    lineage_config,
+    db_service,
+    metadata,
+    db_name,
+):
+    search_cache.clear()
+    run_workflow(MetadataWorkflow, ingestion_config)
+    run_workflow(MetadataWorkflow, lineage_config)
+    department_table = metadata.get_by_name(
+        Table,
+        f"{db_service.fullyQualifiedName.root}.{db_name}.SalesLT.Customer",
+        nullable=False,
+    )
+    lineage = metadata.get_lineage_by_id(Table, department_table.id.root)
+    assert lineage is not None

ingestion/tests/integration/sql_server/test_metadata.py

@@ -0,0 +1,36 @@
+from metadata.generated.schema.entity.data.table import Constraint, Table
+from metadata.workflow.metadata import MetadataWorkflow
+
+
+def test_ingest_metadata(
+    patch_passwords_for_db_services,
+    run_workflow,
+    ingestion_config,
+    db_service,
+    metadata,
+    db_name,
+):
+    run_workflow(MetadataWorkflow, ingestion_config)
+    table: Table = metadata.get_by_name(
+        Table,
+        f"{db_service.fullyQualifiedName.root}.{db_name}.SalesLT.Customer",
+    )
+    assert table is not None
+    assert [c.name.root for c in table.columns] == [
+        "CustomerID",
+        "NameStyle",
+        "Title",
+        "FirstName",
+        "MiddleName",
+        "LastName",
+        "Suffix",
+        "CompanyName",
+        "SalesPerson",
+        "EmailAddress",
+        "Phone",
+        "PasswordHash",
+        "PasswordSalt",
+        "rowguid",
+        "ModifiedDate",
+    ]
+    assert table.columns[0].constraint == Constraint.PRIMARY_KEY

ingestion/tests/integration/sql_server/test_profiler.py

@@ -0,0 +1,9 @@
+from metadata.ingestion.lineage.sql_lineage import search_cache
+from metadata.workflow.metadata import MetadataWorkflow
+from metadata.workflow.profiler import ProfilerWorkflow
+
+
+def test_profiler(patch_passwords_for_db_services, run_workflow, ingestion_config, profiler_config):
+    search_cache.clear()
+    run_workflow(MetadataWorkflow, ingestion_config)
+    run_workflow(ProfilerWorkflow, profiler_config)

ingestion/tests/integration/sql_server/test_usage.py

@@ -0,0 +1,38 @@
+import pytest
+
+from metadata.workflow.metadata import MetadataWorkflow
+from metadata.workflow.usage import UsageWorkflow
+
+
+@pytest.fixture()
+def usage_config(db_service, workflow_config, db_name):
+    return {
+        "source": {
+            "type": "mssql-usage",
+            "serviceName": db_service.fullyQualifiedName.root,
+            "sourceConfig": {
+                "config": {
+                    "queryLogDuration": 2,
+                    "resultLimit": 1000,
+                    "databaseFilterPattern": {"includes": ["TestDB", db_name]},
+                },
+            },
+        },
+        "processor": {"type": "query-parser", "config": {}},
+        "stage": {"type": "table-usage", "config": {"filename": "/tmp/mssql_usage"}},
+        "bulkSink": {
+            "type": "metadata-usage",
+            "config": {"filename": "/tmp/mssql_usage"},
+        },
+        "workflowConfig": workflow_config,
+    }
+
+
+def test_usage(
+    patch_passwords_for_db_services,
+    run_workflow,
+    ingestion_config,
+    usage_config,
+):
+    run_workflow(MetadataWorkflow, ingestion_config)
+    run_workflow(UsageWorkflow, usage_config)