chore: import upstream snapshot with attribution

ingestion/tests/unit/workflow/test_progress_rendering.py

@@ -0,0 +1,354 @@
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""One snapshot -> CLI tree string + nested progressNode payload."""
+
+from unittest.mock import patch
+
+from metadata.generated.schema.entity.services.ingestionPipelines.progressUpdate import (
+    ProgressNode,
+    ProgressUpdate,
+)
+from metadata.ingestion.progress import (
+    format_eta,
+    render_progress_tree,
+    snapshot_to_progress_payload,
+)
+from metadata.ingestion.progress.registry import ProgressNodeSnapshot, ProgressRegistry
+
+
+def _tree_snapshot():
+    reg = ProgressRegistry()
+    reg.open([], "Database", None)
+    reg.open(["xyz"], "DatabaseSchema", 10)
+    reg.open(["xyz", "abc"], "Table", 100)
+    for _ in range(20):
+        reg.advance(["xyz", "abc"])
+    return reg.snapshot()
+
+
+class TestReporterAssetsIngested:
+    def test_counts_all_leaf_types(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.open(["db"], "DatabaseSchema", 1)
+        reg.open(["db", "sch"], "Table", 3)
+        reg.open(["db", "sch"], "StoredProcedure", 2)
+        for _ in range(3):
+            reg.advance(["db", "sch"], "Table")
+        for _ in range(2):
+            reg.advance(["db", "sch"], "StoredProcedure")
+        assert reg.assets_ingested() == 5
+
+    def test_survives_scope_pruning(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.open(["db", "sch"], "Table", 2)
+        reg.advance(["db", "sch"], "Table")
+        reg.advance(["db", "sch"], "Table")
+        reg.close(["db", "sch"])
+        reg.close(["db"])
+        assert reg.assets_ingested() == 2
+
+
+class TestProgressRendering:
+    def test_cli_renders_database_line(self):
+        out = render_progress_tree(_tree_snapshot())
+        assert "Database" in out
+
+    def test_cli_renders_active_table_line_with_denominator(self):
+        out = render_progress_tree(_tree_snapshot())
+        assert any("abc" in line and "20/100" in line for line in out.splitlines())
+
+    def test_cli_handles_none(self):
+        assert render_progress_tree(None) == ""
+
+    def test_payload_is_nested_and_typed(self):
+        payload = snapshot_to_progress_payload(_tree_snapshot())
+        node = ProgressNode.model_validate(payload)  # round-trips through the generated model
+        assert node.entityType == "Database"
+        schema = node.children[0]
+        assert schema.label == "xyz"
+        assert schema.entityType == "DatabaseSchema"
+        table = schema.children[0]
+        assert table.label == "abc"
+        assert table.expected == 100
+        assert table.processed == 20
+
+    def test_payload_is_none_without_progress(self):
+        assert snapshot_to_progress_payload(None) is None
+
+
+def test_label_only_grouping_node_renders_without_count():
+    leaf = ProgressNodeSnapshot(
+        label="",
+        child_type="Table",
+        expected=47,
+        processed=12,
+        active=True,
+        overflow=0,
+        children=(),
+    )
+    schema = ProgressNodeSnapshot(
+        label="analytics.public",
+        child_type=None,
+        expected=None,
+        processed=0,
+        active=True,
+        overflow=0,
+        children=(leaf,),
+    )
+    rendered = render_progress_tree(schema)
+    assert "analytics.public" in rendered
+    assert "analytics.public 0" not in rendered  # no spurious count on the label node
+    assert "Table 12/47" in rendered
+
+
+def _snowflake_like_registry() -> ProgressRegistry:
+    registry = ProgressRegistry()
+    registry.open([], "Database", None)
+    registry.open(["sales"], "DatabaseSchema", None)
+    registry.open(["sales", "public"], "Table", 310)
+    for _ in range(45):
+        registry.advance(["sales", "public"], "Table")
+    return registry
+
+
+class TestRegistryReporting:
+    def test_cli_header_is_assets_ingested(self):
+        registry = _snowflake_like_registry()
+        text = registry.render_cli()
+        assert text.splitlines()[0] == f"Ingested: {registry.assets_ingested():,} assets"
+
+    def test_cli_still_shows_active_scope(self):
+        text = _snowflake_like_registry().render_cli()
+        assert "sales.public" in text
+        assert "45/310" in text
+
+    def test_payload_root_carries_asset_total(self):
+        registry = _snowflake_like_registry()
+        payload = registry.sse_payload()
+        assert set(payload) == {
+            "label",
+            "entityType",
+            "processed",
+            "expected",
+            "active",
+            "overflow",
+            "children",
+        }
+        assert payload["processed"] == registry.assets_ingested()
+        assert payload["expected"] is None
+
+    def test_payload_validates_against_progress_update_model(self):
+        update = ProgressUpdate(
+            runId="r",
+            timestamp=1,
+            updateType="PROCESSING",
+            progress=_snowflake_like_registry().sse_payload(),
+        )
+        assert update.progress is not None
+        assert update.progress.processed == 45
+
+    def test_payload_is_none_before_anything_starts(self):
+        assert ProgressRegistry().sse_payload() is None
+
+    def test_progress_update_carries_global_counters(self):
+        reg = ProgressRegistry()
+        reg.set_total("Workspaces", 10)
+        reg.track("Workspaces")
+        reg.track("Workspaces")
+        reg.track("Workspaces")
+        counters = reg.global_counters()
+        update = ProgressUpdate(
+            runId="r1",
+            timestamp=1,
+            updateType="PROCESSING",
+            globalCounters=[{"entityType": t, "done": d, "total": total} for t, d, total in counters],
+        )
+        assert update.globalCounters[0].entityType == "Workspaces"
+        assert update.globalCounters[0].done == 3
+        assert update.globalCounters[0].total == 10
+
+
+class TestGroupHeader:
+    def test_header_shows_counters_then_assets(self):
+        reg = ProgressRegistry()
+        reg.set_total("Database", 4)
+        reg.seed_scope_total("DatabaseSchema", "db1", 45)
+        reg.track("Database")
+        reg.track("Database")
+        for _ in range(12):
+            reg.track("DatabaseSchema")
+        reg.advance([], "Table")
+        lines = reg.render_cli().splitlines()
+        assert lines[0] == "Database 2/4"
+        assert lines[1].startswith("DatabaseSchema 12/45")
+        assert lines[2] == "Ingested: 1 assets"
+
+    def test_header_unknown_total_renders_bare_count(self):
+        reg = ProgressRegistry()
+        reg.set_total("Workspaces", None)
+        reg.track("Workspaces")
+        assert reg.render_cli() == "Workspaces 1"
+
+    def test_header_renders_with_empty_tree_when_counter_active(self):
+        reg = ProgressRegistry()
+        reg.set_total("Workspaces", 4)
+        for _ in range(4):
+            reg.track("Workspaces")
+        assert reg.render_cli() == "Workspaces 4/4"
+
+    def test_no_group_keeps_legacy_header(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.open(["x"], "Table", 2)
+        reg.advance(["x"], "Table")
+        assert reg.render_cli().splitlines()[0] == "Ingested: 1 assets"
+
+    def test_cli_empty_when_no_progress(self):
+        reg = ProgressRegistry()
+        assert reg.render_cli() == ""
+
+
+class TestGlobalCountersOnSseUpdate:
+    def test_reporter_global_counters_passthrough(self):
+        reg = ProgressRegistry()
+        reg.set_total("Workspaces", 10)
+        reg.track("Workspaces")
+        reg.track("Workspaces")
+        reg.track("Workspaces")
+        assert reg.global_counters() == [("Workspaces", 3, 10)]
+
+
+class TestFormatEta:
+    def test_sub_minute(self):
+        assert format_eta(59) == "~59s"
+
+    def test_minute_boundary(self):
+        assert format_eta(60) == "~1m"
+
+    def test_minutes(self):
+        assert format_eta(360) == "~6m"
+
+    def test_hour_boundary(self):
+        assert format_eta(3600) == "~1h 0m"
+
+    def test_hours_and_minutes(self):
+        assert format_eta(4800) == "~1h 20m"
+
+
+class TestEtaSeconds:
+    def test_warmup_done_zero_is_none(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.set_total("DatabaseSchema", 45)
+        assert reg.eta_seconds() is None
+
+    def test_no_counter_with_total_is_none(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.set_total("Workspaces", None)
+        reg.track("Workspaces")
+        assert reg.eta_seconds() is None
+
+    def test_complete_is_none(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.set_total("DatabaseSchema", 4)
+        for _ in range(4):
+            reg.track("DatabaseSchema")
+        assert reg.eta_seconds() is None
+
+    def test_computable_without_open_when_counter_active(self):
+        reg = ProgressRegistry()
+        with patch("metadata.ingestion.progress.registry.time.monotonic") as clock:
+            clock.return_value = 0.0
+            reg.set_total("DatabaseSchema", 10)
+            for _ in range(2):
+                reg.track("DatabaseSchema")
+            clock.return_value = 60.0
+            assert reg.eta_seconds() == 240
+
+    def test_normal_case_uses_cumulative_rate(self):
+        reg = ProgressRegistry()
+        with patch("metadata.ingestion.progress.registry.time.monotonic") as clock:
+            clock.return_value = 0.0
+            reg.open([], "Database", None)
+            reg.set_total("DatabaseSchema", 10)
+            for _ in range(2):
+                reg.track("DatabaseSchema")
+            clock.return_value = 60.0
+            assert reg.eta_seconds() == 240
+
+    def test_driver_is_last_counter_with_total(self):
+        reg = ProgressRegistry()
+        with patch("metadata.ingestion.progress.registry.time.monotonic") as clock:
+            clock.return_value = 0.0
+            reg.open([], "Database", None)
+            reg.set_total("Database", 4)
+            reg.set_total("DatabaseSchema", 20)
+            reg.track("Database")
+            for _ in range(4):
+                reg.track("DatabaseSchema")
+            clock.return_value = 100.0
+            assert reg.eta_seconds() == 400
+
+
+class TestEtaInHeader:
+    def test_eta_suffix_on_driver_line_only(self):
+        reg = ProgressRegistry()
+        with patch("metadata.ingestion.progress.registry.time.monotonic") as clock:
+            clock.return_value = 0.0
+            reg.open([], "Database", None)
+            reg.set_total("Database", 4)
+            reg.set_total("DatabaseSchema", 45)
+            reg.track("Database")
+            reg.track("Database")
+            for _ in range(9):
+                reg.track("DatabaseSchema")
+            clock.return_value = 120.0
+            lines = reg.render_cli().splitlines()
+        assert lines[0] == "Database 2/4"
+        assert lines[1].startswith("DatabaseSchema 9/45")
+        assert "~8m" in lines[1]
+        assert "~" not in lines[0]
+
+    def test_no_eta_suffix_when_not_computable(self):
+        reg = ProgressRegistry()
+        reg.open([], "Database", None)
+        reg.set_total("DatabaseSchema", 45)
+        assert "~" not in reg.render_cli()
+
+
+class TestHeaderIngestedSuppression:
+    def test_ingested_line_suppressed_when_no_assets(self):
+        registry = ProgressRegistry()
+        registry.set_total("Queries", 100)
+        registry.track("Queries", 12)
+        text = registry.render_cli()
+        assert "Queries 12/100" in text
+        assert "Ingested:" not in text
+
+    def test_totalless_counter_renders_without_denominator_or_eta(self):
+        registry = ProgressRegistry()
+        registry.set_total("LineageRecords", None)
+        registry.track("LineageRecords", 8210)
+        text = registry.render_cli()
+        assert "LineageRecords 8210" in text
+        assert "/" not in text
+        assert "Ingested:" not in text
+
+    def test_ingested_line_kept_when_assets_present(self):
+        registry = ProgressRegistry()
+        registry.open(["db", "schema"], "Table", 3)
+        registry.advance(["db", "schema"], "Table")
+        text = registry.render_cli()
+        assert "Ingested: 1 assets" in text