chore: import upstream snapshot with attribution

scripts/convert/elements_json_to_format.py

@@ -0,0 +1,99 @@
+import argparse
+import logging
+import os
+from pathlib import Path
+
+from unstructured.partition.html.convert import elements_to_html
+from unstructured.staging.base import elements_from_json, elements_to_md
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def json_to_format(
+    filepath: Path,
+    outdir: Path,
+    format_type: str,
+    exclude_binary_image_data: bool,
+    no_group_by_page: bool,
+):
+    logger.info("Processing: %s", filepath)
+    elements = elements_from_json(str(filepath))
+
+    if format_type == "html":
+        output_content = elements_to_html(elements, exclude_binary_image_data, no_group_by_page)
+        file_extension = ".html"
+    elif format_type == "markdown":
+        output_content = elements_to_md(
+            elements, exclude_binary_image_data=exclude_binary_image_data
+        )
+        file_extension = ".md"
+    else:
+        raise ValueError(f"Unsupported format: {format_type}. Supported formats: html, markdown")
+
+    outpath = outdir / filepath.with_suffix(file_extension).name
+    os.makedirs(outpath.parent, exist_ok=True)
+    with open(outpath, "w+") as f:
+        f.write(output_content)
+    logger.info(f"{format_type.upper()} rendered and saved to: %s", outpath)
+
+
+def multiple_json_to_format(
+    path: Path,
+    outdir: Path,
+    format_type: str,
+    exclude_binary_image_data: bool,
+    no_group_by_page: bool,
+):
+    for root, _, files in os.walk(path):
+        for file in files:
+            if file.endswith(".json"):
+                json_file_path = Path(root) / file
+                outpath = outdir / json_file_path.relative_to(path).parent
+                json_to_format(
+                    json_file_path,
+                    outpath,
+                    format_type,
+                    exclude_binary_image_data,
+                    no_group_by_page,
+                )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert JSON elements to HTML or Markdown.")
+    parser.add_argument(
+        "filepath",
+        type=str,
+        help="""Path to the JSON file or directory containing elements.
+        If given directory it will convert all JSON files in directory
+        and all sub-directories.""",
+    )
+    parser.add_argument(
+        "--outdir", type=str, help="Output directory for the output file.", default=""
+    )
+    parser.add_argument(
+        "--format",
+        type=str,
+        choices=["html", "markdown"],
+        default="html",
+        help="Output format: html or markdown (default: html)",
+    )
+    parser.add_argument(
+        "--exclude-img", action="store_true", help="Exclude binary image data from the output."
+    )
+    parser.add_argument(
+        "--no-group", action="store_true", help="Don't group elements by pages (HTML only)."
+    )
+    args = parser.parse_args()
+
+    filepath = Path(args.filepath)
+    outdir = Path(args.outdir)
+
+    if filepath.is_file():
+        json_to_format(filepath, outdir, args.format, args.exclude_img, args.no_group)
+    else:
+        multiple_json_to_format(filepath, outdir, args.format, args.exclude_img, args.no_group)
+
+
+if __name__ == "__main__":
+    main()

scripts/convert/rendered_html_from_elements.py

@@ -0,0 +1,92 @@
+# pyright: reportPrivateUsage=false
+
+"""
+Script to render HTML from unstructured elements.
+NOTE: This script is not intended to be used as a module.
+NOTE: For now script is only intended to be used with elements generated with
+      `partition_html(html_parser_version=v2)`
+TODO: It was noted that unstructured_elements_to_ontology func always returns a single page
+      This script is using helper functions to handle multiple pages.
+"""
+
+import argparse
+import html
+import logging
+import os
+import select
+import sys
+
+from unstructured.partition.html.transformations import unstructured_elements_to_ontology
+from unstructured.staging.base import elements_from_json
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def rendered_html(*, filepath: str | None = None, text: str | None = None) -> str:
+    """Renders HTML from a JSON file with unstructured elements.
+
+    Args:
+        filepath (str): path to JSON file with unstructured elements.
+
+    Returns:
+        str: HTML content.
+    """
+    if filepath is None and text is None:
+        logger.error("Either filepath or text must be provided.")
+        raise ValueError("Either filepath or text must be provided.")
+    if filepath is not None and text is not None:
+        logger.error("Both filepath and text cannot be provided.")
+        raise ValueError("Both filepath and text cannot be provided.")
+    if filepath is not None:
+        logger.info("Rendering HTML from file: %s", filepath)
+    else:
+        logger.info("Rendering HTML from text.")
+
+    unstructured_elements = elements_from_json(filename=filepath, text=text)
+    ontology_root = unstructured_elements_to_ontology(unstructured_elements)
+    html_document = ontology_root.to_html()
+    unescaped_html = html.unescape(html_document)
+    return unescaped_html
+
+
+def _main():
+    if os.getenv("PROCESS_FROM_STDIN") == "true":
+        logger.info("Processing from STDIN (PROCESS_FROM_STDIN is set to 'true')")
+        if select.select([sys.stdin], [], [], 0.1)[0]:
+            content = sys.stdin.read()
+            html = rendered_html(text=content)
+            sys.stdout.write(html)
+        else:
+            logger.error("No input provided via STDIN. Exiting.")
+            sys.exit(1)
+    else:
+        logger.info("Processing from command line arguments")
+        parser = argparse.ArgumentParser(description="Render HTML from unstructured elements.")
+        parser.add_argument(
+            "filepath", help="Path to JSON file with unstructured elements.", type=str
+        )
+        parser.add_argument(
+            "--outdir",
+            help="Path to directory where the rendered html will be stored.",
+            type=str,
+            default=None,
+            nargs="?",
+        )
+        args = parser.parse_args()
+
+        html = rendered_html(filepath=args.filepath)
+        if args.outdir is None:
+            args.outdir = os.path.dirname(args.filepath)
+        os.makedirs(args.outdir, exist_ok=True)
+        outpath = os.path.join(
+            args.outdir, os.path.basename(args.filepath).replace(".json", ".html")
+        )
+        with open(outpath, "w") as f:
+            f.write(html)
+        logger.info("HTML rendered and saved to: %s", outpath)
+
+
+if __name__ == "__main__":
+    _main()