microsoft--promptflow/scripts/readme/workflow_generator.py

import os
import glob
import argparse
from pathlib import Path
import ntpath
import re
import hashlib
import json
from jinja2 import Environment, FileSystemLoader
from ghactions_driver.readme_step import ReadmeStepsManage
from ghactions_driver.resource_resolver import resolve_tutorial_resource
from ghactions_driver.telemetry_obj import Telemetry


def format_ipynb(notebooks):
    # run code formatter on .ipynb files
    for notebook in notebooks:
        os.system(f"black-nb --clear-output {notebook}")


def _get_paths(paths_list):
    """
    Convert the path list to unix format.
    :param paths_list: The input path list.
    :returns: The same list with unix-like paths.
    """
    paths_list.sort()
    if ntpath.sep == os.path.sep:
        return [pth.replace(ntpath.sep, "/") for pth in paths_list]
    return paths_list


def write_notebook_workflow(notebook, name, output_telemetry=Telemetry()):
    temp_name_list = re.split(r"/|\.", notebook)
    temp_name_list = [
        x
        for x in temp_name_list
        if x != "tutorials" and x != "examples" and x != "ipynb"
    ]
    temp_name_list = [x.replace("-", "") for x in temp_name_list]
    workflow_name = "_".join(["samples"] + temp_name_list)

    place_to_write = (
        Path(ReadmeStepsManage.git_base_dir())
        / ".github"
        / "workflows"
        / f"{workflow_name}.yml"
    )

    gh_working_dir = "/".join(notebook.split("/")[:-1])
    env = Environment(
        loader=FileSystemLoader("./scripts/readme/ghactions_driver/workflow_templates")
    )
    template = env.get_template("basic_workflow.yml.jinja2")

    # Schedule notebooks at different times to reduce maximum quota usage.
    name_hash = int(hashlib.sha512(workflow_name.encode()).hexdigest(), 16)
    schedule_minute = name_hash % 60
    schedule_hour = (name_hash // 60) % 4 + 19  # 19-22 UTC

    notebook_path = Path(ReadmeStepsManage.git_base_dir()) / str(notebook)
    try:
        # resolve tutorial resources
        path_filter = resolve_tutorial_resource(workflow_name, notebook_path.resolve(), output_telemetry)
    except Exception:
        if "examples/tutorials" in gh_working_dir:
            raise
        else:
            pass
    if "samples_configuration" in workflow_name:
        # exception, samples configuration is very simple and not related to other prompt flow examples
        path_filter = (
            "[ examples/configuration.ipynb, .github/workflows/samples_configuration.yml ]"
        )
    else:
        path_filter = f"[ {gh_working_dir}/**, examples/*requirements.txt, .github/workflows/{workflow_name}.yml ]"

    # these workflows require config.json to init PF/ML client
    workflows_require_config_json = [
        "configuration",
        "runflowwithpipeline",
        "quickstartazure",
        "cloudrunmanagement",
        "chatwithclassbasedflowazure",
    ]
    if any(keyword in workflow_name for keyword in workflows_require_config_json):
        template = env.get_template("workflow_config_json.yml.jinja2")
    elif "chatwithpdf" in workflow_name:
        template = env.get_template("pdf_workflow.yml.jinja2")
    elif "flowasfunction" in workflow_name:
        template = env.get_template("flow_as_function.yml.jinja2")
    elif "traceautogengroupchat" in workflow_name:
        template = env.get_template("autogen_workflow.yml.jinja2")

    content = template.render(
        {
            "workflow_name": workflow_name,
            "ci_name": "samples_notebook_ci",
            "name": name,
            "gh_working_dir": gh_working_dir,
            "path_filter": path_filter,
            "crontab": f"{schedule_minute} {schedule_hour} * * *",
            "crontab_comment": f"Every day starting at {schedule_hour - 16}:{schedule_minute} BJT",
        }
    )

    # To customize workflow, add new steps in steps.py
    # make another function for special cases.
    with open(place_to_write.resolve(), "w") as f:
        f.write(content)
    print(f"Write workflow: {place_to_write.resolve()}")
    output_telemetry.workflow_name = workflow_name
    output_telemetry.name = name
    output_telemetry.gh_working_dir = gh_working_dir
    output_telemetry.path_filter = path_filter


def write_workflows(notebooks, output_telemetries=[]):
    # process notebooks
    for notebook in notebooks:
        # get notebook name
        output_telemetry = Telemetry()
        nb_path = Path(notebook)
        name, _ = os.path.splitext(nb_path.parts[-1])

        # write workflow file
        write_notebook_workflow(notebook, name, output_telemetry)
        output_telemetry.notebook = nb_path
        output_telemetries.append(output_telemetry)


def local_filter(callback, array):
    results = []
    for index, item in enumerate(array):
        result = callback(item, index, array)
        # if returned true, append item to results
        if result:
            results.append(item)
    return results


def no_readme_generation_filter(item, index, array) -> bool:
    """
    Set each ipynb metadata no_readme_generation to "true" to skip readme generation
    """
    try:
        if item.endswith("test.ipynb"):
            return False
        if "examples/flows/integrations/" in item:
            return False
        # read in notebook
        with open(item, "r", encoding="utf-8") as f:
            data = json.load(f)
        try:
            if data["metadata"]["no_readme_generation"] is not None:
                # no_readme_generate == "true", then no generation
                return data["metadata"]["no_readme_generation"] != "true"
        except Exception:
            return True  # generate readme
    except Exception:
        return False  # not generate readme


def main(input_glob, output_files=[], check=False):
    # get list of workflows

    notebooks = _get_paths(
        [j for i in [glob.glob(p, recursive=True) for p in input_glob] for j in i]
    )

    # check each workflow, get metadata.
    notebooks = local_filter(no_readme_generation_filter, notebooks)

    # format code
    if not check:
        format_ipynb(notebooks)

    # write workflows
    write_workflows(notebooks, output_files)


# run functions
if __name__ == "__main__":
    # setup argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-g", "--input-glob", nargs="+", help="Input glob example 'examples/**/*.ipynb'"
    )
    args = parser.parse_args()

    # call main
    main(input_glob=args.input_glob)