chore: import upstream snapshot with attribution

scripts/dev-setup/main.py

@@ -0,0 +1,160 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import argparse
+import os
+import site
+import typing
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from test_resources import REGISTERED_TEST_RESOURCES_FUNCTIONS
+from utils import REPO_ROOT_DIR, change_cwd, print_blue, run_cmd
+
+PROMPT_FLOW_PKGS = [
+    "promptflow-tracing",
+    "promptflow-core",
+    "promptflow-devkit",
+    "promptflow-azure",
+    "promptflow[azure]",
+    "promptflow-tools",
+    "promptflow-evals",
+    "promptflow-recording",
+]
+
+
+def set_up_git_hook_scripts(verbose: bool) -> None:
+    run_cmd(["pip", "install", "pre-commit"], verbose=False)  # ensure pre-commit is installed
+    cmd = ["pre-commit", "install"]
+    print_blue("Running `pre-commit install` to set up the git hook scripts")
+    run_cmd(cmd, verbose=verbose)
+
+
+def collect_and_install_from_pyproject() -> None:
+    # collect dependencies from pyproject.toml and install
+    deps = []
+    # cwd is already changed to the package folder in the context
+    # so we can safely open pyproject.toml here
+    with open("pyproject.toml", mode="r", encoding="utf-8") as f:
+        lines = f.readlines()
+        collect_flag = False
+        for line in lines:
+            line = line.strip()
+            if line.startswith("["):
+                if "[tool.poetry.group.dev.dependencies]" in line or "[tool.poetry.group.test.dependencies]" in line:
+                    collect_flag = True
+                    continue
+                else:
+                    collect_flag = False
+            if collect_flag and line:
+                deps.append(line.split("=")[0].strip())
+    cmd = ["pip", "install"] + deps
+    run_cmd(cmd)
+
+
+def inject_pth_file() -> None:
+    # content of the .pth file will be added to `sys.path`
+    # for packages installed from pyproject.toml, this file should already be there
+    # for `promptflow`, inject this, and we can avoid `conda env config vars set`
+    # reference: https://docs.python.org/3/library/site.html
+    site_packages_path = Path(site.getsitepackages()[0])
+    with open(site_packages_path / "promptflow.pth", mode="w", encoding="utf-8") as f:
+        f.write((REPO_ROOT_DIR / "src" / "promptflow").resolve().absolute().as_posix())
+
+
+def install_pkg_editable(pkg: str, verbose: bool, is_vscode: bool = False) -> None:
+    if "[" in pkg:
+        folder_name, extras = pkg.split("[")
+        extras = f"[{extras}"
+    else:
+        folder_name = pkg
+        extras = ""
+    pkg_working_dir = REPO_ROOT_DIR / "src" / folder_name
+    with change_cwd(pkg_working_dir):
+        print_blue(f"- Setting up {pkg}")
+
+        # pip install -e . from pyproject.toml/setup.py
+        print_blue(f"- Installing {pkg} from source")
+        cmd = [sys.executable, "-m", "pip", "install", "--editable", f".{extras}"]
+        run_cmd(cmd, verbose=verbose)
+
+        # dev and test dependencies
+        print_blue(f"- Installing dev and test dependencies for {pkg}")
+        # promptflow folder has "dev_requirements.txt", directly install it
+        if folder_name == "promptflow":
+            cmd = ["pip", "install", "-r", "dev_requirements.txt"]
+            run_cmd(cmd, verbose=verbose)
+            inject_pth_file()
+        # promptflow-tools
+        # reference: https://github.com/microsoft/promptflow/blob/main/src/promptflow-tools/README.dev.md
+        elif pkg == "promptflow-tools":
+            cmd = ["pip", "install", "-r", "requirements.txt"]
+            run_cmd(cmd, verbose=verbose)
+            cmd = ["pip", "install", "pytest", "pytest-mock"]
+            run_cmd(cmd, verbose=verbose)
+        # promptflow-* will have "pyproject.toml", parse and install
+        # we can refine this leveraging `poetry export` later
+        elif os.path.exists("pyproject.toml"):
+            collect_and_install_from_pyproject()
+
+            # touch __init__.py for the package for VS Code
+            # NOTE that this is a workaround to enable VS Code to recognize the namespace package
+            #      we should be able to remove this after we fully deprecate promptflow in local development
+            if is_vscode:
+                with open(pkg_working_dir / "promptflow" / "__init__.py", mode="w", encoding="utf-8") as f:
+                    f.write("__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n")
+
+
+@dataclass
+class Arguments:
+    packages: typing.List[str]
+    verbose: bool = False
+    vscode: bool = False
+
+    @staticmethod
+    def parse_args() -> "Arguments":
+        epilog = "Example usage: main.py --packages promptflow-tracing promptflow-core"
+        parser = argparse.ArgumentParser(
+            description="Welcome to promptflow dev setup!",
+            epilog=epilog,
+        )
+        parser.add_argument(
+            "--packages",
+            required=False,
+            nargs="+",
+            type=str,
+            default=PROMPT_FLOW_PKGS,
+            help="packages to install in editable mode",
+        )
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            action="store_true",
+            help="turn on verbose output",
+        )
+        parser.add_argument(
+            "--vscode",
+            action="store_true",
+            help="extra setup step for Visual Studio Code",
+        )
+        args = parser.parse_args()
+        return Arguments(
+            packages=args.packages,
+            verbose=args.verbose,
+            vscode=args.vscode,
+        )
+
+
+def main(args: Arguments) -> None:
+    for pkg in args.packages:
+        install_pkg_editable(pkg, verbose=args.verbose, is_vscode=args.vscode)
+        # invoke test resource template creation function if available
+        if pkg in REGISTERED_TEST_RESOURCES_FUNCTIONS:
+            REGISTERED_TEST_RESOURCES_FUNCTIONS[pkg]()
+    set_up_git_hook_scripts(verbose=args.verbose)
+
+
+if __name__ == "__main__":
+    main(args=Arguments.parse_args())

scripts/dev-setup/test_resources.py

@@ -0,0 +1,76 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import json
+import shutil
+from pathlib import Path
+
+from utils import REPO_ROOT_DIR, print_yellow
+
+
+def _prompt_user_for_test_resources(path: Path) -> None:
+    prompt_msg = (
+        f"Created test-required file {path.name!r} at {path.as_posix()!r}, "
+        "please update with your test resource(s)."
+    )
+    print_yellow(prompt_msg)
+
+
+def create_tracing_test_resource_template() -> None:
+    working_dir = REPO_ROOT_DIR / "src" / "promptflow-tracing"
+    connections_filename = "connections.json"
+    connections_file_path = (working_dir / connections_filename).resolve().absolute()
+    connections_template = {
+        "azure_open_ai_connection": {
+            "value": {
+                "api_key": "aoai-api-key",
+                "api_base": "aoai-api-endpoint",
+                "api_version": "2023-07-01-preview",
+            }
+        }
+    }
+    with open(connections_file_path, mode="w", encoding="utf-8") as f:
+        json.dump(connections_template, f, ensure_ascii=False, indent=4)
+    _prompt_user_for_test_resources(connections_file_path)
+
+
+def create_evals_test_resource_template() -> None:
+    working_dir = REPO_ROOT_DIR / "src" / "promptflow-evals"
+    connections_filename = "connections.json"
+    connections_file_path = (working_dir / connections_filename).resolve().absolute()
+    connections_template = {
+        "azure_openai_model_config": {
+            "value": {
+                "azure_endpoint": "aoai-api-endpoint",
+                "api_key": "aoai-api-key",
+                "api_version": "2023-07-01-preview",
+                "azure_deployment": "aoai-deployment"
+            },
+        },
+        "azure_ai_project_scope": {
+            "value": {
+                "subscription_id": "subscription-id",
+                "resource_group_name": "resource-group-name",
+                "project_name": "project-name"
+            }
+        }
+    }
+    with open(connections_file_path, mode="w", encoding="utf-8") as f:
+        json.dump(connections_template, f, ensure_ascii=False, indent=4)
+    _prompt_user_for_test_resources(connections_file_path)
+
+
+def create_tools_test_resource_template() -> None:
+    working_dir = REPO_ROOT_DIR / "src" / "promptflow-tools"
+    example_file_path = (working_dir / "connections.json.example").resolve().absolute()
+    target_file_path = (working_dir / "connections.json").resolve().absolute()
+    shutil.copy(example_file_path, target_file_path)
+    _prompt_user_for_test_resources(target_file_path)
+
+
+REGISTERED_TEST_RESOURCES_FUNCTIONS = {
+    "promptflow-tracing": create_tracing_test_resource_template,
+    "promptflow-tools": create_tools_test_resource_template,
+    "promptflow-evals": create_evals_test_resource_template,
+}

scripts/dev-setup/utils.py

@@ -0,0 +1,52 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import contextlib
+import os
+import platform
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT_DIR = Path(__file__).parent.parent.parent
+
+
+class Color:
+    BLUE = "\033[94m"
+    YELLOW = "\033[93m"
+    END = "\033[0m"
+
+
+def print_blue(msg: str) -> None:
+    print(Color.BLUE + msg + Color.END)
+
+
+def print_yellow(msg: str) -> None:
+    print(Color.YELLOW + msg + Color.END)
+
+
+@contextlib.contextmanager
+def change_cwd(path):
+    cwd = os.getcwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(cwd)
+
+
+def run_cmd(cmd, verbose: bool = False) -> None:
+    print_blue(f"Running {' '.join(cmd)}")
+    shell = platform.system() == "Windows"
+    p = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        shell=shell,
+    )
+    for line in p.stdout:
+        line = line.decode("utf-8").rstrip()
+        if verbose:
+            sys.stdout.write(f"{line}\n")
+    p.communicate()