chore: import upstream snapshot with attribution

scripts/code_qa/assert_local_install.py

@@ -0,0 +1,20 @@
+"""Tests checking that azure packages are NOT installed."""
+import importlib
+import pytest
+
+
+class TestPackagesNotInstalles():
+    """Test imports."""
+
+    @pytest.mark.parametrize('package', [
+        'promptflow.azure',
+        'azure.ai.ml',
+        'azure.storage.blob'
+    ])
+    def test_promptflow_azure(self, package):
+        """Test promptflow. azure is not installed."""
+        try:
+            importlib.import_module(package)
+            assert False, f'Package {package} must be uninstalled for local test.'
+        except (ModuleNotFoundError, ImportError):
+            pass

scripts/code_qa/calculate_install_time.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+print_usage(){
+  if [ $# -gt 0 ]; then
+      echo "Missing argument ${1}"
+  fi
+  echo "Usage:"
+  echo "$0 -r [github run id] -w [github workflow] -a [github action id] -b [github ref id] -e [Optional extras] -f [ should we fail?] -l [instal, time limit]"
+  echo "Extras should be written as it appears in pip, for example for promptflow-evals[azure], it will be [azure]"
+  echo "Flag -f does not require parameter."
+  exit 1
+}
+
+run_id=""
+workflow=""
+action=""
+ref=""
+fail=0
+extras=""
+limit=""
+
+
+while getopts ":r:w:a:b:e:l:f" opt; do
+# Parse options
+  case $opt in
+    (r) run_id="$OPTARG";;
+    (w) workflow="$OPTARG";;
+    (a) action="$OPTARG";;
+    (b) ref="$OPTARG";;
+    (e) extras="$OPTARG";;
+    (f) ((fail++));;
+    (l) limit="$OPTARG";;
+    \?) print_usage;;
+  esac
+done
+  
+for v in "run_id" "workflow" "action" "ref" "limit"; do
+    if [ -z ${!v} ]; then
+        print_usage "$v"
+    fi
+done
+
+ENV_DIR="test_pf_ev"
+python -m virtualenv "${ENV_DIR}"
+# Make activate command platform independent
+ACTIVATE="${ENV_DIR}/bin/activate"
+if [ ! -f "$ACTIVATE" ]; then
+  ACTIVATE="${ENV_DIR}/Scripts/activate"
+fi
+source "${ACTIVATE}"
+# Estimate the installation time.
+pf_evals_wheel=`ls -1 promptflow_evals-*`
+echo "The downloaded wheel file ${pf_evals_wheel}"
+packages=`python -m pip freeze | wc -l`
+start_tm=`date +%s`
+echo "python -m pip install \"./${pf_evals_wheel}${extras}\" --no-cache-dir"
+python -m pip install "./${pf_evals_wheel}${extras}" --no-cache-dir
+install_time=$((`date +%s` - ${start_tm}))
+packages_installed=$((`python -m pip freeze | wc -l` - packages))
+# Log the install time
+python `dirname "$0"`/report_to_app_insights.py --activity "install_time_s" --value "{\"install_time_s\": ${install_time}, \"number_of_packages_installed\": ${packages_installed}}" --git-hub-action-run-id "${run_id}" --git-hub-workflow "${workflow}" --git-hub-action "${action}" --git-branch "${ref}"
+deactivate
+rm -rf test_pf_ev
+echo "Installed ${packages_installed} packages per ${install_time} seconds."
+if [ $fail -eq 0 ]; then
+    # Swallow the exit code 1 and just show the warning, understandable by
+    # github UI.
+    test ${install_time} -le $limit || echo "::warning file=pyproject.toml,line=40,col=0::The installation took ${install_time} seconds, the limit is ${limit}."
+else
+    test ${install_time} -le $limit
+fi
+# Return the exit code of test command of of echo i.e. 0.
+exit $?

scripts/code_qa/report_to_app_insights.py

@@ -0,0 +1,109 @@
+from typing import Dict, Optional, Union
+
+import argparse
+import json
+import platform
+
+from promptflow._sdk._configuration import Configuration
+from promptflow._sdk._telemetry.telemetry import get_telemetry_logger
+from xml.dom import minidom
+
+
+def parse_junit_xml(fle: str) -> Dict[str, Dict[str, Union[float, str]]]:
+    """
+    Parse the xml in Junit xml format.
+
+    :param fle: The file in JUnit xml format.
+    :type fle: str
+    :return: The dictionary with tests, their run times and pass/fail status.
+    """
+    test_results = {}
+    dom = minidom.parse(fle)
+    # Take node list Document/testsuites/testsuite/
+    for test in dom.firstChild.firstChild.childNodes:
+        test_name = f"{test.attributes['classname'].value}::{test.attributes['name'].value}"
+        test_results[test_name] = {'fail_message': '', 'time': float(test.attributes['time'].value)}
+
+        for child in test.childNodes:
+            if child.nodeName == 'failure':
+                test_results[test_name]['fail_message'] = child.attributes["message"].value
+    return test_results
+
+
+def main(activity_name: str,
+         value: Union[float, str],
+         run_id: str,
+         workflow: str,
+         action: str,
+         branch: str,
+         junit_file: Optional[str]) -> None:
+    """
+    Log the CI-CD event.
+
+    :param activity_name: The name of a n activity to be logged, for example, installation time.
+    :type activity_name: str
+    :param value: The value of a parameter
+    :type value: float
+    :param run_id: The CI-CD run id.
+    :type run_id: str
+    :param workflow: The name of a workflow or path to a workflow file.
+    :type workflow: str
+    :param action: The name of running action or a step.
+    :type action: str
+    :param branch: The branch from which the CI-CD was triggered.
+    :type branch: str
+    :param junit_file: The path to junit test file results.
+    :type junit_file: str
+    """
+    # Enable telemetry
+    config = Configuration.get_instance()
+    config.set_config(Configuration.COLLECT_TELEMETRY, True)
+    logger = get_telemetry_logger()
+    activity_info = {
+        "activity_name": activity_name,
+        "activity_type": "ci_cd_analytics",
+        "OS": platform.system(),
+        "OS_release": platform.release(),
+        "branch": branch,
+        "git_hub_action_run_id": run_id,
+        "git_hub_workflow": workflow
+    }
+    if junit_file:
+        junit_dict = parse_junit_xml(junit_file)
+        for k, v in junit_dict.items():
+            if v["fail_message"]:
+                # Do not log time together with fail message.
+                continue
+            activity_info[k] = v['time']
+    else:
+        if isinstance(value, str):
+            activity_info.update(json.loads(value))
+        else:
+            activity_info["value"] = value
+
+    # write information to the application insights.
+    logger.info(action, extra={"custom_dimensions": activity_info})
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Log the value to application insights along with platform characteristics and run ID.")
+    parser.add_argument('--activity', help='The activity to be logged.',
+                        required=True)
+    parser.add_argument(
+        '--value',
+        help='The floating point value for activity or a set of values in key-value format.',
+        required=False,
+        default=-1)
+    parser.add_argument('--junit-xml', help='The path to junit-xml file.',
+                        dest="junit_xml", required=False, default=None)
+    parser.add_argument('--git-hub-action-run-id', dest='run_id',
+                        help='The run ID of GitHub action run.', required=True)
+    parser.add_argument('--git-hub-workflow', dest='workflow',
+                        help='The name of a workflow or a path to workflow file.', required=True)
+    parser.add_argument('--git-hub-action', dest='action',
+                        help='Git hub action or step.', required=True)
+    parser.add_argument('--git-branch', dest='branch',
+                        help='Git hub Branch.', required=True)
+    args = parser.parse_args()
+    main(args.activity, args.value, args.run_id, args.workflow, args.action, args.branch, args.junit_xml)

scripts/code_qa/test_reporter.py

@@ -0,0 +1,67 @@
+import os
+import platform
+import pytest
+
+from unittest.mock import patch, MagicMock
+
+import report_to_app_insights
+import tempfile
+
+
+class TestReporter:
+    """The set of local tests to test reporting to application insights."""
+
+    @pytest.mark.parametrize(
+        'value,expected_val',
+        [
+            (42, {'value': 42.}),
+            ('{"foo": 1, "bar": 2}', {"foo": 1, "bar": 2})
+        ]
+    )
+    def test_logging_value(self, value, expected_val):
+        """Test loading values from."""
+        mock_logger = MagicMock()
+        expected = {
+            "activity_name": 'test_act',
+            "activity_type": "ci_cd_analytics",
+            "OS": platform.system(),
+            "OS_release": platform.release(),
+            "branch": "some_branch",
+            "git_hub_action_run_id": "gh_run_id",
+            "git_hub_workflow": "gh_wf"
+        }
+        expected.update(expected_val)
+        with patch('report_to_app_insights.get_telemetry_logger', return_value=mock_logger):
+            report_to_app_insights.main(
+                'test_act', value, "gh_run_id", "gh_wf", 'my_action', "some_branch", junit_file=None)
+        mock_logger.info.assert_called_with('my_action', extra={'custom_dimensions': expected})
+
+    def test_log_junit_xml(self):
+        """Test that we are loading junit xml files as expected."""
+        content = (
+            '<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest">'
+            '<testcase classname="MyTestClass1" name="my_successful_test_method" time="4.2"/>'
+            '<testcase classname="MyTestClass2" name="my_unsuccessful_test_method" time="4.2">'
+            '<failure message="Everything failed">fail :(</failure></testcase>'
+            '</testsuite></testsuites>'
+        )
+        mock_logger = MagicMock()
+        expected = {
+            "activity_name": 'test_act',
+            "activity_type": "ci_cd_analytics",
+            "OS": platform.system(),
+            "OS_release": platform.release(),
+            "MyTestClass1::my_successful_test_method": 4.2,
+            "branch": "some_branch",
+            "git_hub_action_run_id": "gh_run_id",
+            "git_hub_workflow": "gh_wf"
+        }
+        with tempfile.TemporaryDirectory() as d:
+            file_xml = os.path.join(d, "test-results.xml")
+            with open(file_xml, 'w') as f:
+                f.write(content)
+            with patch('report_to_app_insights.get_telemetry_logger', return_value=mock_logger):
+                report_to_app_insights.main(
+                    'test_act', -1, "gh_run_id", "gh_wf", 'my_action', "some_branch", junit_file=file_xml)
+
+        mock_logger.info.assert_called_with('my_action', extra={'custom_dimensions': expected})