{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# setup pf client and execution path\n", "\n", "from promptflow.client import PFClient\n", "import json\n", "import os\n", "\n", "pf = PFClient()\n", "\n", "root = os.path.join(os.getcwd(), \"../\")\n", "flow = os.path.join(root, \"maths-to-code\")\n", "data = os.path.join(flow, \"math_data.jsonl\")\n", "eval_flow = os.path.join(root, \"../evaluation/eval-accuracy-maths-to-code\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# start batch run of maths-to-code\n", "base_run = pf.run(\n", " flow = flow,\n", " data = data,\n", " column_mapping={\"math_question\": \"${data.question}\"},\n", " display_name=\"maths_to_code_batch_run\",\n", " stream=True\n", ")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
inputs.math_questioninputs.line_numberoutputs.answeroutputs.code
0What is the sum of 5 and 3?08.000000print(5 + 3)
1Subtract 7 from 10.13.000000print(10 - 7)
2Multiply 6 by 4.224.000000print(6 * 4)
3Divide 20 by 5.34.000000print(20 / 5)
4What is the square of 7?449.000000print(7**2)
5What is the square root of 81?59.000000import math\\nprint(math.sqrt(81))
6If a rectangle has a length of 10 and width of...650.000000length = 10\\nwidth = 5\\narea = length * width\\...
7A circle has a radius of 7, what is the area? ...7153.860000area = 3.14 * (7**2)\\nprint(area)
8Solve for x in the equation 2x + 3 = 9.83.000000print((9-3)/2)
9What is the value of x if 5x = 25?95.000000print(25/5)
10A car travels 200 miles in 4 hours. What is th...1050.000000print(200/4)
11A car travels at a speed of 60 mph. How long w...113.000000print(180 / 60)
12If a car travels at a speed of 40 mph for 2 ho...1280.000000print(40 * 2)
13A rectangle has length = 10 cm and width = 5 c...1350.000000print(10 * 5)
14A circle has radius = 7 cm. What is its circum...1443.960000radius = 7\\ncircumference = 2 * 3.14 * radius\\...
15A triangle has base =10 cm and height =5 cm. W...1525.000000print((10*5)/2)
16What is the slope of the line that passes thro...162.000000x1, y1 = 2, 3\\nx2, y2 = 4, 7\\nslope = (y2 - y1...
17The distance between A and B is 2000km, A is m...1710.000000print(2000/(80+120))
18The lengths of the two perpendicular sides of ...1810.000000import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...
19A is running with average speed 10km/hour, A a...190.333333A_distance = 10 * 0.5\\nB_speed = 15\\nB_time = ...
\n", "
" ], "text/plain": [ " inputs.math_question inputs.line_number \\\n", "0 What is the sum of 5 and 3? 0 \n", "1 Subtract 7 from 10. 1 \n", "2 Multiply 6 by 4. 2 \n", "3 Divide 20 by 5. 3 \n", "4 What is the square of 7? 4 \n", "5 What is the square root of 81? 5 \n", "6 If a rectangle has a length of 10 and width of... 6 \n", "7 A circle has a radius of 7, what is the area? ... 7 \n", "8 Solve for x in the equation 2x + 3 = 9. 8 \n", "9 What is the value of x if 5x = 25? 9 \n", "10 A car travels 200 miles in 4 hours. What is th... 10 \n", "11 A car travels at a speed of 60 mph. How long w... 11 \n", "12 If a car travels at a speed of 40 mph for 2 ho... 12 \n", "13 A rectangle has length = 10 cm and width = 5 c... 13 \n", "14 A circle has radius = 7 cm. What is its circum... 14 \n", "15 A triangle has base =10 cm and height =5 cm. W... 15 \n", "16 What is the slope of the line that passes thro... 16 \n", "17 The distance between A and B is 2000km, A is m... 17 \n", "18 The lengths of the two perpendicular sides of ... 18 \n", "19 A is running with average speed 10km/hour, A a... 19 \n", "\n", " outputs.answer outputs.code \n", "0 8.000000 print(5 + 3) \n", "1 3.000000 print(10 - 7) \n", "2 24.000000 print(6 * 4) \n", "3 4.000000 print(20 / 5) \n", "4 49.000000 print(7**2) \n", "5 9.000000 import math\\nprint(math.sqrt(81)) \n", "6 50.000000 length = 10\\nwidth = 5\\narea = length * width\\... \n", "7 153.860000 area = 3.14 * (7**2)\\nprint(area) \n", "8 3.000000 print((9-3)/2) \n", "9 5.000000 print(25/5) \n", "10 50.000000 print(200/4) \n", "11 3.000000 print(180 / 60) \n", "12 80.000000 print(40 * 2) \n", "13 50.000000 print(10 * 5) \n", "14 43.960000 radius = 7\\ncircumference = 2 * 3.14 * radius\\... \n", "15 25.000000 print((10*5)/2) \n", "16 2.000000 x1, y1 = 2, 3\\nx2, y2 = 4, 7\\nslope = (y2 - y1... \n", "17 10.000000 print(2000/(80+120)) \n", "18 10.000000 import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**... \n", "19 0.333333 A_distance = 10 * 0.5\\nB_speed = 15\\nB_time = ... " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Show output of flow run\n", "pf.get_details(base_run)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# evaluate against the batch run and groundtruth data\n", "eval_run = pf.run(\n", " flow = eval_flow,\n", " data = data,\n", " run = base_run,\n", " column_mapping={\"groundtruth\": \"${data.answer}\", \"prediction\": \"${run.outputs.answer}\"},\n", " display_name=\"maths_to_code_eval_run\",\n", " stream=True\n", ")\n", "\n", "pf.get_details(eval_run)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.95, 'error_rate': 0.0}" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Get metrics of the evaluation flow run\n", "pf.get_metrics(eval_run)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Visualize the flow run and evaluation run with HTML\n", "pf.visualize([base_run, eval_run])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run on Azure\n", "If you want to run and evaluate your flow on Azure, you can using following example to setup your Azure ML workspace " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", "\n", "# init credential\n", "try:\n", " credential = DefaultAzureCredential()\n", " # Check if given credential can get token successfully.\n", " credential.get_token(\"https://management.azure.com/.default\")\n", "except Exception as ex:\n", " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", " credential = InteractiveBrowserCredential()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from promptflow.azure import PFClient\n", "\n", "try:\n", " pf = PFClient.from_config(credential=credential)\n", "except Exception as ex:\n", " # NOTE: Update following workspace information if not correctly configure before\n", " client_config = {\n", " \"subscription_id\": \"\",\n", " \"resource_group\": \"\",\n", " \"workspace_name\": \"\",\n", " }\n", "\n", " if client_config[\"subscription_id\"].startswith(\"<\"):\n", " print(\n", " \"please update your in notebook cell\"\n", " )\n", " raise ex\n", " else: # write and reload from config file\n", " import json, os\n", "\n", " config_path = \"../.azureml/config.json\"\n", " os.makedirs(os.path.dirname(config_path), exist_ok=True)\n", " with open(config_path, \"w\") as fo:\n", " fo.write(json.dumps(client_config))\n", " pf = PFClient.from_config(credential=credential, path=config_path)\n", "\n", "print(pf)\n", "\n", "# NOTE: note that you need to replace and with your own connection and deployment name in your Azure Machine Learning workspace\n", "connection_mapping = {\"code_gen\": {\"connection\": \"\", \"deployment_name\": \"\"}}" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# batch run of maths to code\n", "\n", "base_run = pf.run(\n", " flow = flow,\n", " data = data,\n", " column_mapping = {\"math_question\": \"${data.question}\"},\n", " connections = connection_mapping,\n", " stream = True,\n", ")" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
inputs.math_questionoutputs.codeoutputs.answer
0Subtract 7 from 10.print(10 - 7)3
1What is the sum of 5 and 3?print(5+3)8
2What is the square of 7?print(7**2)49
3A car travels 200 miles in 4 hours. What is th...print(200 / 4)50.0
4Multiply 6 by 4.print(6 * 4)24
5What is the value of x if 5x = 25?print(25/5)5.0
6If a car travels at a speed of 40 mph for 2 ho...print(40 * 2)80
7A car travels at a speed of 60 mph. How long w...print(180 / 60)3.0
8A triangle has base =10 cm and height =5 cm. W...print((10*5)/2)25.0
9Divide 20 by 5.print(20 / 5)4.0
10If a rectangle has a length of 10 and width of...print(10 * 5)50
11What is the square root of 81?import math\\nprint(math.sqrt(81))9.0
12Solve for x in the equation 2x + 3 = 9.print((9-3)/2)3.0
13A rectangle has length = 10 cm and width = 5 c...length = 10\\ncm_width = 5\\narea = length * wid...name 'width' is not defined
14A circle has a radius of 7, what is the area? ...pi = 3.14\\nradius = 7\\narea = pi * radius ** 2...153.86
15What is the slope of the line that passes thro...slope = (7-3)/(4-2)\\nprint(slope)2.0
16The distance between A and B is 2000km, A is m...distance = 2000\\nspeed_A = 80\\nspeed_B = 120\\n...10.0
17A circle has radius = 7 cm. What is its circum...pi = 3.14\\nradius = 7\\ncircumference = 2 * pi ...43.96
18The lengths of the two perpendicular sides of ...import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...10.0
19A is running with average speed 10km/hour, A a...distance_A = 10 * (0.5)\\n\\nspeed_B = 15\\n\\ntim...
\n", "
" ], "text/plain": [ " inputs.math_question \\\n", "0 Subtract 7 from 10. \n", "1 What is the sum of 5 and 3? \n", "2 What is the square of 7? \n", "3 A car travels 200 miles in 4 hours. What is th... \n", "4 Multiply 6 by 4. \n", "5 What is the value of x if 5x = 25? \n", "6 If a car travels at a speed of 40 mph for 2 ho... \n", "7 A car travels at a speed of 60 mph. How long w... \n", "8 A triangle has base =10 cm and height =5 cm. W... \n", "9 Divide 20 by 5. \n", "10 If a rectangle has a length of 10 and width of... \n", "11 What is the square root of 81? \n", "12 Solve for x in the equation 2x + 3 = 9. \n", "13 A rectangle has length = 10 cm and width = 5 c... \n", "14 A circle has a radius of 7, what is the area? ... \n", "15 What is the slope of the line that passes thro... \n", "16 The distance between A and B is 2000km, A is m... \n", "17 A circle has radius = 7 cm. What is its circum... \n", "18 The lengths of the two perpendicular sides of ... \n", "19 A is running with average speed 10km/hour, A a... \n", "\n", " outputs.code \\\n", "0 print(10 - 7) \n", "1 print(5+3) \n", "2 print(7**2) \n", "3 print(200 / 4) \n", "4 print(6 * 4) \n", "5 print(25/5) \n", "6 print(40 * 2) \n", "7 print(180 / 60) \n", "8 print((10*5)/2) \n", "9 print(20 / 5) \n", "10 print(10 * 5) \n", "11 import math\\nprint(math.sqrt(81)) \n", "12 print((9-3)/2) \n", "13 length = 10\\ncm_width = 5\\narea = length * wid... \n", "14 pi = 3.14\\nradius = 7\\narea = pi * radius ** 2... \n", "15 slope = (7-3)/(4-2)\\nprint(slope) \n", "16 distance = 2000\\nspeed_A = 80\\nspeed_B = 120\\n... \n", "17 pi = 3.14\\nradius = 7\\ncircumference = 2 * pi ... \n", "18 import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**... \n", "19 distance_A = 10 * (0.5)\\n\\nspeed_B = 15\\n\\ntim... \n", "\n", " outputs.answer \n", "0 3 \n", "1 8 \n", "2 49 \n", "3 50.0 \n", "4 24 \n", "5 5.0 \n", "6 80 \n", "7 3.0 \n", "8 25.0 \n", "9 4.0 \n", "10 50 \n", "11 9.0 \n", "12 3.0 \n", "13 name 'width' is not defined \n", "14 153.86 \n", "15 2.0 \n", "16 10.0 \n", "17 43.96 \n", "18 10.0 \n", "19 " ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# get output of flow run\n", "pf.get_details(base_run)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# evaluation run against base run\n", "\n", "eval_run = pf.run(\n", " flow = eval_flow,\n", " data = data,\n", " run = base_run,\n", " column_mapping={\"groundtruth\": \"${data.answer}\", \"prediction\": \"${run.outputs.answer}\"},\n", " stream = True,\n", ")\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
inputs.groundtruthinputs.predictionoutputs.score
04.004.01
13.0031
224.00241
349.00491
45.005.01
550.00501
68.0081
73.003.01
843.9643.961
93.003.01
10153.86153.861
119.009.01
1280.00801
1325.0025.01
1450.0050.01
152.002.01
1650.00name 'width' is not defined-1
1710.0010.01
1810.0010.01
191.00-1
\n", "
" ], "text/plain": [ " inputs.groundtruth inputs.prediction outputs.score\n", "0 4.00 4.0 1\n", "1 3.00 3 1\n", "2 24.00 24 1\n", "3 49.00 49 1\n", "4 5.00 5.0 1\n", "5 50.00 50 1\n", "6 8.00 8 1\n", "7 3.00 3.0 1\n", "8 43.96 43.96 1\n", "9 3.00 3.0 1\n", "10 153.86 153.86 1\n", "11 9.00 9.0 1\n", "12 80.00 80 1\n", "13 25.00 25.0 1\n", "14 50.00 50.0 1\n", "15 2.00 2.0 1\n", "16 50.00 name 'width' is not defined -1\n", "17 10.00 10.0 1\n", "18 10.00 10.0 1\n", "19 1.00 -1" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# get output of evaluation run\n", "pf.get_details(eval_run)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"accuracy\": 0.9,\n", " \"error_rate\": 0.1\n", "}\n" ] } ], "source": [ "metrics = pf.get_metrics(eval_run)\n", "print(json.dumps(metrics, indent=4))" ] } ], "metadata": { "kernelspec": { "display_name": "pf", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }