wehub-resource-sync
e768098d0e
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Publish Promptflow Doc / Build (push) Has been cancelled
Publish Promptflow Doc / Deploy (push) Has been cancelled
Flake8 Lint / flake8 (push) Has been cancelled
Spell check CI / Spell_Check (push) Has been cancelled
117 lines
2.7 KiB
Plaintext
117 lines
2.7 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Setup execution path and pf client\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import promptflow\n",
|
|
"\n",
|
|
"root = os.path.join(os.getcwd(), \"../\")\n",
|
|
"flow_path = os.path.join(root, \"named-entity-recognition\")\n",
|
|
"data_path = os.path.join(flow_path, \"data.jsonl\")\n",
|
|
"eval_match_rate_flow_path = os.path.join(root, \"../evaluation/eval-entity-match-rate\")\n",
|
|
"\n",
|
|
"pf = promptflow.PFClient()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Run flow against test data\n",
|
|
"run = pf.run(\n",
|
|
" flow=flow_path,\n",
|
|
" data=data_path,\n",
|
|
" column_mapping={\n",
|
|
" \"text\": \"${data.text}\",\n",
|
|
" \"entity_type\": \"${data.entity_type}\"\n",
|
|
" },\n",
|
|
" display_name=\"ner_bulk_run\",\n",
|
|
" tags={\"unittest\": \"true\"},\n",
|
|
" stream=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Show output of flow run\n",
|
|
"pf.get_details(run)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Evaluate the match rate of the entity recognition result of the flow run\n",
|
|
"eval = pf.run(\n",
|
|
" flow=eval_match_rate_flow_path,\n",
|
|
" run=run,\n",
|
|
" data=data_path,\n",
|
|
" column_mapping={\n",
|
|
" \"entities\": \"${run.outputs.entities}\",\n",
|
|
" \"ground_truth\": \"${data.results}\"\n",
|
|
" },\n",
|
|
" display_name=\"eval_match_rate\",\n",
|
|
" tags={\"unittest\": \"true\"},\n",
|
|
" stream=True)\n",
|
|
"\n",
|
|
"pf.get_details(eval)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Get metrics of the evaluation flow run\n",
|
|
"pf.get_metrics(eval)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Visualize the flow run and evaluation run with HTML\n",
|
|
"pf.visualize([run, eval])"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "pf",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|