chore: import upstream snapshot with attribution

examples/tutorials/run-flow-with-pipeline/components/data-prep/conda.yaml

@@ -0,0 +1,15 @@
+name: my-env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+  - pip
+  - pip:
+      - mlflow
+      - azureml-core
+      - azure-ai-ml>=1.12.0
+      - azureml-dataset-runtime[pandas,fuse]
+      - azureml-telemetry
+      - mltable>=1.2.0
+      - pandas
+      - pillow

examples/tutorials/run-flow-with-pipeline/components/data-prep/data-prep.py

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license.
+
+import os
+import pandas as pd
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--input_data_file", type=str)
+parser.add_argument("--output_data_folder", type=str)
+
+args, _ = parser.parse_known_args()
+
+input_df = pd.read_json(args.input_data_file, lines=True)
+
+# data preparation, e.g. data sampling, data cleaning, etc.
+processed_data = input_df.sample(n=20, replace=True, random_state=1)
+
+# export data into output folder
+output_file_path = os.path.join(args.output_data_folder, "processed_data.csv")
+processed_data.to_csv(output_file_path, index=False, header=True)

examples/tutorials/run-flow-with-pipeline/components/data-prep/data-prep.yaml

@@ -0,0 +1,26 @@
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+
+type: command
+
+name: data_prep
+display_name: data preparation
+version: 0.0.1
+
+inputs:
+  input_data_file:
+    type: uri_file
+
+outputs:
+  output_data_folder:
+    type: uri_folder
+
+code: ./
+
+environment: 
+  image: mcr.microsoft.com/azureml/inference-base-2004:latest
+  conda_file: ./conda.yaml
+
+command: >-
+  python data-prep.py
+  --input_data_file ${{inputs.input_data_file}}
+  --output_data_folder ${{outputs.output_data_folder}}

examples/tutorials/run-flow-with-pipeline/components/result-parser/conda.yaml

@@ -0,0 +1,15 @@
+name: my-env
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+  - pip
+  - pip:
+      - mlflow
+      - azureml-core
+      - azure-ai-ml>=1.12.0
+      - azureml-dataset-runtime[pandas,fuse]
+      - azureml-telemetry
+      - mltable>=1.2.0
+      - pandas
+      - pillow

examples/tutorials/run-flow-with-pipeline/components/result-parser/result-parser.py

@@ -0,0 +1,51 @@
+# Copyright (c) Microsoft. All rights reserved.
+# Licensed under the MIT license.
+
+import os
+import pandas as pd
+import argparse
+import glob
+import numpy as np
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--source_data", type=str)
+parser.add_argument("--pf_output_data", type=str)
+parser.add_argument("--pf_debug_data", type=str)
+parser.add_argument("--merged_data", type=str)
+
+args, _ = parser.parse_known_args()
+
+source_data_path = os.path.join(args.source_data, "processed_data.csv")
+pf_output_path = os.path.join(args.pf_output_data, "parallel_run_step.jsonl")
+merged_data_path = os.path.join(args.merged_data, "merged_data.jsonl")
+
+if args.pf_debug_data is not None:
+    pf_debug_files = glob.glob(os.path.join(args.pf_debug_data, "flow_artifacts/*.jsonl"))
+
+source_data_df = pd.read_csv(source_data_path)
+pf_output_df = pd.read_json(pf_output_path, lines=True)
+pf_output_df.sort_values(by="line_number", inplace=True, ignore_index=True)
+
+if len(source_data_df) != len(pf_output_df):
+    raise Exception("Index mismatch between data source and pf result")
+
+source_data_df.loc[:, "line_number"] = pf_output_df.loc[:, "line_number"]
+source_data_df.loc[:, "pred_category"] = pf_output_df.loc[:, "category"]
+source_data_df.loc[:, "pred_evidence"] = pf_output_df.loc[:, "evidence"]
+
+if pf_debug_files is not None and len(pf_debug_files) > 0:
+    debug_df = pd.concat([pd.read_json(file, lines=True) for file in pf_debug_files])
+    debug_df.sort_values(by="line_number", inplace=True, ignore_index=True)
+    for i in range(len(debug_df)):
+        source_data_df.loc[i, "prompt_tokens"] = debug_df.loc[i, "run_info"]["system_metrics"]["prompt_tokens"]
+        source_data_df.loc[i, "duration"] = debug_df.loc[i, "run_info"]["system_metrics"]["duration"]
+        source_data_df.loc[i, "completion_tokens"] = debug_df.loc[i, "run_info"]["system_metrics"]["completion_tokens"]
+        source_data_df.loc[i, "total_tokens"] = debug_df.loc[i, "run_info"]["system_metrics"]["total_tokens"]
+else:
+    source_data_df.loc[:, "prompt_tokens"] = np.nan
+    source_data_df.loc[:, "duration"] = np.nan
+    source_data_df.loc[:, "completion_tokens"] = np.nan
+    source_data_df.loc[:, "total_tokens"] = np.nan
+
+with open(merged_data_path, "w") as file:
+    file.write(source_data_df.to_json(orient="records", lines=True))

examples/tutorials/run-flow-with-pipeline/components/result-parser/result-parser.yaml

@@ -0,0 +1,34 @@
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+
+type: command
+
+name: result_parser
+description: Aggregate pf source data with output and debug data.
+display_name: result parser.
+version: 0.0.1
+
+inputs:
+  source_data:
+    type: uri_folder
+  pf_output_data:
+    type: uri_folder
+  pf_debug_data:
+    optional: true
+    type: uri_folder
+
+outputs:
+  merged_data :
+    type: uri_folder
+
+code: ./
+
+environment: 
+  image: mcr.microsoft.com/azureml/inference-base-2004:latest
+  conda_file: ./conda.yaml
+
+command: >-
+  python result-parser.py
+  --source_data ${{inputs.source_data}}
+  --pf_output_data ${{inputs.pf_output_data}}
+  $[[--pf_debug_data ${{inputs.pf_debug_data}}]]
+  --merged_data ${{outputs.merged_data}}