chore: import upstream snapshot with attribution

.github/scripts/download_pretrained.py

@@ -0,0 +1,120 @@
+import argparse
+import logging
+import time
+from typing import List, NamedTuple, Optional, Text
+
+from transformers import AutoTokenizer, TFAutoModel
+
+import rasa.shared.utils.io
+from rasa.nlu.utils.hugging_face.registry import (
+    model_weights_defaults,
+    model_class_dict,
+)
+
+logger = logging.getLogger(__name__)
+
+COMP_NAME = "LanguageModelFeaturizer"
+DEFAULT_MODEL_NAME = "bert"
+
+
+class LmfSpec(NamedTuple):
+    """Holds information about the LanguageModelFeaturizer."""
+
+    model_name: Text
+    model_weights: Text
+    cache_dir: Optional[Text] = None
+
+
+def get_model_name_and_weights_from_config(
+    config_path: str,
+) -> List[LmfSpec]:
+    config = rasa.shared.utils.io.read_config_file(config_path)
+    logger.info(config)
+    steps = config.get("pipeline", [])
+
+    # Look for LanguageModelFeaturizer steps
+    steps = list(filter(lambda x: x["name"] == COMP_NAME, steps))
+
+    lmf_specs = []
+    for lmfeat_step in steps:
+        if "model_name" not in lmfeat_step:
+            if "model_weights" in lmfeat_step:
+                model_weights = lmfeat_step["model_weights"]
+                raise KeyError(
+                    "When model_name is not given, then model_weights cannot be set. "
+                    f"Here, model_weigths is set to {model_weights}"
+                )
+            model_name = DEFAULT_MODEL_NAME
+            model_weights = model_weights_defaults[DEFAULT_MODEL_NAME]
+        else:
+            model_name = lmfeat_step["model_name"]
+
+            if model_name not in model_class_dict:
+                raise KeyError(
+                    f"'{model_name}' not a valid model name. Choose from "
+                    f"{str(list(model_class_dict.keys()))} or create"
+                    f"a new class inheriting from this class to support your model."
+                )
+
+            model_weights = lmfeat_step.get("model_weights")
+            if not model_weights:
+                logger.info(
+                    f"Model weights not specified. Will choose default model "
+                    f"weights: {model_weights_defaults[model_name]}"
+                )
+                model_weights = model_weights_defaults[model_name]
+        cache_dir = lmfeat_step.get("cache_dir", None)
+        lmf_specs.append(LmfSpec(model_name, model_weights, cache_dir))
+
+    return lmf_specs
+
+
+def instantiate_to_download(comp: LmfSpec) -> None:
+    """Instantiates Auto class instances, but only to download."""
+
+    _ = AutoTokenizer.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+    logger.info("Done with AutoTokenizer, now doing TFAutoModel")
+    _ = TFAutoModel.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+
+
+def download(config_path: str):
+    lmf_specs = get_model_name_and_weights_from_config(config_path)
+
+    if not lmf_specs:
+        logger.info(f"No {COMP_NAME} found, therefore, skipping download")
+        return
+
+    for lmf_spec in lmf_specs:
+        logger.info(
+            f"model_name: {lmf_spec.model_name}, "
+            f"model_weights: {lmf_spec.model_weights}, "
+            f"cache_dir: {lmf_spec.cache_dir}"
+        )
+        start = time.time()
+
+        instantiate_to_download(lmf_spec)
+
+        duration_in_sec = time.time() - start
+        logger.info(f"Instantiating Auto classes takes {duration_in_sec:.2f}seconds")
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Downloads pretrained models, i.e., Huggingface weights."""
+    parser = argparse.ArgumentParser(
+        description="Downloads pretrained models, i.e., Huggingface weights, "
+        "e.g. path to bert_diet_responset2t.yml"
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="The path to the config yaml file.",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    download(cmdline_args.config)

.github/scripts/mr_generate_summary.py

@@ -0,0 +1,55 @@
+# Collect the results of the various model test runs which are done as part of
+# the model regression CI pipeline and dump them as a single file artifact.
+# This artifact will the then be published at the end of the tests.
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+from typing import Dict, List
+
+
+def combine_result(
+    result1: Dict[str, dict], result2: Dict[str, Dict[str, Dict]]
+) -> Dict[str, Dict[str, List]]:
+    """Combines 2 result dicts to accumulated dict of the same format.
+
+    Args:
+        result1: dict of key: dataset, value: (dict of key: config, value: list of res)
+                 Example: {
+                              "Carbon Bot": {
+                                  "Sparse + DIET(bow) + ResponseSelector(bow)": [{
+                                      "Entity Prediction": {
+                                          "macro avg": {
+                                              "f1-score": 0.88,
+                                          }
+                                      },
+                                      "test_run_time": "47s",
+                                  }]
+                              }
+                          }
+        result2: dict of key: dataset, value: (dict of key: config, value: list of res)
+
+    Returns:
+        dict of key: dataset, and value: (dict of key: config value: list of results)
+    """
+    combined_dict = defaultdict(lambda: defaultdict(list))
+    for new_dict in [result1, result2]:
+        for dataset, results_for_dataset in new_dict.items():
+            for config, res in results_for_dataset.items():
+                for res_dict in res:
+                    combined_dict[dataset][config].append(res_dict)
+    return combined_dict
+
+
+if __name__ == "__main__":
+    data = {}
+    reports_dir = Path(os.environ["REPORTS_DIR"])
+    reports_paths = list(reports_dir.glob("*/report.json"))
+
+    for report_path in reports_paths:
+        report_dict = json.load(open(report_path))
+        data = combine_result(data, report_dict)
+
+    summary_file = os.environ["SUMMARY_FILE"]
+    with open(summary_file, "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)

.github/scripts/mr_publish_results.py

@@ -0,0 +1,293 @@
+# Send model regression test results to Datadog
+# with a summary of all test results.
+# Also write them into a report file.
+import copy
+import datetime
+import json
+import os
+from typing import Any, Dict, List, Text, Tuple
+
+from datadog_api_client.v1 import ApiClient, Configuration
+from datadog_api_client.v1.api.metrics_api import MetricsApi
+from datadog_api_client.v1.model.metrics_payload import MetricsPayload
+from datadog_api_client.v1.model.point import Point
+from datadog_api_client.v1.model.series import Series
+
+DD_ENV = "rasa-regression-tests"
+DD_SERVICE = "rasa"
+METRIC_RUNTIME_PREFIX = "rasa.perf.benchmark."
+METRIC_ML_PREFIX = "rasa.perf.ml."
+CONFIG_REPOSITORY = "training-data"
+
+TASK_MAPPING = {
+    "intent_report.json": "intent_classification",
+    "CRFEntityExtractor_report.json": "entity_prediction",
+    "DIETClassifier_report.json": "entity_prediction",
+    "response_selection_report.json": "response_selection",
+    "story_report.json": "story_prediction",
+}
+
+METRICS = {
+    "test_run_time": "TEST_RUN_TIME",
+    "train_run_time": "TRAIN_RUN_TIME",
+    "total_run_time": "TOTAL_RUN_TIME",
+}
+
+MAIN_TAGS = {
+    "config": "CONFIG",
+    "dataset": "DATASET_NAME",
+}
+
+OTHER_TAGS = {
+    "config_repository_branch": "DATASET_REPOSITORY_BRANCH",
+    "dataset_commit": "DATASET_COMMIT",
+    "accelerator_type": "ACCELERATOR_TYPE",
+    "type": "TYPE",
+    "index_repetition": "INDEX_REPETITION",
+    "host_name": "HOST_NAME",
+}
+
+GIT_RELATED_TAGS = {
+    "pr_id": "PR_ID",
+    "pr_url": "PR_URL",
+    "github_event": "GITHUB_EVENT_NAME",
+    "github_run_id": "GITHUB_RUN_ID",
+    "github_sha": "GITHUB_SHA",
+    "workflow": "GITHUB_WORKFLOW",
+}
+
+
+def create_dict_of_env(name_to_env: Dict[Text, Text]) -> Dict[Text, Text]:
+    return {name: os.environ[env_var] for name, env_var in name_to_env.items()}
+
+
+def _get_is_external_and_dataset_repository_branch() -> Tuple[bool, Text]:
+    is_external = os.environ["IS_EXTERNAL"]
+    dataset_repository_branch = os.environ["DATASET_REPOSITORY_BRANCH"]
+    if is_external.lower() in ("yes", "true", "t", "1"):
+        is_external_flag = True
+        dataset_repository_branch = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
+    else:
+        is_external_flag = False
+    return is_external_flag, dataset_repository_branch
+
+
+def prepare_datasetrepo_and_external_tags() -> Dict[Text, Any]:
+    is_external, dataset_repo_branch = _get_is_external_and_dataset_repository_branch()
+    return {
+        "dataset_repository_branch": dataset_repo_branch,
+        "external_dataset_repository": is_external,
+    }
+
+
+def prepare_dsrepo_and_external_tags_as_str() -> Dict[Text, Text]:
+    return {
+        "dataset_repository_branch": os.environ["DATASET_REPOSITORY_BRANCH"],
+        "external_dataset_repository": os.environ["IS_EXTERNAL"],
+    }
+
+
+def transform_to_seconds(duration: Text) -> float:
+    """Transform string (with hours, minutes, and seconds) to seconds.
+
+    Args:
+        duration: Examples: '1m27s', '1m27.3s', '27s', '1h27s', '1h1m27s'
+
+    Raises:
+        Exception: If the input is not supported.
+
+    Returns:
+        Duration converted in seconds.
+    """
+    h_split = duration.split("h")
+    if len(h_split) == 1:
+        rest = h_split[0]
+        hours = 0
+    else:
+        hours = int(h_split[0])
+        rest = h_split[1]
+    m_split = rest.split("m")
+    if len(m_split) == 2:
+        minutes = int(m_split[0])
+        seconds = float(m_split[1].rstrip("s"))
+    elif len(m_split) == 1:
+        minutes = 0
+        seconds = float(m_split[0].rstrip("s"))
+    else:
+        raise Exception(f"Unsupported duration: {duration}")
+    overall_seconds = hours * 60 * 60 + minutes * 60 + seconds
+    return overall_seconds
+
+
+def prepare_ml_metric(result: Dict[Text, Any]) -> Dict[Text, float]:
+    """Converts a nested result dict into a list of metrics.
+
+    Args:
+        result: Example
+            {'accuracy': 1.0,
+             'weighted avg': {
+                'precision': 1.0, 'recall': 1.0, 'f1-score': 1.0, 'support': 28
+             }
+            }
+
+    Returns:
+        Dict of metric name and metric value
+    """
+    metrics_ml = {}
+    result = copy.deepcopy(result)
+    result.pop("file_name", None)
+    task = result.pop("task", None)
+
+    for metric_name, metric_value in result.items():
+        if isinstance(metric_value, float):
+            metric_full_name = f"{task}.{metric_name}"
+            metrics_ml[metric_full_name] = float(metric_value)
+        elif isinstance(metric_value, dict):
+            for mname, mval in metric_value.items():
+                metric_full_name = f"{task}.{metric_name}.{mname}"
+                metrics_ml[metric_full_name] = float(mval)
+        else:
+            raise Exception(
+                f"metric_value {metric_value} has",
+                f"unexpected type {type(metric_value)}",
+            )
+    return metrics_ml
+
+
+def prepare_ml_metrics(results: List[Dict[Text, Any]]) -> Dict[Text, float]:
+    metrics_ml = {}
+    for result in results:
+        new_metrics_ml = prepare_ml_metric(result)
+        metrics_ml.update(new_metrics_ml)
+
+    return metrics_ml
+
+
+def prepare_datadog_tags() -> List[Text]:
+    tags = {
+        "env": DD_ENV,
+        "service": DD_SERVICE,
+        "branch": os.environ["BRANCH"],
+        "config_repository": CONFIG_REPOSITORY,
+        **prepare_dsrepo_and_external_tags_as_str(),
+        **create_dict_of_env(MAIN_TAGS),
+        **create_dict_of_env(OTHER_TAGS),
+        **create_dict_of_env(GIT_RELATED_TAGS),
+    }
+    tags_list = [f"{k}:{v}" for k, v in tags.items()]
+    return tags_list
+
+
+def send_to_datadog(results: List[Dict[Text, Any]]) -> None:
+    """Sends metrics to datadog."""
+    # Prepare
+    tags_list = prepare_datadog_tags()
+    timestamp = datetime.datetime.now().timestamp()
+    series = []
+
+    # Send metrics about runtime
+    metrics_runtime = create_dict_of_env(METRICS)
+    for metric_name, metric_value in metrics_runtime.items():
+        overall_seconds = transform_to_seconds(metric_value)
+        series.append(
+            Series(
+                metric=f"{METRIC_RUNTIME_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, overall_seconds])],
+                tags=tags_list,
+            )
+        )
+
+    # Send metrics about ML model performance
+    metrics_ml = prepare_ml_metrics(results)
+    for metric_name, metric_value in metrics_ml.items():
+        series.append(
+            Series(
+                metric=f"{METRIC_ML_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, float(metric_value)])],
+                tags=tags_list,
+            )
+        )
+
+    body = MetricsPayload(series=series)
+    with ApiClient(Configuration()) as api_client:
+        api_instance = MetricsApi(api_client)
+        response = api_instance.submit_metrics(body=body)
+        if response.get("status") != "ok":
+            print(response)
+
+
+def read_results(file: Text) -> Dict[Text, Any]:
+    with open(file) as json_file:
+        data = json.load(json_file)
+
+        keys = [
+            "accuracy",
+            "weighted avg",
+            "macro avg",
+            "micro avg",
+            "conversation_accuracy",
+        ]
+        result = {key: data[key] for key in keys if key in data}
+
+    return result
+
+
+def get_result(file_name: Text, file: Text) -> Dict[Text, Any]:
+    result = read_results(file)
+    result["file_name"] = file_name
+    result["task"] = TASK_MAPPING[file_name]
+    return result
+
+
+def send_all_to_datadog() -> None:
+    results = []
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if any(f.endswith(valid_name) for valid_name in TASK_MAPPING.keys()):
+                result = get_result(f, os.path.join(dirpath, f))
+                results.append(result)
+    send_to_datadog(results)
+
+
+def generate_json(file: Text, task: Text, data: dict) -> dict:
+    config = os.environ["CONFIG"]
+    dataset = os.environ["DATASET_NAME"]
+
+    if dataset not in data:
+        data = {dataset: {config: []}, **data}
+    elif config not in data[dataset]:
+        data[dataset] = {config: [], **data[dataset]}
+
+    assert len(data[dataset][config]) <= 1
+
+    data[dataset][config] = [
+        {
+            "config_repository": CONFIG_REPOSITORY,
+            **prepare_datasetrepo_and_external_tags(),
+            **create_dict_of_env(METRICS),
+            **create_dict_of_env(OTHER_TAGS),
+            **(data[dataset][config][0] if data[dataset][config] else {}),
+            task: read_results(file),
+        }
+    ]
+    return data
+
+
+def create_report_file() -> None:
+    data = {}
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if f not in TASK_MAPPING.keys():
+                continue
+
+            data = generate_json(os.path.join(dirpath, f), TASK_MAPPING[f], data)
+
+    with open(os.environ["SUMMARY_FILE"], "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    send_all_to_datadog()
+    create_report_file()

.github/scripts/start_dd_agent.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+DD_API_KEY=$1
+ACCELERATOR_TYPE=$2
+NVML_INTERVAL_IN_SEC=${3:-15}  # 15 seconds are the default interval
+
+# Install Datadog system agent
+DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$DD_API_KEY DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+DATADOG_YAML_PATH=/etc/datadog-agent/datadog.yaml
+sudo chmod 666 $DATADOG_YAML_PATH
+
+# Associate metrics with tags and env
+{
+    echo "env: rasa-regression-tests"
+    echo "tags:"
+    echo "- service:rasa"
+    echo "- accelerator_type:${ACCELERATOR_TYPE}"
+    echo "- dataset:${DATASET_NAME}"
+    echo "- config:${CONFIG}"
+    echo "- dataset_commit:${DATASET_COMMIT}"
+    echo "- branch:${BRANCH}"
+    echo "- github_sha:${GITHUB_SHA}"
+    echo "- pr_id:${PR_ID:-schedule}"
+    echo "- pr_url:${PR_URL:-schedule}"
+    echo "- type:${TYPE}"
+    echo "- dataset_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- external_dataset_repository:${IS_EXTERNAL:-none}"
+    echo "- config_repository:training-data"
+    echo "- config_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- workflow:${GITHUB_WORKFLOW:-none}"
+    echo "- github_run_id:${GITHUB_RUN_ID:-none}"
+    echo "- github_event:${GITHUB_EVENT_NAME:-none}"
+    echo "- index_repetition:${INDEX_REPETITION}"
+    echo "- host_name:${HOST_NAME}"
+    echo ""
+    echo "apm_config:"
+    echo "    enabled: true"
+    echo "process_config:"
+    echo "    enabled: false"
+    echo "use_dogstatsd: true"
+} >> $DATADOG_YAML_PATH
+
+# Enable system_core integration
+sudo mv /etc/datadog-agent/conf.d/system_core.d/conf.yaml.example /etc/datadog-agent/conf.d/system_core.d/conf.yaml
+
+if [[ "${ACCELERATOR_TYPE}" == "GPU" ]]; then
+    # Install and enable NVML integration
+    sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
+    sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
+    NVML_CONF_FPATH="/etc/datadog-agent/conf.d/nvml.d/conf.yaml"
+    sudo mv "${NVML_CONF_FPATH}.example" ${NVML_CONF_FPATH}
+    if [[ "${NVML_INTERVAL_IN_SEC}" != 15 ]]; then
+        # Append a line to the NVML config file
+        sudo echo "    min_collection_interval: ${NVML_INTERVAL_IN_SEC}" | sudo tee -a ${NVML_CONF_FPATH} > /dev/null
+    fi
+fi
+
+# Apply changes
+sudo service datadog-agent stop
+
+# Restart agent (such that GPU/NVML metrics are collected)
+# Adusted code from /etc/init/datadog-agent.conf
+INSTALL_DIR="/opt/datadog-agent"
+AGENTPATH="$INSTALL_DIR/bin/agent/agent"
+PIDFILE="$INSTALL_DIR/run/agent.pid"
+AGENT_USER="dd-agent"
+LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $AGENTPATH run -p $PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-process.conf
+TRACE_AGENTPATH="$INSTALL_DIR/embedded/bin/trace-agent"
+TRACE_PIDFILE="$INSTALL_DIR/run/trace-agent.pid"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $TRACE_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TRACE_AGENTPATH --config $DATADOG_YAML_PATH --pid $TRACE_PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-trace.conf
+PROCESS_AGENTPATH="$INSTALL_DIR/embedded/bin/process-agent"
+PROCESS_PIDFILE="$INSTALL_DIR/run/process-agent.pid"
+SYSTEM_PROBE_YAML="/etc/datadog-agent/system-probe.yaml"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PROCESS_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $PROCESS_AGENTPATH --config=$DATADOG_YAML_PATH --sysprobe-config=$SYSTEM_PROBE_YAML --pid=$PROCESS_PIDFILE"

.github/scripts/validate_cpu.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_not_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus > 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_not_available()

.github/scripts/validate_gpus.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus <= 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_available()