chore: import upstream snapshot with attribution

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,2 @@
+<!-- IF YOU ARE ASKING A USAGE QUESTION (E.G. "HOW DO I DO XYZ") PLEASE POST
+YOUR QUESTION ON https://forum.rasa.com INSTEAD -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Bug Report
+    url: https://rasa-open-source.atlassian.net/browse/OSS
+    about: Create a report to help us improve https://rasa-open-source.atlassian.net/browse/OSS
+  - name: Feature request
+    url: https://rasa-open-source.atlassian.net/browse/OSS
+    about: Suggest an idea on how to improve Rasa https://rasa-open-source.atlassian.net/browse/OSS
+  - name: Ask a question
+    url: https://forum.rasa.com/
+    about: If you have a "How do I?" question please ask in the forum https://forum.rasa.com

.github/PULL_REQUEST_AUTOMATIC_TEMPLATE.md

@@ -0,0 +1,10 @@
+:bulb: This pull request was created automatically to merge a release branch back into the `main` branch.
+
+The changes you see here should have already been reviewed by someone, and shouldn't need an extra
+review. Nonetheless, if you notice something that needs to be addressed, please reach out to the person
+responsible for the original changes. In case additional changes need to be made, they need to target the release branch
+(not this pull request nor `main`).
+
+:auto_rickshaw: This PR should be merged automatically once it has been approved. If it doesn't happen:
+- [ ] Handle merge conflicts
+- [ ] Fix build errors

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+**Proposed changes**:
+- ...
+
+**Status (please check what you already did)**:
+- [ ] added some tests for the functionality
+- [ ] updated the documentation
+- [ ] updated the changelog (please check [changelog](https://github.com/RasaHQ/rasa/tree/main/changelog) for instructions)
+- [ ] reformat files using `black` (please check [Readme](https://github.com/RasaHQ/rasa#code-style) for instructions)

.github/change_filters.yml

@@ -0,0 +1,28 @@
+backend:
+  - 'pyproject.toml'
+  - 'poetry.lock'
+  - 'rasa/**/*'
+  - 'tests/**/*'
+  - 'data/**/*'
+  - 'examples/**/*'
+  - 'Makefile'
+  - '.github/workflows/continous-integration.yml'
+  - '.github/workflows/security-scans.yml'
+
+docker:
+  - 'pyproject.toml'
+  - 'poetry.lock'
+  - 'rasa/**/*'
+  - 'docker/**/*'
+  - 'Makefile'
+
+docs:
+  - 'docs/**/*'
+  - 'changelog/*'
+  - 'CHANGELOG.mdx'
+  - 'tests/docs/*'
+  - 'data/**/*'
+  - 'examples/**/*'
+  - 'Makefile'
+  - '.github/workflows/documentation.yml'
+  - '.github/workflows/ci-docs-tests.yml'

.github/configs/mr-test-example.yaml

@@ -0,0 +1,49 @@
+## Example configuration
+#################### syntax #################
+## include:
+##   - dataset: ["<dataset_name>"]
+##     config: ["<configuration_name>"]
+#
+## Example:
+## include:
+##  - dataset: ["Carbon Bot"]
+##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
+#
+## Shortcut:
+## You can use the "all" shortcut to include all available configurations or datasets
+#
+## Example: Use the "Sparse + EmbeddingIntent + ResponseSelector(bow)" configuration
+## for all available datasets
+## include:
+##  - dataset: ["all"]
+##    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]
+#
+## Example: Use all available configurations for the "Carbon Bot" and "Sara" datasets
+## and for the "Hermit" dataset use the "Sparse + DIET + ResponseSelector(T2T)" and
+## "BERT + DIET + ResponseSelector(T2T)" configurations:
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["all"]
+##  - dataset: ["Hermit"]
+##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)", "BERT + DIET(seq) + ResponseSelector(t2t)"]
+#
+## Example: Define a branch name to check-out for a dataset repository. Default branch is 'main'
+## dataset_branch: "test-branch"
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["all"]
+#
+## Example: Define number of repetitions. This will inform how often to repeat all runs defined in the include section. Default is 1
+## num_repetitions: 2
+## include:
+##  - dataset: ["Carbon Bot", "Sara"]
+##    config: ["Sparse + DIET(seq) + ResponseSelector(t2t)"]
+##
+## Shortcuts:
+## You can use the "all" shortcut to include all available configurations or datasets.
+## You can use the "all-nlu" shortcut to include all available NLU configurations or datasets.
+## You can use the "all-core" shortcut to include all available core configurations or datasets.
+
+include:
+  - dataset: ["Carbon Bot"]
+    config: ["Sparse + DIET(bow) + ResponseSelector(bow)"]

.github/configs/mr-test-schedule.json

@@ -0,0 +1,3 @@
+{
+  "body": "```yml\r\ninclude:\r\n  - dataset: [\"all\"]\r\n    config: [\"all\"]\r\n```"
+}

.github/configs/tf-cuda.json

@@ -0,0 +1,25 @@
+{
+  "default_image_tag": "latest",
+  "config": [
+    {
+      "TF": "2.3",
+      "IMAGE_TAG": "cuda-10.1-cudnn7"
+    },
+    {
+      "TF": "2.5",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.6",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.7",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    },
+    {
+      "TF": "2.11",
+      "IMAGE_TAG": "cuda-11.2.0-cudnn8"
+    }
+  ]
+}

.github/dependabot.yml

@@ -0,0 +1,34 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+      time: "13:00"
+    pull-request-branch-name:
+      separator: "-"
+    open-pull-requests-limit: 10
+    labels:
+      - type:dependencies
+      - release:main
+    ignore:
+      - dependency-name: prompt-toolkit
+        versions:
+          - "> 2.0.10"
+      - dependency-name: pytest-asyncio
+        versions:
+          - "> 0.10.0"
+
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+      day: monday
+      time: "12:00"
+    pull-request-branch-name:
+      separator: "-"
+    open-pull-requests-limit: 10
+    reviewers:
+      - RasaHQ/infrastructure-squad
+    labels:
+      - type:dependencies

.github/matchers/flake8-error-matcher.json

@@ -0,0 +1,17 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "flake8-error",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+):\\s+([DCFNWE]\\d+\\s+.+)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.github/no-response.yml

@@ -0,0 +1,12 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: status:more-details-needed
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. Without this,
+  we don't have enough information to help you. Please comment below with the
+  requested information if you still need help.

.github/poetry_version.txt

@@ -0,0 +1,2 @@
+# The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316
+poetry-version=1.8.2

.github/runner/github-runner-deployment.yaml.tmpl

@@ -0,0 +1,74 @@
+# GitHub Runner deployment - uses to deploy a github runner
+# which is used by the CI for model regression tests
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: github-runner-{{getenv "GITHUB_RUN_ID"}}
+  namespace: github-runner
+  labels:
+    app: github-runner
+    pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+spec:
+  replicas: {{getenv "NUM_REPLICAS" "1"}}
+  selector:
+    matchLabels:
+      app: github-runner
+      pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+  template:
+    metadata:
+      labels:
+        app: github-runner
+        pod: github-runner-{{getenv "GITHUB_RUN_ID"}}
+    spec:
+      priorityClassName: high-priority
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 720
+      containers:
+      - name: github-runner
+        image: {{getenv "GH_RUNNER_IMAGE"}}:{{getenv "GH_RUNNER_IMAGE_TAG" "latest"}}
+        imagePullPolicy: Always
+        livenessProbe:
+          initialDelaySeconds: 30
+          periodSeconds: 15
+          failureThreshold: 3
+          exec:
+            command:
+            - /bin/bash
+            - -c
+            - "if [[ `curl -sX GET -H \"Authorization: token ${GITHUB_PAT}\" \
+              https://api.github.com/repos/${GITHUB_OWNER}/${GITHUB_REPOSITORY}/actions/runners | \
+              jq -r '.runners[] | select(.name == \"'${POD_NAME}'\") | .status'` == \"offline\" ]]; then \
+              echo \"The GitHub API returns offline status for the ${POD_NAME} runner\" && exit 1; fi"
+        resources:
+          limits:
+            nvidia.com/gpu: 1
+          requests:
+            nvidia.com/gpu: 1
+            memory: 10G
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        # RUNNER_LABELS - defines labels
+        # with which a github-runner will be registered
+        - name: RUNNER_LABELS
+          value: "self-hosted,gpu,kubernetes,{{getenv "GITHUB_RUN_ID"}}"
+        # GITHUB_OWNER - a name of the repository owner
+        - name: GITHUB_OWNER
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: owner
+        # GITHUB_REPOSITORY - a name of the repository
+        - name: GITHUB_REPOSITORY
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: repository
+        # GITHUB_PAT - Personal Access Token
+        - name: GITHUB_PAT
+          valueFrom:
+            secretKeyRef:
+              name: github-rasa
+              key: pat

.github/scripts/download_pretrained.py

@@ -0,0 +1,120 @@
+import argparse
+import logging
+import time
+from typing import List, NamedTuple, Optional, Text
+
+from transformers import AutoTokenizer, TFAutoModel
+
+import rasa.shared.utils.io
+from rasa.nlu.utils.hugging_face.registry import (
+    model_weights_defaults,
+    model_class_dict,
+)
+
+logger = logging.getLogger(__name__)
+
+COMP_NAME = "LanguageModelFeaturizer"
+DEFAULT_MODEL_NAME = "bert"
+
+
+class LmfSpec(NamedTuple):
+    """Holds information about the LanguageModelFeaturizer."""
+
+    model_name: Text
+    model_weights: Text
+    cache_dir: Optional[Text] = None
+
+
+def get_model_name_and_weights_from_config(
+    config_path: str,
+) -> List[LmfSpec]:
+    config = rasa.shared.utils.io.read_config_file(config_path)
+    logger.info(config)
+    steps = config.get("pipeline", [])
+
+    # Look for LanguageModelFeaturizer steps
+    steps = list(filter(lambda x: x["name"] == COMP_NAME, steps))
+
+    lmf_specs = []
+    for lmfeat_step in steps:
+        if "model_name" not in lmfeat_step:
+            if "model_weights" in lmfeat_step:
+                model_weights = lmfeat_step["model_weights"]
+                raise KeyError(
+                    "When model_name is not given, then model_weights cannot be set. "
+                    f"Here, model_weigths is set to {model_weights}"
+                )
+            model_name = DEFAULT_MODEL_NAME
+            model_weights = model_weights_defaults[DEFAULT_MODEL_NAME]
+        else:
+            model_name = lmfeat_step["model_name"]
+
+            if model_name not in model_class_dict:
+                raise KeyError(
+                    f"'{model_name}' not a valid model name. Choose from "
+                    f"{str(list(model_class_dict.keys()))} or create"
+                    f"a new class inheriting from this class to support your model."
+                )
+
+            model_weights = lmfeat_step.get("model_weights")
+            if not model_weights:
+                logger.info(
+                    f"Model weights not specified. Will choose default model "
+                    f"weights: {model_weights_defaults[model_name]}"
+                )
+                model_weights = model_weights_defaults[model_name]
+        cache_dir = lmfeat_step.get("cache_dir", None)
+        lmf_specs.append(LmfSpec(model_name, model_weights, cache_dir))
+
+    return lmf_specs
+
+
+def instantiate_to_download(comp: LmfSpec) -> None:
+    """Instantiates Auto class instances, but only to download."""
+
+    _ = AutoTokenizer.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+    logger.info("Done with AutoTokenizer, now doing TFAutoModel")
+    _ = TFAutoModel.from_pretrained(comp.model_weights, cache_dir=comp.cache_dir)
+
+
+def download(config_path: str):
+    lmf_specs = get_model_name_and_weights_from_config(config_path)
+
+    if not lmf_specs:
+        logger.info(f"No {COMP_NAME} found, therefore, skipping download")
+        return
+
+    for lmf_spec in lmf_specs:
+        logger.info(
+            f"model_name: {lmf_spec.model_name}, "
+            f"model_weights: {lmf_spec.model_weights}, "
+            f"cache_dir: {lmf_spec.cache_dir}"
+        )
+        start = time.time()
+
+        instantiate_to_download(lmf_spec)
+
+        duration_in_sec = time.time() - start
+        logger.info(f"Instantiating Auto classes takes {duration_in_sec:.2f}seconds")
+
+
+def create_argument_parser() -> argparse.ArgumentParser:
+    """Downloads pretrained models, i.e., Huggingface weights."""
+    parser = argparse.ArgumentParser(
+        description="Downloads pretrained models, i.e., Huggingface weights, "
+        "e.g. path to bert_diet_responset2t.yml"
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        type=str,
+        required=True,
+        help="The path to the config yaml file.",
+    )
+    return parser
+
+
+if __name__ == "__main__":
+    arg_parser = create_argument_parser()
+    cmdline_args = arg_parser.parse_args()
+    download(cmdline_args.config)

.github/scripts/mr_generate_summary.py

@@ -0,0 +1,55 @@
+# Collect the results of the various model test runs which are done as part of
+# the model regression CI pipeline and dump them as a single file artifact.
+# This artifact will the then be published at the end of the tests.
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+from typing import Dict, List
+
+
+def combine_result(
+    result1: Dict[str, dict], result2: Dict[str, Dict[str, Dict]]
+) -> Dict[str, Dict[str, List]]:
+    """Combines 2 result dicts to accumulated dict of the same format.
+
+    Args:
+        result1: dict of key: dataset, value: (dict of key: config, value: list of res)
+                 Example: {
+                              "Carbon Bot": {
+                                  "Sparse + DIET(bow) + ResponseSelector(bow)": [{
+                                      "Entity Prediction": {
+                                          "macro avg": {
+                                              "f1-score": 0.88,
+                                          }
+                                      },
+                                      "test_run_time": "47s",
+                                  }]
+                              }
+                          }
+        result2: dict of key: dataset, value: (dict of key: config, value: list of res)
+
+    Returns:
+        dict of key: dataset, and value: (dict of key: config value: list of results)
+    """
+    combined_dict = defaultdict(lambda: defaultdict(list))
+    for new_dict in [result1, result2]:
+        for dataset, results_for_dataset in new_dict.items():
+            for config, res in results_for_dataset.items():
+                for res_dict in res:
+                    combined_dict[dataset][config].append(res_dict)
+    return combined_dict
+
+
+if __name__ == "__main__":
+    data = {}
+    reports_dir = Path(os.environ["REPORTS_DIR"])
+    reports_paths = list(reports_dir.glob("*/report.json"))
+
+    for report_path in reports_paths:
+        report_dict = json.load(open(report_path))
+        data = combine_result(data, report_dict)
+
+    summary_file = os.environ["SUMMARY_FILE"]
+    with open(summary_file, "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)

.github/scripts/mr_publish_results.py

@@ -0,0 +1,293 @@
+# Send model regression test results to Datadog
+# with a summary of all test results.
+# Also write them into a report file.
+import copy
+import datetime
+import json
+import os
+from typing import Any, Dict, List, Text, Tuple
+
+from datadog_api_client.v1 import ApiClient, Configuration
+from datadog_api_client.v1.api.metrics_api import MetricsApi
+from datadog_api_client.v1.model.metrics_payload import MetricsPayload
+from datadog_api_client.v1.model.point import Point
+from datadog_api_client.v1.model.series import Series
+
+DD_ENV = "rasa-regression-tests"
+DD_SERVICE = "rasa"
+METRIC_RUNTIME_PREFIX = "rasa.perf.benchmark."
+METRIC_ML_PREFIX = "rasa.perf.ml."
+CONFIG_REPOSITORY = "training-data"
+
+TASK_MAPPING = {
+    "intent_report.json": "intent_classification",
+    "CRFEntityExtractor_report.json": "entity_prediction",
+    "DIETClassifier_report.json": "entity_prediction",
+    "response_selection_report.json": "response_selection",
+    "story_report.json": "story_prediction",
+}
+
+METRICS = {
+    "test_run_time": "TEST_RUN_TIME",
+    "train_run_time": "TRAIN_RUN_TIME",
+    "total_run_time": "TOTAL_RUN_TIME",
+}
+
+MAIN_TAGS = {
+    "config": "CONFIG",
+    "dataset": "DATASET_NAME",
+}
+
+OTHER_TAGS = {
+    "config_repository_branch": "DATASET_REPOSITORY_BRANCH",
+    "dataset_commit": "DATASET_COMMIT",
+    "accelerator_type": "ACCELERATOR_TYPE",
+    "type": "TYPE",
+    "index_repetition": "INDEX_REPETITION",
+    "host_name": "HOST_NAME",
+}
+
+GIT_RELATED_TAGS = {
+    "pr_id": "PR_ID",
+    "pr_url": "PR_URL",
+    "github_event": "GITHUB_EVENT_NAME",
+    "github_run_id": "GITHUB_RUN_ID",
+    "github_sha": "GITHUB_SHA",
+    "workflow": "GITHUB_WORKFLOW",
+}
+
+
+def create_dict_of_env(name_to_env: Dict[Text, Text]) -> Dict[Text, Text]:
+    return {name: os.environ[env_var] for name, env_var in name_to_env.items()}
+
+
+def _get_is_external_and_dataset_repository_branch() -> Tuple[bool, Text]:
+    is_external = os.environ["IS_EXTERNAL"]
+    dataset_repository_branch = os.environ["DATASET_REPOSITORY_BRANCH"]
+    if is_external.lower() in ("yes", "true", "t", "1"):
+        is_external_flag = True
+        dataset_repository_branch = os.environ["EXTERNAL_DATASET_REPOSITORY_BRANCH"]
+    else:
+        is_external_flag = False
+    return is_external_flag, dataset_repository_branch
+
+
+def prepare_datasetrepo_and_external_tags() -> Dict[Text, Any]:
+    is_external, dataset_repo_branch = _get_is_external_and_dataset_repository_branch()
+    return {
+        "dataset_repository_branch": dataset_repo_branch,
+        "external_dataset_repository": is_external,
+    }
+
+
+def prepare_dsrepo_and_external_tags_as_str() -> Dict[Text, Text]:
+    return {
+        "dataset_repository_branch": os.environ["DATASET_REPOSITORY_BRANCH"],
+        "external_dataset_repository": os.environ["IS_EXTERNAL"],
+    }
+
+
+def transform_to_seconds(duration: Text) -> float:
+    """Transform string (with hours, minutes, and seconds) to seconds.
+
+    Args:
+        duration: Examples: '1m27s', '1m27.3s', '27s', '1h27s', '1h1m27s'
+
+    Raises:
+        Exception: If the input is not supported.
+
+    Returns:
+        Duration converted in seconds.
+    """
+    h_split = duration.split("h")
+    if len(h_split) == 1:
+        rest = h_split[0]
+        hours = 0
+    else:
+        hours = int(h_split[0])
+        rest = h_split[1]
+    m_split = rest.split("m")
+    if len(m_split) == 2:
+        minutes = int(m_split[0])
+        seconds = float(m_split[1].rstrip("s"))
+    elif len(m_split) == 1:
+        minutes = 0
+        seconds = float(m_split[0].rstrip("s"))
+    else:
+        raise Exception(f"Unsupported duration: {duration}")
+    overall_seconds = hours * 60 * 60 + minutes * 60 + seconds
+    return overall_seconds
+
+
+def prepare_ml_metric(result: Dict[Text, Any]) -> Dict[Text, float]:
+    """Converts a nested result dict into a list of metrics.
+
+    Args:
+        result: Example
+            {'accuracy': 1.0,
+             'weighted avg': {
+                'precision': 1.0, 'recall': 1.0, 'f1-score': 1.0, 'support': 28
+             }
+            }
+
+    Returns:
+        Dict of metric name and metric value
+    """
+    metrics_ml = {}
+    result = copy.deepcopy(result)
+    result.pop("file_name", None)
+    task = result.pop("task", None)
+
+    for metric_name, metric_value in result.items():
+        if isinstance(metric_value, float):
+            metric_full_name = f"{task}.{metric_name}"
+            metrics_ml[metric_full_name] = float(metric_value)
+        elif isinstance(metric_value, dict):
+            for mname, mval in metric_value.items():
+                metric_full_name = f"{task}.{metric_name}.{mname}"
+                metrics_ml[metric_full_name] = float(mval)
+        else:
+            raise Exception(
+                f"metric_value {metric_value} has",
+                f"unexpected type {type(metric_value)}",
+            )
+    return metrics_ml
+
+
+def prepare_ml_metrics(results: List[Dict[Text, Any]]) -> Dict[Text, float]:
+    metrics_ml = {}
+    for result in results:
+        new_metrics_ml = prepare_ml_metric(result)
+        metrics_ml.update(new_metrics_ml)
+
+    return metrics_ml
+
+
+def prepare_datadog_tags() -> List[Text]:
+    tags = {
+        "env": DD_ENV,
+        "service": DD_SERVICE,
+        "branch": os.environ["BRANCH"],
+        "config_repository": CONFIG_REPOSITORY,
+        **prepare_dsrepo_and_external_tags_as_str(),
+        **create_dict_of_env(MAIN_TAGS),
+        **create_dict_of_env(OTHER_TAGS),
+        **create_dict_of_env(GIT_RELATED_TAGS),
+    }
+    tags_list = [f"{k}:{v}" for k, v in tags.items()]
+    return tags_list
+
+
+def send_to_datadog(results: List[Dict[Text, Any]]) -> None:
+    """Sends metrics to datadog."""
+    # Prepare
+    tags_list = prepare_datadog_tags()
+    timestamp = datetime.datetime.now().timestamp()
+    series = []
+
+    # Send metrics about runtime
+    metrics_runtime = create_dict_of_env(METRICS)
+    for metric_name, metric_value in metrics_runtime.items():
+        overall_seconds = transform_to_seconds(metric_value)
+        series.append(
+            Series(
+                metric=f"{METRIC_RUNTIME_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, overall_seconds])],
+                tags=tags_list,
+            )
+        )
+
+    # Send metrics about ML model performance
+    metrics_ml = prepare_ml_metrics(results)
+    for metric_name, metric_value in metrics_ml.items():
+        series.append(
+            Series(
+                metric=f"{METRIC_ML_PREFIX}{metric_name}.gauge",
+                type="gauge",
+                points=[Point([timestamp, float(metric_value)])],
+                tags=tags_list,
+            )
+        )
+
+    body = MetricsPayload(series=series)
+    with ApiClient(Configuration()) as api_client:
+        api_instance = MetricsApi(api_client)
+        response = api_instance.submit_metrics(body=body)
+        if response.get("status") != "ok":
+            print(response)
+
+
+def read_results(file: Text) -> Dict[Text, Any]:
+    with open(file) as json_file:
+        data = json.load(json_file)
+
+        keys = [
+            "accuracy",
+            "weighted avg",
+            "macro avg",
+            "micro avg",
+            "conversation_accuracy",
+        ]
+        result = {key: data[key] for key in keys if key in data}
+
+    return result
+
+
+def get_result(file_name: Text, file: Text) -> Dict[Text, Any]:
+    result = read_results(file)
+    result["file_name"] = file_name
+    result["task"] = TASK_MAPPING[file_name]
+    return result
+
+
+def send_all_to_datadog() -> None:
+    results = []
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if any(f.endswith(valid_name) for valid_name in TASK_MAPPING.keys()):
+                result = get_result(f, os.path.join(dirpath, f))
+                results.append(result)
+    send_to_datadog(results)
+
+
+def generate_json(file: Text, task: Text, data: dict) -> dict:
+    config = os.environ["CONFIG"]
+    dataset = os.environ["DATASET_NAME"]
+
+    if dataset not in data:
+        data = {dataset: {config: []}, **data}
+    elif config not in data[dataset]:
+        data[dataset] = {config: [], **data[dataset]}
+
+    assert len(data[dataset][config]) <= 1
+
+    data[dataset][config] = [
+        {
+            "config_repository": CONFIG_REPOSITORY,
+            **prepare_datasetrepo_and_external_tags(),
+            **create_dict_of_env(METRICS),
+            **create_dict_of_env(OTHER_TAGS),
+            **(data[dataset][config][0] if data[dataset][config] else {}),
+            task: read_results(file),
+        }
+    ]
+    return data
+
+
+def create_report_file() -> None:
+    data = {}
+    for dirpath, dirnames, files in os.walk(os.environ["RESULT_DIR"]):
+        for f in files:
+            if f not in TASK_MAPPING.keys():
+                continue
+
+            data = generate_json(os.path.join(dirpath, f), TASK_MAPPING[f], data)
+
+    with open(os.environ["SUMMARY_FILE"], "w") as f:
+        json.dump(data, f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    send_all_to_datadog()
+    create_report_file()

.github/scripts/start_dd_agent.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+DD_API_KEY=$1
+ACCELERATOR_TYPE=$2
+NVML_INTERVAL_IN_SEC=${3:-15}  # 15 seconds are the default interval
+
+# Install Datadog system agent
+DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$DD_API_KEY DD_SITE="datadoghq.eu" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+DATADOG_YAML_PATH=/etc/datadog-agent/datadog.yaml
+sudo chmod 666 $DATADOG_YAML_PATH
+
+# Associate metrics with tags and env
+{
+    echo "env: rasa-regression-tests"
+    echo "tags:"
+    echo "- service:rasa"
+    echo "- accelerator_type:${ACCELERATOR_TYPE}"
+    echo "- dataset:${DATASET_NAME}"
+    echo "- config:${CONFIG}"
+    echo "- dataset_commit:${DATASET_COMMIT}"
+    echo "- branch:${BRANCH}"
+    echo "- github_sha:${GITHUB_SHA}"
+    echo "- pr_id:${PR_ID:-schedule}"
+    echo "- pr_url:${PR_URL:-schedule}"
+    echo "- type:${TYPE}"
+    echo "- dataset_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- external_dataset_repository:${IS_EXTERNAL:-none}"
+    echo "- config_repository:training-data"
+    echo "- config_repository_branch:${DATASET_REPOSITORY_BRANCH}"
+    echo "- workflow:${GITHUB_WORKFLOW:-none}"
+    echo "- github_run_id:${GITHUB_RUN_ID:-none}"
+    echo "- github_event:${GITHUB_EVENT_NAME:-none}"
+    echo "- index_repetition:${INDEX_REPETITION}"
+    echo "- host_name:${HOST_NAME}"
+    echo ""
+    echo "apm_config:"
+    echo "    enabled: true"
+    echo "process_config:"
+    echo "    enabled: false"
+    echo "use_dogstatsd: true"
+} >> $DATADOG_YAML_PATH
+
+# Enable system_core integration
+sudo mv /etc/datadog-agent/conf.d/system_core.d/conf.yaml.example /etc/datadog-agent/conf.d/system_core.d/conf.yaml
+
+if [[ "${ACCELERATOR_TYPE}" == "GPU" ]]; then
+    # Install and enable NVML integration
+    sudo datadog-agent integration --allow-root install -t datadog-nvml==1.0.1
+    sudo -u dd-agent -H /opt/datadog-agent/embedded/bin/pip3 install grpcio pynvml
+    NVML_CONF_FPATH="/etc/datadog-agent/conf.d/nvml.d/conf.yaml"
+    sudo mv "${NVML_CONF_FPATH}.example" ${NVML_CONF_FPATH}
+    if [[ "${NVML_INTERVAL_IN_SEC}" != 15 ]]; then
+        # Append a line to the NVML config file
+        sudo echo "    min_collection_interval: ${NVML_INTERVAL_IN_SEC}" | sudo tee -a ${NVML_CONF_FPATH} > /dev/null
+    fi
+fi
+
+# Apply changes
+sudo service datadog-agent stop
+
+# Restart agent (such that GPU/NVML metrics are collected)
+# Adusted code from /etc/init/datadog-agent.conf
+INSTALL_DIR="/opt/datadog-agent"
+AGENTPATH="$INSTALL_DIR/bin/agent/agent"
+PIDFILE="$INSTALL_DIR/run/agent.pid"
+AGENT_USER="dd-agent"
+LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $AGENTPATH run -p $PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-process.conf
+TRACE_AGENTPATH="$INSTALL_DIR/embedded/bin/trace-agent"
+TRACE_PIDFILE="$INSTALL_DIR/run/trace-agent.pid"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $TRACE_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $TRACE_AGENTPATH --config $DATADOG_YAML_PATH --pid $TRACE_PIDFILE"
+
+# Adusted code from /etc/init/datadog-agent-trace.conf
+PROCESS_AGENTPATH="$INSTALL_DIR/embedded/bin/process-agent"
+PROCESS_PIDFILE="$INSTALL_DIR/run/process-agent.pid"
+SYSTEM_PROBE_YAML="/etc/datadog-agent/system-probe.yaml"
+sudo -E start-stop-daemon --start --background --quiet --chuid $AGENT_USER --pidfile $PROCESS_PIDFILE --user $AGENT_USER --startas /bin/bash -- -c "LD_LIBRARY_PATH=$LD_LIBRARY_PATH $PROCESS_AGENTPATH --config=$DATADOG_YAML_PATH --sysprobe-config=$SYSTEM_PROBE_YAML --pid=$PROCESS_PIDFILE"

.github/scripts/validate_cpu.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_not_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus > 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_not_available()

.github/scripts/validate_gpus.py

@@ -0,0 +1,14 @@
+import sys
+
+import tensorflow as tf
+
+
+def check_gpu_available():
+    num_gpus = len(tf.config.list_physical_devices("GPU"))
+    print(f"Num GPUs Available: {num_gpus}")
+    if num_gpus <= 0:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    check_gpu_available()

.github/stale.yml

@@ -0,0 +1,18 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+# Label to use when marking an issue as stale
+staleLabel: stale
+
+pulls:
+  # Give more time before closing PRs
+  daysUntilClose: 21
+  # Comment to post when marking a PR as stale. Set to `false` to disable
+  markComment: >
+    This PR has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.
+  # Comment to post when closing a stale PR. Set to `false` to disable
+  closeComment: >
+    This PR has been automatically closed due to inactivity. Please reopen
+    this PR or a new one if you plan to follow-up on it. Thank you for your
+    contributions.

.github/templates/README.md

@@ -0,0 +1,63 @@
+# gomplate templates for GitHub Actions
+
+This document describes gomplate templates use for GitHub Actions.
+
+## Requirements
+
+You have to have installed [gomplate](https://docs.gomplate.ca/installing/) tool in order to render a template file.
+
+> gomplate is a template renderer which supports a growing list of datastores, such as: JSON (including EJSON - encrypted JSON), YAML, AWS EC2 metadata, BoltDB, Hashicorp Consul and Hashicorp Vault secrets.
+
+## Templates
+
+Below you can find a list of templates with their description and the commands to render them.
+
+### `configuration_variables.tmpl`
+
+The template maps dataset name and configuration name for the model regression tests into paths where files are located. As a result, the template returns two environment variables `DATASET` and `CONFIG` which contain paths to file/directory.
+
+#### How to run locally
+
+```shell
+gomplate -d mapping=<path_to_json_file_with_mapping> -f .github/templates/configuration_variables.tmpl
+```
+
+### `model_regression_test_config_comment.tmpl`
+
+The template returns a comment message which is used as a help description in a PR. The template reads the `.github/configs/mr-test-example.yaml` file and include it as example content.
+
+The help message is triggered by adding `status:model-regression-tests` label.
+Comment with a help message is added if a PR doesn't contain a comment with a configuration for the model regression tests.
+
+#### How to run locally
+
+```shell
+gomplate -f .github/templates/model_regression_test_config_comment.tmpl
+```
+
+The template uses the `GITHUB_ACTOR` environment variable, you have to export the variable before executing the command.
+
+### `model_regression_test_config_to_json.tmpl`
+
+The template reads an issue/a PR comment and transforms a YAML code block into JSON.
+
+#### How to run locally
+
+```shell
+gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl
+```
+
+### `model_regression_test_results.tmpl`
+
+The template reads a file with a report (the report file is available as an artifact in the model regression tests workflow) and returns markdown table with a summary of tests.
+
+#### How to run locally
+
+```shell
+gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl
+```
+
+In order to be able to use the `.github/templates/model_regression_test_results.tmpl` template you need the following files:
+
+- `report.json` - the file with a report generated by the `CI - Model Regression` workflow run in a PR. The report is available to download as an artifact in the workflow related to the PR.
+- `report_main.json` - the file with a report generated by the `CI - Model Regression` workflow that is triggered on schedule event. A list of the workflows that you can download an artifact from, can be found [here](https://github.com/RasaHQ/rasa/actions?query=workflow%3A%22CI+-+Model+Regression%22+event%3Aschedule).

.github/templates/configuration_variables.tmpl

@@ -0,0 +1,43 @@
+{{- /*
+
+The template maps dataset name and configuration name for the model
+regression tests into paths where files are located. As a result,
+the template returns two environment variables `DATASET` and `CONFIG`
+which contain paths to file/directory.
+
+ */ -}}
+{{- $mapping := (datasource "mapping") -}}
+{{- $dataset := (index $mapping.datasets (getenv "DATASET_NAME")) -}}
+{{- $config := $mapping.configurations -}}
+{{- if has $dataset "repository" }}
+export DATASET="{{ $dataset.repository }}"
+export IS_EXTERNAL="true"
+echo "::add-mask::{{ $dataset.repository }}"
+{{ if has $dataset "repository_branch" }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="{{ $dataset.repository_branch }}"
+{{ else }}
+export EXTERNAL_DATASET_REPOSITORY_BRANCH="main"
+{{ end }}
+{{- else if has $dataset "path" }}
+export DATASET="{{ $dataset.path }}"
+export IS_EXTERNAL="false"
+echo "::add-mask::{{ $dataset.path }}"
+{{ end }}
+
+{{- if has $dataset "train" }}
+export TRAIN_DIR="{{ $dataset.train }}"
+{{ end }}
+{{- if has $dataset "test" }}
+export TEST_DIR="{{ $dataset.test }}"
+{{ end }}
+{{- if has $dataset "domain" }}
+export DOMAIN_FILE="{{ $dataset.domain }}"
+{{ end }}
+
+{{- if (has $config.nlu (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/nlu/{{ index $config.nlu (getenv "CONFIG_NAME") }}"
+{{ else if (has $config.core (getenv "CONFIG_NAME")) }}
+export CONFIG="{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+echo "::add-mask::{{ $dataset.language }}/core/{{ index $config.core (getenv "CONFIG_NAME") }}"
+{{ end -}}

.github/templates/model_regression_test_config_comment.tmpl

@@ -0,0 +1,45 @@
+{{- /*
+
+The template returns a comment message which is used as a help description
+in a PR. The template reads the `.github/configs/mr-test-example.yaml` file
+and include it as example content.
+
+The help message is triggered by adding `status:model-regression-tests` label.
+Comment with a help message is added if a PR doesn't contain a comment
+with a configuration for the model regression tests.
+
+ */ -}}
+{{ define "check_available_configuration" -}}
+NLU
+{{- if has .dataset "domain" -}}
+, Core
+{{- end -}}
+{{- end -}}
+Hey @{{ .Env.GITHUB_ACTOR }}! :wave: To run model regression tests, comment with the `/modeltest` command and a configuration.
+
+_Tips :bulb:: The model regression test will be run on `push` events. You can re-run the tests by re-add `status:model-regression-tests` label or use a `Re-run jobs` button in Github Actions workflow._
+
+_Tips :bulb:: Every time when you want to change a configuration you should edit the comment with the previous configuration._
+
+You can copy this in your comment and customize:
+
+> /modeltest
+> ~~~yml
+>```yml
+>##########
+>## Available datasets
+>##########
+{{range (coll.Keys (datasource "mapping").datasets)}}># - "{{ . }}" ({{ template "check_available_configuration" (dict "dataset" (index (datasource "mapping").datasets .)) }}){{"\n"}}{{ end -}}
+>
+>##########
+>## Available NLU configurations
+>##########
+{{range (coll.Keys (datasource "mapping").configurations.nlu)}}># - "{{.}}"{{"\n"}}{{ end -}}
+>
+>##########
+>## Available Core configurations
+>##########
+{{range (coll.Keys (datasource "mapping").configurations.core)}}># - "{{.}}"{{"\n"}}{{ end -}}
+>
+{{range split (file.Read ".github/configs/mr-test-example.yaml") "\n"}}>{{.}}{{"\n"}}{{ end -}}
+>```

.github/templates/model_regression_test_config_to_json.tmpl

@@ -0,0 +1,71 @@
+{{- /*
+
+The template reads an issue/a PR comment and transforms a YAML code block into JSON.
+
+*/ -}}
+{{ define "check_config_type" -}}
+{{- if has (datasource "mapping").configurations.nlu . -}}
+nlu
+{{- else if has (datasource "mapping").configurations.core . -}}
+core
+{{- end -}}
+{{- end -}}
+{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
+{{- $num_repetitions := 1 -}}
+{{- if has $config "num_repetitions" -}}
+{{- $num_repetitions = $config.num_repetitions -}}
+{{- end -}}
+{"include":[
+{{- $inc := coll.Slice -}}
+{{- $dataset := coll.Slice -}}
+{{- range $pair := $config.include -}}
+{{- /* use all available datasets if value is equal to all */ -}}
+{{- if eq (index $pair.dataset 0) "all" -}}
+{{ $dataset = (coll.Keys (datasource "mapping").datasets) }}
+{{- else if eq (index $pair.dataset 0) "all-core" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if has $dataset_spec "domain" -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq (index $pair.dataset 0) "all-nlu" -}}
+{{- range $dataset_name, $dataset_spec := (datasource "mapping").datasets -}}
+{{- if not (has $dataset_spec "domain") -}}
+{{ $dataset = (coll.Append $dataset_name $dataset) -}}
+{{- end -}}
+{{- end -}}
+{{- else -}}
+{{- $dataset = $pair.dataset -}}
+{{- end -}}
+{{- range $index_dataset, $value_dataset := $dataset -}}
+{{- range $index_config, $value_config := $pair.config -}}
+{{ range $index_repetition, $element := (strings.Repeat $num_repetitions "x " | strings.Trim " " | strings.Split " ") }}
+{{- /* use all available configurations if value is equal to all */ -}}
+{{- if eq $value_config "all" -}}
+{{- range $config_type := (coll.Keys (datasource "mapping").configurations) -}}
+{{- range $config_name, $config_file := (index (datasource "mapping").configurations $config_type ) -}}
+
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" $config_type | toJSON) $inc) -}}
+{{- end -}}
+{{- end -}}
+{{- else if eq $value_config "all-core" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.core -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "core" | toJSON) $inc) -}}
+{{- end -}}
+{{- else if eq $value_config "all-nlu" -}}
+{{- range $config_name, $config_file := (datasource "mapping").configurations.nlu -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $config_name "type" "nlu" | toJSON) $inc) -}}
+{{- end -}}
+{{- else -}}
+{{- if has (datasource "mapping").configurations.nlu $value_config -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "nlu" | toJSON) $inc) -}}
+{{- else if has (datasource "mapping").configurations.core $value_config -}}
+{{ $inc = (coll.Append (dict "index_repetition" $index_repetition "dataset" $value_dataset "config" $value_config "type" "core" | toJSON) $inc) -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- join $inc "," -}}
+]}

.github/templates/model_regression_test_read_dataset_branch.tmpl

@@ -0,0 +1,13 @@
+{{- /*
+
+The template reads a PR comment and gets the dataset branch for the training-data
+repository.
+
+ */ -}}
+{{- $config := ((datasource "github").body | regexp.Find "```(?s)(.*)```" | regexp.ReplaceLiteral "```.*|\r" "" | yaml | toJSON | json) -}}
+{{- $dataset_branch := "main" -}}
+{{- /* if a branch name for dataset repository is not defined use the main branch */ -}}
+{{- if has $config "dataset_branch" -}}
+{{- $dataset_branch = $config.dataset_branch -}}
+{{- end -}}
+export DATASET_BRANCH="{{ $dataset_branch }}"

.github/templates/model_regression_test_results.tmpl

@@ -0,0 +1,159 @@
+{{- /*
+
+The template reads a file with a report (the report file is available
+as an artifact in the model regression tests workflow) and returns
+a markdown table with a summary of the tests.
+
+*/ -}}
+{{- /*
+
+The print_result_nlu template returns data depends on available fields.
+
+*/ -}}
+{{ define "print_result_nlu" -}}
+{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
+{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
+{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if (has .branch "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} (`no data`)
+{{- else if has (index .branch "micro avg") "f1-score" -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+{{- /*
+
+The print_result_core template returns data depends on available fields.
+
+*/ -}}
+{{ define "print_result_core_micro_avg" -}}
+{{- if and (has (index .branch "micro avg") "f1-score") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if and (has .branch "accuracy") (has .main "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" (.main.accuracy | math.Sub .branch.accuracy) }})
+{{- else if and (has .branch "accuracy") (has (index .main "micro avg") "f1-score") -}}
+{{ printf "%.4f" .branch.accuracy }} ({{ printf "%.2f" ((index (index .main "micro avg") "f1-score") | math.Sub .branch.accuracy) }})
+{{- else if and (has (index .branch "micro avg") "f1-score") (has .main "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} ({{ printf "%.2f" (.main.accuracy | math.Sub (index (index .branch "micro avg") "f1-score")) }})
+{{- else if (has .branch "accuracy") -}}
+{{ printf "%.4f" .branch.accuracy }} (`no data`)
+{{- else if has (index .branch "micro avg") "f1-score" -}}
+{{ printf "%.4f" (index (index .branch "micro avg") "f1-score") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+
+{{ define "print_result_core_conversation_accuracy" -}}
+{{- if and (has (index .branch "conversation_accuracy") "accuracy") (has (index .main "conversation_accuracy") "accuracy") -}}
+{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} ({{ printf "%.2f" ((index (index .main "conversation_accuracy") "accuracy") | math.Sub (index (index .branch "conversation_accuracy") "accuracy")) }})
+{{- else if has (index .branch "conversation_accuracy") "accuracy" -}}
+{{ printf "%.4f" (index (index .branch "conversation_accuracy") "accuracy") }} (`no data`)
+{{- else -}}
+`no data`
+{{- end -}}
+{{- end -}}
+
+{{ define "print_table_nlu" }}
+{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
+{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
+{{- if has $available_types "nlu" -}}
+| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
+|---------------|-----------------|-----------------|-------------------|
+{{ range $config_name, $config_data_array := .results_for_dataset -}}
+{{ range $config_data := $config_data_array }}
+{{- if eq $config_data.type "nlu" -}}
+| `{{ $config_name }}`<br> test: `{{ $config_data.test_run_time }}`, train: `{{ $config_data.train_run_time }}`, total: `{{ $config_data.total_run_time }}`|
+{{- if has $config_data "intent_classification" -}}
+{{- $intent_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $intent_class_main = (index (index $.results_for_dataset_main $config_name) 0).intent_classification -}}
+{{- end -}}
+{{- $intent_class := $config_data.intent_classification -}}
+{{ template "print_result_nlu" (dict "branch" $intent_class "main" $intent_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "entity_prediction" -}}
+{{- $entity_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $entity_class_main = (index (index $.results_for_dataset_main $config_name) 0).entity_prediction -}}
+{{- end -}}
+{{- $entity_class := $config_data.entity_prediction -}}
+{{ template "print_result_nlu" (dict "branch" $entity_class "main" $entity_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "response_selection" -}}
+{{- $response_class_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $response_class_main = (index (index $.results_for_dataset_main $config_name) 0).response_selection -}}
+{{- end -}}
+{{- $response_class := $config_data.response_selection -}}
+{{ template "print_result_nlu" (dict "branch" $response_class "main" $response_class_main) }}|
+{{- else -}}
+`no data`|
+{{- end }}
+{{end}}
+{{- end}}
+{{- end}}
+{{- end -}}
+{{- end -}}
+
+{{- define "print_table_core" -}}
+{{- $available_types := (index .results_for_dataset | jsonpath `@..type`) -}}
+{{- if isKind "string" $available_types }}{{- $available_types = (index .results_for_dataset | jsonpath `@..type` | slice) -}}{{- end -}}
+{{- if has $available_types "core" -}}
+| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
+|---------------|-----------------|-----------------|-------------------|-------------------|
+{{ range $config_name, $config_data_array := .results_for_dataset -}}
+{{ range $config_data := $config_data_array }}
+{{- if eq $config_data.type "core" -}}
+| `{{ $config_name }}` |
+{{- if has $config_data "story_prediction" -}}
+{{- $story_prediction_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
+{{- end -}}
+{{- $story_prediction := $config_data.story_prediction -}}
+{{ template "print_result_core_micro_avg" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+{{- if has $config_data "story_prediction" -}}
+{{- $story_prediction_main := dict -}}
+{{- if has $.results_for_dataset_main $config_name -}}
+{{- $story_prediction_main = (index (index $.results_for_dataset_main $config_name) 0).story_prediction -}}
+{{- end -}}
+{{- $story_prediction := index $config_data.story_prediction -}}
+{{ template "print_result_core_conversation_accuracy" (dict "branch" $story_prediction "main" $story_prediction_main) }}|
+{{- else -}}
+`no data`|
+{{- end -}}
+`{{ $config_data.train_run_time }}`| `{{ $config_data.test_run_time }}`|
+{{ end }}
+{{- end}}
+{{- end}}
+{{- end -}}
+{{- end -}}
+
+{{- $results_main := (datasource "results_main") -}}
+{{ range $dataset, $results_for_dataset := (datasource "data")}}
+{{ $results_for_dataset_main := (index $results_main $dataset) -}}
+{{ $content_dicts := index $results_for_dataset (index (keys $results_for_dataset) 0) -}}
+{{ $one_content_dict := index $content_dicts 0 -}}
+{{- if ($one_content_dict).external_dataset_repository -}}
+Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}` (external repository), commit: `{{ ($one_content_dict).dataset_commit }}`
+Configuration repository branch: `{{ ($one_content_dict).config_repository_branch }}`
+{{ else -}}
+Dataset: `{{$dataset}}`, Dataset repository branch: `{{ ($one_content_dict).dataset_repository_branch }}`, commit: `{{ ($one_content_dict).dataset_commit }}`
+{{ end -}}
+{{ template "print_table_nlu" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
+{{ template "print_table_core" (dict "results_for_dataset" $results_for_dataset "results_for_dataset_main" $results_for_dataset_main) }}
+{{- end }}

.github/tests/test_data/bert_diet_response2t.yml

@@ -0,0 +1,23 @@
+# Configuration for Rasa NLU.
+# https://rasa.com/docs/rasa/nlu/components/
+language: en
+pipeline:
+  - name: WhitespaceTokenizer
+  - name: LanguageModelFeaturizer
+    alias: "lmf"
+  - name: RegexFeaturizer
+    alias: "rf"
+  - name: LexicalSyntacticFeaturizer
+    alias: "lsf"
+  - name: DIETClassifier
+    epochs: 50
+    random_seed: 42
+  - name: ResponseSelector
+    epochs: 100
+    num_transformer_layers: 2
+    transformer_size: 256
+    hidden_layers_size:
+      text: []
+      label: []
+    random_seed: 42
+    featurizers: ["lmf"]

.github/tests/test_data/comment_body.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ndataset_branch: \"test_dataset_branch\"\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

.github/tests/test_data/comment_body_no_dataset_branch.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

.github/tests/test_data/intent_report.json

@@ -0,0 +1,120 @@
+{
+	"search_transactions": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"greet": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"out_of_scope": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"thankyou": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"help": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"inform": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"goodbye": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"affirm": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"pay_cc": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_balance": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 5,
+		"confused_with": {}
+	},
+	"deny": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"ask_transfer_charge": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"transfer_money": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"check_recipients": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_earnings": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"accuracy": 1.0,
+	"macro avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	},
+	"weighted avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	}
+}

.github/tests/test_data/report-on-schedule-2022-02-02.json

@@ -0,0 +1,303 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "35s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m28s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "55s",
+      "total_run_time": "2m8s",
+      "train_run_time": "1m14s",
+      "type": "nlu"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 48,
+          "total": 48,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        }
+      },
+      "test_run_time": "51s",
+      "total_run_time": "8m15s",
+      "train_run_time": "7m24s",
+      "type": "core"
+    }]
+  },
+  "RasaHQ/retail-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "macro avg": {
+          "f1-score": 0.8,
+          "precision": 0.8,
+          "recall": 0.85,
+          "support": 16
+        },
+        "micro avg": {
+          "f1-score": 0.8387096774193549,
+          "precision": 0.8666666666666667,
+          "recall": 0.8125,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.8125,
+          "precision": 0.875,
+          "recall": 0.8125,
+          "support": 16
+        }
+      },
+      "test_run_time": "29s",
+      "total_run_time": "1m16s",
+      "train_run_time": "47s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 0.875,
+        "macro avg": {
+          "f1-score": 0.8300000000000001,
+          "precision": 0.8166666666666667,
+          "recall": 0.85,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.85,
+          "precision": 0.8333333333333333,
+          "recall": 0.875,
+          "support": 16
+        }
+      },
+      "test_run_time": "56s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m6s",
+      "type": "nlu"
+    }],
+    "Rules + Memo": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "conversation_accuracy": {
+          "accuracy": 0.8888888888888888,
+          "correct": 8,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 0.9663698541747322,
+          "precision": 1.0,
+          "recall": 0.946007696007696,
+          "support": 67
+        },
+        "micro avg": {
+          "f1-score": 0.9692307692307692,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 0.9656317714563074,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        }
+      },
+      "test_run_time": "10s",
+      "total_run_time": "19s",
+      "train_run_time": "10s",
+      "type": "core"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    }]
+  }
+}

.github/tests/test_data/report_listformat_core.json

@@ -0,0 +1,70 @@
+{
+  "RasaHQ/retail-demo": {
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    },
+    {
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "41s",
+      "total_run_time": "5m57s",
+      "train_run_time": "5m27s",
+      "type": "core"
+    }]
+  }
+}

.github/tests/test_data/report_listformat_nlu.json

@@ -0,0 +1,98 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "1m29s",
+      "total_run_time": "4m24s",
+      "train_run_time": "2m55s",
+      "type": "nlu"
+    },
+    {
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "2m29s",
+      "total_run_time": "5m24s",
+      "train_run_time": "3m55s",
+      "type": "nlu"
+    }]
+  }
+}

.github/tests/test_download_pretrained.py

@@ -0,0 +1,101 @@
+from copy import deepcopy
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+
+sys.path.append(".github/scripts")
+import download_pretrained  # noqa: E402
+
+CONFIG_FPATH = Path(__file__).parent / "test_data" / "bert_diet_response2t.yml"
+
+
+def test_download_pretrained_lmf_exists_no_params():
+    lmf_specs = download_pretrained.get_model_name_and_weights_from_config(CONFIG_FPATH)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "rasa/LaBSE"
+
+
+def test_download_pretrained_lmf_exists_with_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "roberta"
+    step["cache_dir"] = "/this/dir"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "roberta"
+    assert lmf_specs[0].model_weights == "roberta-base"
+    assert lmf_specs[0].cache_dir == "/this/dir"
+
+
+def test_download_pretrained_unknown_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "unknown"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        with pytest.raises(KeyError):
+            download_pretrained.get_model_name_and_weights_from_config(fp.name)
+
+
+def test_download_pretrained_multiple_model_names():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step_new = deepcopy(step)
+    step_new["model_name"] = "roberta"
+    steps.append(step_new)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 2
+    assert lmf_specs[1].model_name == "roberta"
+
+
+def test_download_pretrained_with_model_name_and_nondefault_weight():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "bert"
+    step["model_weights"] = "bert-base-uncased"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "bert-base-uncased"
+
+
+def test_download_pretrained_lmf_doesnt_exists():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    steps.remove(step)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 0

.github/tests/test_model_regression_test_read_dataset_branch_tmpl.py

@@ -0,0 +1,27 @@
+import pathlib
+import subprocess
+import pytest
+from typing import Text
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_read_dataset_branch.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+DEFAULT_DATASET_BRANCH = "main"
+
+
+@pytest.mark.parametrize(
+    "comment_body_file,expected_dataset_branch",
+    [
+        ("comment_body.json", "test_dataset_branch"),
+        ("comment_body_no_dataset_branch.json", DEFAULT_DATASET_BRANCH),
+    ],
+)
+def test_read_dataset_branch(comment_body_file: Text, expected_dataset_branch: Text):
+    cmd = (
+        "gomplate "
+        f"-d github={TEST_DATA_DIR}/{comment_body_file} "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8").strip()
+    assert output == f'export DATASET_BRANCH="{expected_dataset_branch}"'

.github/tests/test_model_regression_test_results_tmpl.py

@@ -0,0 +1,50 @@
+import pathlib
+import subprocess
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_results.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+
+
+def test_comment_nlu():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_nlu.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/financial-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `52a3ad3eb5292d56542687e23b06703431f15ead`
+Configuration repository branch: `main`
+| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
+|---------------|-----------------|-----------------|-------------------|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `1m29s`, train: `2m55s`, total: `4m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `2m29s`, train: `3m55s`, total: `5m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+
+
+"""  # noqa E501
+    assert output == expected_output
+
+
+def test_comment_core():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_core.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/retail-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `8226b51b4312aa4d3723098cf6d4028feea040b4`
+Configuration repository branch: `main`
+
+| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
+|---------------|-----------------|-----------------|-------------------|-------------------|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`4m27s`| `31s`|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`5m27s`| `41s`|
+
+"""  # noqa E501
+    assert output == expected_output

.github/tests/test_mr_generate_summary.py

@@ -0,0 +1,208 @@
+import sys
+
+sys.path.append(".github/scripts")
+from mr_generate_summary import combine_result  # noqa: E402
+
+
+RESULT1 = {
+    "financial-demo": {
+        "BERT + DIET(bow) + ResponseSelector(bow)": [
+            {
+                "Entity Prediction": {
+                    "macro avg": {
+                        "f1-score": 0.7333333333333333,
+                    }
+                },
+                "test_run_time": "47s",
+            }
+        ]
+    }
+}
+
+
+def test_same_ds_different_config():
+    result2 = {
+        "financial-demo": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_different_ds_same_config():
+    result2 = {
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        },
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        },
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_start_empty():
+    result2 = {}
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition_3times():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    tmp_combined = combine_result(RESULT1, RESULT1)
+    actual_combined = combine_result(tmp_combined, RESULT1)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(RESULT1, tmp_combined)
+    assert actual_combined == expected_combined

.github/tests/test_mr_publish_results.py

@@ -0,0 +1,132 @@
+import os
+from pathlib import Path
+import sys
+from unittest import mock
+
+sys.path.append(".github/scripts")
+from mr_publish_results import (  # noqa: E402
+    prepare_ml_metric,
+    prepare_ml_metrics,
+    transform_to_seconds,
+    generate_json,
+    prepare_datadog_tags,
+)
+
+EXAMPLE_CONFIG = "Sparse + BERT + DIET(seq) + ResponseSelector(t2t)"
+EXAMPLE_DATASET_NAME = "financial-demo"
+
+ENV_VARS = {
+    "BRANCH": "my-branch",
+    "PR_ID": "10927",
+    "PR_URL": "https://github.com/RasaHQ/rasa/pull/10856/",
+    "GITHUB_EVENT_NAME": "pull_request",
+    "GITHUB_RUN_ID": "1882718340",
+    "GITHUB_SHA": "abc",
+    "GITHUB_WORKFLOW": "CI - Model Regression",
+    "IS_EXTERNAL": "false",
+    "DATASET_REPOSITORY_BRANCH": "main",
+    "CONFIG": EXAMPLE_CONFIG,
+    "DATASET_NAME": EXAMPLE_DATASET_NAME,
+    "CONFIG_REPOSITORY_BRANCH": "main",
+    "DATASET_COMMIT": "52a3ad3eb5292d56542687e23b06703431f15ead",
+    "ACCELERATOR_TYPE": "CPU",
+    "TEST_RUN_TIME": "1m54s",
+    "TRAIN_RUN_TIME": "4m4s",
+    "TOTAL_RUN_TIME": "5m58s",
+    "TYPE": "nlu",
+    "INDEX_REPETITION": "0",
+    "HOST_NAME": "github-runner-2223039222-22df222fcd-2cn7d",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_generate_json():
+    f = Path(__file__).parent / "test_data" / "intent_report.json"
+    result = generate_json(f, task="intent_classification", data={})
+    assert isinstance(result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG], list)
+
+    actual = result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG][0]["intent_classification"]
+    expected = {
+        "accuracy": 1.0,
+        "weighted avg": {
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1-score": 1.0,
+            "support": 28,
+        },
+        "macro avg": {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 28},
+    }
+    assert expected == actual
+
+
+def test_transform_to_seconds():
+    assert 87.0 == transform_to_seconds("1m27s")
+    assert 87.3 == transform_to_seconds("1m27.3s")
+    assert 27.0 == transform_to_seconds("27s")
+    assert 3627.0 == transform_to_seconds("1h27s")
+    assert 3687.0 == transform_to_seconds("1h1m27s")
+
+
+def test_prepare_ml_model_perf_metrics():
+    results = [
+        {
+            "macro avg": {
+                "precision": 0.8,
+                "recall": 0.8,
+                "f1-score": 0.8,
+                "support": 14,
+            },
+            "micro avg": {
+                "precision": 1.0,
+                "recall": 0.7857142857142857,
+                "f1-score": 0.88,
+                "support": 14,
+            },
+            "file_name": "DIETClassifier_report.json",
+            "task": "Entity Prediction",
+        },
+        {
+            "accuracy": 1.0,
+            "weighted avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "macro avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "file_name": "intent_report.json",
+            "task": "Intent Classification",
+        },
+    ]
+    metrics_ml = prepare_ml_metrics(results)
+    assert len(metrics_ml) == 17
+
+
+def test_prepare_ml_model_perf_metrics_simple():
+    result = {
+        "accuracy": 1.0,
+        "weighted avg": {"precision": 1, "recall": 1.0, "f1-score": 1, "support": 28},
+        "task": "Intent Classification",
+    }
+    metrics_ml = prepare_ml_metric(result)
+    assert len(metrics_ml) == 5
+
+    for _, v in metrics_ml.items():
+        assert isinstance(v, float)
+
+    key, value = "Intent Classification.accuracy", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+    key, value = "Intent Classification.weighted avg.f1-score", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_prepare_datadog_tags():
+    tags_list = prepare_datadog_tags()
+    assert "dataset:financial-demo" in tags_list

.github/tests/test_validate_gpus.py

@@ -0,0 +1,27 @@
+import os
+import sys
+from unittest import mock
+
+import pytest
+
+sys.path.append(".github/scripts")
+import validate_cpu  # noqa: E402
+import validate_gpus  # noqa: E402
+
+ENV_VARS = {
+    "CUDA_VISIBLE_DEVICES": "-1",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_cpu_succeeds_when_there_are_no_gpus():
+    validate_cpu.check_gpu_not_available()
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_gpus_exits_when_there_are_no_gpus():
+    # This unit test assumes that unit tests are run on a CPU
+    with pytest.raises(SystemExit) as pytest_wrapped_e:
+        validate_gpus.check_gpu_available()
+    assert pytest_wrapped_e.type == SystemExit
+    assert pytest_wrapped_e.value.code == 1

.github/workflows/automatic-pr-update.yml

@@ -0,0 +1,22 @@
+name: Automatic PR Merger
+
+on:
+  push: {} # update PR when base branch is updated
+
+jobs:
+  # thats's all. single step is needed - if PR is mergeable according to
+  # branch protection rules it will be merged automatically
+  mergepal:
+    runs-on: ubuntu-24.04
+    if: github.repository == 'RasaHQ/rasa'
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - uses: rasahq/update-pr-branch@f7012036a6d5659cfbc37f180716963511e81f95
+        with:
+          token: ${{ secrets.UPDATE_BRANCH_PAT }}
+          # required parameter by original action -
+          # check is already done through protected branches so not needed for us
+          required_approval_count: 0
+          # update branch despite failing check runs
+          require_passed_checks: false

.github/workflows/ci-docs-tests.yml

@@ -0,0 +1,164 @@
+name: Docs Tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, labeled]
+
+concurrency:
+  group: ci-docs-tests-${{ github.ref }} # branch name
+  cancel-in-progress: true
+
+env:
+  DEFAULT_PYTHON_VERSION: "3.10"
+
+jobs:
+  changes:
+    name: Check for file changes
+    runs-on: ubuntu-24.04
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
+        id: filter
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: .github/change_filters.yml
+
+  test_documentation:
+    name: Test Documentation
+    runs-on: ubuntu-24.04
+    needs: [changes]
+    if: needs.changes.outputs.docs == 'true' && false # disabled as docs are moved out in new versions
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Set up Node 12.x 🦙
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: "12.x"
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests')
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests'))
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        if: needs.changes.outputs.docs == 'true'
+        run: poetry config virtualenvs.in-project true
+
+      - name: Load Yarn Cached Packages ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: docs/node_modules
+          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-12.x
+
+      - name: Install Dependencies 📦
+        run: |
+          sudo apt-get -y install libpq-dev
+          make install-full install-docs
+
+      - name: Run Swagger 🕵️‍♀️
+        run: |
+          npm install -g swagger-cli
+          swagger-cli validate docs/static/spec/action-server.yml
+          swagger-cli validate docs/static/spec/rasa.yml
+
+      - name: Test Docs 🕸
+        run: make test-docs
+
+  documentation_lint:
+    name: Documentation Linting Checks
+    runs-on: ubuntu-24.04
+    needs: [changes]
+    if: needs.changes.outputs.docs == 'true' && false # disabled as docs are moved out in new versions
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Set up Node 12.x 🦙
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: "12.x"
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests')
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-docs-tests'))
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        if: needs.changes.outputs.docs == 'true'
+        run: poetry config virtualenvs.in-project true
+
+      - name: Load Yarn Cached Packages ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: docs/node_modules
+          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-12.x
+
+      - name: Install Dependencies 📦
+        run: |
+          sudo apt-get -y install libpq-dev
+          make install-full install-docs
+
+      - name: Docs Linting Checks 🕸
+        run: make lint-docs

.github/workflows/ci-github-actions.yml

@@ -0,0 +1,76 @@
+name: CI Github Actions
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+  pull_request:
+
+env:
+  DEFAULT_PYTHON_VERSION: "3.10"
+
+jobs:
+  test:
+    name: Run Tests
+    runs-on: ubuntu-24.04
+    #missing matrix
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests'))
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        run: poetry config virtualenvs.in-project true
+
+      - name: Install Dependencies 📦
+        run: |
+          make install-full
+
+      - name: Lint Code 🎎
+        run: |
+          poetry run ruff check .github --extend-ignore D
+          poetry run black --check .github
+
+      - name: Test Code 🔍
+        run: |
+          make test-gh-actions

.github/workflows/ci-model-regression-on-schedule.yml

@@ -0,0 +1,724 @@
+# The docs: https://www.notion.so/rasa/The-CI-for-model-regression-tests-92af7185e08e4fb2a0c764770a8e9095
+name: CI - Model Regression on schedule
+
+on:
+  schedule:
+    # Run once a week
+    - cron: "1 23 * * */7"
+
+env:
+  GKE_ZONE: us-central1
+  GCLOUD_VERSION: "318.0.0"
+  TF_FORCE_GPU_ALLOW_GROWTH: true
+  GITHUB_ISSUE_LABELS: '["type:bug :bug:", "tool:model-regression-tests"]'
+  PERFORMANCE_DROP_THRESHOLD: -0.05
+  NVML_INTERVAL_IN_SEC: 1
+
+jobs:
+  read_test_configuration:
+    name: Reads tests configuration
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      matrix_length: ${{ steps.set-matrix.outputs.matrix_length }}
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Checkout dataset
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repository: ${{ secrets.DATASET_REPOSITORY }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset"
+          ref: "main"
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Check if a configuration file exists
+        run: test -f .github/configs/mr-test-schedule.json
+
+      - name: Set matrix values
+        id: set-matrix
+        shell: bash
+        run: |-
+          matrix=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -d github=.github/configs/mr-test-schedule.json -f .github/templates/model_regression_test_config_to_json.tmpl)
+          matrix_length=$(echo $matrix | jq '.[] | length')
+          echo "matrix_length=$matrix_length" >> $GITHUB_OUTPUT
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  deploy_runner_gpu:
+    name: Deploy Github Runner - GPU
+    needs: read_test_configuration
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Setup Python
+        id: python
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.8' 
+
+      - name: Export CLOUDSDK_PYTHON env variable
+        run: |
+          echo "CLOUDSDK_PYTHON=${{ steps.python.outputs.python-path }}" >> $GITHUB_OUTPUT
+          export CLOUDSDK_PYTHON=${{ steps.python.outputs.python-path }}
+
+      - name: Download gomplate
+        run: |-
+          curl -o gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          chmod 755 gomplate
+
+      - name: Get TensorFlow version
+        run: |-
+          # Read TF version from poetry.lock file
+          pip install toml
+          TF_VERSION=$(scripts/read_tensorflow_version.sh)
+          echo "TensorFlow version: $TF_VERSION"
+          echo TF_VERSION=$TF_VERSION >> $GITHUB_ENV
+
+        # Use compatible CUDA/cuDNN with the given TF version
+      - name: Prepare GitHub runner image tag
+        run: |-
+          GH_RUNNER_IMAGE_TAG=$(jq -r 'if (.config | any(.TF == "${{ env.TF_VERSION }}" )) then (.config[] | select(.TF == "${{ env.TF_VERSION }}") | .IMAGE_TAG) else .default_image_tag end' .github/configs/tf-cuda.json)
+          echo "GitHub runner image tag for TensorFlow ${{ env.TF_VERSION }} is ${GH_RUNNER_IMAGE_TAG}"
+          echo GH_RUNNER_IMAGE_TAG=$GH_RUNNER_IMAGE_TAG >> $GITHUB_ENV
+
+          num_max_replicas=3
+          matrix_length=${{ needs.read_test_configuration.outputs.matrix_length }}
+          if [[ $matrix_length -gt $num_max_replicas ]]; then
+            NUM_REPLICAS=$num_max_replicas
+          else
+            NUM_REPLICAS=$matrix_length
+          fi
+          echo NUM_REPLICAS=$NUM_REPLICAS >> $GITHUB_ENV
+
+      - name: Send warning if the current TF version does not have CUDA image tags configured
+        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
+        env:
+          TF_CUDA_FILE: ./github/config/tf-cuda.json
+        run: |-
+          echo "::warning file=${TF_CUDA_FILE},line=3,col=1,endColumn=3::Missing cuda config for tf ${{ env.TF_VERSION }}. If you are not sure how to config CUDA, please reach out to infrastructure."
+
+      - name: Notify slack on tf-cuda config updates
+        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: WARNING
+          color: warning
+
+      - name: Render deployment template
+        run: |-
+          export GH_RUNNER_IMAGE_TAG=${{ env.GH_RUNNER_IMAGE_TAG }}
+          export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }}
+          ./gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml
+
+      # Setup gcloud auth
+      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
+        with:
+          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
+          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+
+      # Get the GKE credentials for the cluster
+      - name: Get GKE Cluster Credentials
+        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
+        with:
+          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
+          location: ${{ env.GKE_ZONE }}
+          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}
+
+      - name: Deploy Github Runner
+        run: |-
+          kubectl apply -f runner_deployment.yaml
+          kubectl -n github-runner rollout status --timeout=15m deployment/github-runner-$GITHUB_RUN_ID
+
+      - name: Notify slack on failure
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: FAILED
+          color: danger
+
+  model_regression_test_gpu:
+    name: Model Regression Tests - GPU
+    continue-on-error: true
+    needs:
+      - deploy_runner_gpu
+      - read_test_configuration
+    env:
+      # Determine where CUDA and Nvidia libraries are located. TensorFlow looks for libraries in the given paths
+      LD_LIBRARY_PATH: "/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
+      ACCELERATOR_TYPE: "GPU"
+      GITHUB_ISSUE_TITLE: "Scheduled Model Regression Test Failed"
+    runs-on: [self-hosted, gpu, "${{ github.run_id }}"]
+    strategy:
+      # max-parallel: By default, GitHub will maximize the number of jobs run in parallel depending on the available runners on GitHub-hosted virtual machines.
+      matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Checkout dataset
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repository: ${{ secrets.DATASET_REPOSITORY }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset"
+          ref: "main"
+
+      - name: Set env variables
+        id: set_dataset_config_vars
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG_NAME: "${{ matrix.config }}"
+        run: |-
+          # determine extra environment variables
+          # - CONFIG
+          # - DATASET
+          # - IS_EXTERNAL
+          # - EXTERNAL_DATASET_REPOSITORY_BRANCH
+          # - TRAIN_DIR
+          # - TEST_DIR
+          # - DOMAIN_FILE
+          source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl)
+
+          # Not all configurations are available for all datasets.
+          # The job will fail and the workflow continues, if the configuration file doesn't exist
+          # for a given dataset
+
+          echo "is_dataset_exists=true" >> $GITHUB_OUTPUT
+          echo "is_config_exists=true" >> $GITHUB_OUTPUT
+          echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT
+
+          if [[ "${IS_EXTERNAL}" == "true" ]]; then
+            echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV
+          else
+            echo "DATASET_DIR=dataset" >> $GITHUB_ENV
+            test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0)
+          fi
+
+          # Skip job if dataset is Hermit and config is BERT + DIET(seq) + ResponseSelector(t2t) or Sparse + BERT + DIET(seq) + ResponseSelector(t2t)
+          if [[ "${{ matrix.dataset }}" == "Hermit" && "${{ matrix.config }}" =~ "BERT + DIET(seq) + ResponseSelector(t2t)" ]]; then
+            echo "::warning::This ${{ matrix.dataset }} dataset / ${{ matrix.config }} config is currently being skipped due to OOM associated with the upgrade to TF 2.6." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
+          fi
+
+          # Skip job if a given type is not available for a given dataset
+          if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then
+            echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
+          fi
+
+          test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \
+              && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0)
+
+          echo "DATASET=${DATASET}" >> $GITHUB_ENV
+          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
+          echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV
+          echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV
+          echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV
+
+          if [[ -z "${TRAIN_DIR}" ]]; then
+            echo "TRAIN_DIR=train" >> $GITHUB_ENV
+          else
+            echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV
+          fi
+
+          if [[ -z "${TEST_DIR}" ]]; then
+            echo "TEST_DIR=test" >> $GITHUB_ENV
+          else
+            echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV
+          fi
+
+          HOST_NAME=`hostname`
+          echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout dataset - external
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        if: steps.set_dataset_config_vars.outputs.is_external == 'true'
+        with:
+          repository: ${{ env.DATASET }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset_external"
+          ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }}
+
+      - name: Set dataset commit
+        id: set-dataset-commit
+        working-directory: ${{ env.DATASET_DIR }}
+        run: |
+          DATASET_COMMIT=$(git rev-parse HEAD)
+          echo $DATASET_COMMIT
+          echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT
+
+      - name: Start Datadog Agent
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG: "${{ matrix.config }}"
+          DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}"
+          BRANCH: ${{ github.ref }}
+          GITHUB_SHA: "${{ github.sha }}"
+          TYPE: "${{ matrix.type }}"
+          DATASET_REPOSITORY_BRANCH: "main"
+          INDEX_REPETITION: "${{ matrix.index_repetition }}"
+        run: |
+          .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }}
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          python-version: '3.10'
+
+      - name: Read Poetry Version 🔢
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+
+      - name: Install Dependencies 📦
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          make install-full
+          poetry run python -m spacy download de_core_news_md
+
+      - name: Install datadog-api-client
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: poetry run pip install -U datadog-api-client
+
+      - name: Validate that GPUs are working
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/validate_gpus.py
+
+      - name: Download pretrained models 💪
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG}
+
+      - name: Run test
+        id: run_test
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          TFHUB_CACHE_DIR: ~/.tfhub_cache/
+          OMP_NUM_THREADS: 1
+        run: |-
+          poetry run rasa --version
+
+          export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+          cd ${{ github.workspace }}
+
+          if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then
+            export DATASET=.
+          fi
+
+          if [[ "${{ matrix.type }}" == "nlu" ]]; then
+            poetry run rasa train nlu --quiet -u "${DATASET_DIR}/${DATASET}/${TRAIN_DIR}" -c "dataset/configs/${CONFIG}" --out "${DATASET_DIR}/models/${DATASET}/${CONFIG}"
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run rasa test nlu --quiet -u "${DATASET_DIR}/$DATASET/${TEST_DIR}" -m "${DATASET_DIR}/models/$DATASET/$CONFIG" --out "${{ github.workspace }}/results/$DATASET/$CONFIG"
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+          elif [[ "${{ matrix.type }}" == "core" ]]; then
+            poetry run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE}
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run rasa test core -s "${DATASET_DIR}/${DATASET}/${TEST_DIR}" --out "${{ github.workspace }}/results/${{ matrix.dataset }}/${{ matrix.config }}"
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Generate a JSON file with a report / Publish results to Datadog
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          SUMMARY_FILE: "./report.json"
+          DATASET_NAME: ${{ matrix.dataset }}
+          RESULT_DIR: "${{ github.workspace }}/results"
+          CONFIG: ${{ matrix.config }}
+          TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }}
+          TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }}
+          TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }}
+          DATASET_REPOSITORY_BRANCH: "main"
+          TYPE: ${{ matrix.type }}
+          INDEX_REPETITION: ${{ matrix.index_repetition }}
+          DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }}
+          BRANCH: ${{ github.ref }}
+          PR_ID: "${{ github.event.number }}"
+          PR_URL: ""
+          DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_SITE: datadoghq.eu
+        run: |-
+          poetry run pip install analytics-python
+          poetry run python .github/scripts/mr_publish_results.py
+          cat $SUMMARY_FILE
+
+      - name: Upload an artifact with the report
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }}
+          path: report.json
+
+      # Prepare diagnostic data for the configs which evaluate dialog policy performance
+      - name: Prepare diagnostic data
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' && matrix.type == 'core'
+        run: |
+          # Create dummy files to preserve directory structure when uploading artifacts
+          # See: https://github.com/actions/upload-artifact/issues/174
+          touch "results/${{ matrix.dataset }}/.keep"
+          touch "results/${{ matrix.dataset }}/${{ matrix.config }}/.keep"
+
+      - name: Upload an artifact with diagnostic data
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' && matrix.type == 'core'
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: diagnostic_data
+          path: |
+            results/**/.keep
+            results/${{ matrix.dataset }}/${{ matrix.config }}/failed_test_stories.yml
+            results/${{ matrix.dataset }}/${{ matrix.config }}/story_confusion_matrix.png
+
+      - name: Stop Datadog Agent
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          sudo service datadog-agent stop
+
+      - name: Check duplicate issue
+        if: failure() && github.event_name == 'schedule'
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410
+        id: issue-exists
+        with:
+          result-encoding: string
+          github-token: ${{ github.token }}
+          script: |
+            // Get all open issues
+            const opts = await github.issues.listForRepo.endpoint.merge({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: ${{ env.GITHUB_ISSUE_LABELS }}
+            });
+            const issues = await github.paginate(opts)
+
+            // Check if issue exist by comparing title and body
+            for (const issue of issues) {
+              if (issue.title.includes('${{ env.GITHUB_ISSUE_TITLE }}') &&
+                  issue.body.includes('${{ matrix.dataset }}') &&
+                  issue.body.includes('${{ matrix.config }}')) {
+                  console.log(`Found an exist issue \#${issue.number}. Skip the following steps.`);
+                  return 'true'
+                }
+            }
+            return 'false'
+
+      - name: Create GitHub Issue 📬
+        id: create-issue
+        if: failure() && steps.issue-exists.outputs.result == 'false' && github.event_name == 'schedule'
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410
+        with:
+          # do not use GITHUB_TOKEN here because it wouldn't trigger subsequent workflows
+          github-token: ${{ secrets.RASABOT_GITHUB_TOKEN }}
+          script: |
+            var issue = await github.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: '${{ env.GITHUB_ISSUE_TITLE }}',
+              labels: ${{ env.GITHUB_ISSUE_LABELS }},
+              body: '*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br> Scheduled Model Regression Test failed. <br> **Configuration**: `${{ matrix.config }}` <br> **Dataset**: `${{ matrix.dataset}}`'
+            })
+            return issue.data.number
+
+      - name: Notify Slack of Failure 😱
+        if: failure() && steps.issue-exists.outputs.result == 'false' && github.event_name == 'schedule'
+        uses: 8398a7/action-slack@fbd6aa58ba854a740e11a35d0df80cb5d12101d8 # v3
+        with:
+          status: custom
+          fields: workflow,job,commit,repo,ref,author,took
+          custom_payload: |
+            {
+              attachments: [{
+                fallback: 'fallback',
+                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+                title: `${process.env.AS_WORKFLOW}`,
+                text: 'Scheduled model regression test failed :no_entry:️',
+                fields: [{
+                  title: 'Configuration',
+                  value: '${{ matrix.config }}',
+                  short: false
+                },
+                {
+                  title: 'Dataset',
+                  value: '${{ matrix.dataset }}',
+                  short: false
+                },
+                {
+                  title: 'GitHub Issue',
+                  value: `https://github.com/${{ github.repository }}/issues/${{ steps.create-issue.outputs.result }}`,
+                  short: false
+                },
+                {
+                  title: 'GitHub Action',
+                  value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                  short: false
+                }],
+                actions: [{
+                }]
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_MODEL_REGRESSION_TEST }}
+
+  combine_reports:
+    name: Combine reports
+    runs-on: ubuntu-24.04
+    needs:
+      - model_regression_test_gpu
+    if: always() && needs.model_regression_test_gpu.result == 'success'
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.10'
+
+      - name: Get reports
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          path: reports/
+
+      - name: Display structure of downloaded files
+        continue-on-error: true
+        run: ls -R
+        working-directory: reports/
+
+      - name: Merge all reports
+        env:
+          SUMMARY_FILE: "./report.json"
+          REPORTS_DIR: "reports/"
+        run: |
+          python .github/scripts/mr_generate_summary.py
+          cat $SUMMARY_FILE
+
+      - name: Upload an artifact with the overall report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: report.json
+          path: ./report.json
+
+  analyse_performance:
+    name: Analyse Performance
+    runs-on: ubuntu-24.04
+    if: always() && github.event_name == 'schedule'
+    needs:
+      - model_regression_test_gpu
+      - combine_reports
+    env:
+      GITHUB_ISSUE_TITLE: "Scheduled Model Regression Test Performance Drops"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Download report from last on-schedule regression test
+        run: |
+          # Get ID of last on-schedule workflow
+          SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \
+            | jq -r '.workflows[] | select(.name == "${{ github.workflow }}") | select(.path | test("schedule")) | .id')
+
+          ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url)
+
+          DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \
+            | jq -r '.artifacts[] | select(.name == "report.json") | .archive_download_url')
+
+          # Download the artifact
+          curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL
+
+          # Unzip and change name
+          unzip report.json.zip && mv report.json report_main.json
+
+      - name: Download the report
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: report.json
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Analyse Performance 🔍
+        id: performance
+        run: |
+          OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)"
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          OUTPUT="$(echo $OUTPUT | sed 's|`|\\`|g')"
+          echo "report_description=${OUTPUT}" >> $GITHUB_OUTPUT
+
+          IS_DROPPED=false
+
+          # Loop through all negative values within parentheses
+          # Set IS_DROPPED to true if there is any value lower
+          # than the threshold
+          for x in $(grep -o '\(-[0-9.]\+\)' <<< $OUTPUT); do
+            if (( $(bc -l <<< "${{ env.PERFORMANCE_DROP_THRESHOLD }} > $x") )); then
+              IS_DROPPED=true
+              echo "The decrease of some test performance is > ${{ env.PERFORMANCE_DROP_THRESHOLD }}. Executing the following steps..."
+              break
+            fi
+          done
+
+          echo "is_dropped=$IS_DROPPED" >> $GITHUB_OUTPUT
+
+      - name: Check duplicate issue
+        if: steps.performance.outputs.is_dropped == 'true'
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410
+        id: issue-exists
+        with:
+          result-encoding: string
+          github-token: ${{ github.token }}
+          script: |
+            // Get all open issues based on labels
+            const opts = await github.issues.listForRepo.endpoint.merge({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              labels: ${{ env.GITHUB_ISSUE_LABELS }}
+            });
+            const issues = await github.paginate(opts)
+
+            // Check if issue exist by comparing title
+            for (const issue of issues) {
+              if (issue.title.includes('${{ env.GITHUB_ISSUE_TITLE }}') ) {
+                  console.log(`Found an exist issue \#${issue.number}. Skip the following steps.`);
+                  return 'true'
+                }
+            }
+            return 'false'
+
+      - name: Create GitHub Issue 📬
+        id: create-issue
+        if: steps.performance.outputs.is_dropped == 'true' && steps.issue-exists.outputs.result == 'false'
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410
+        with:
+          # do not use GITHUB_TOKEN here because it wouldn't trigger subsequent workflows
+          github-token: ${{ secrets.RASABOT_GITHUB_TOKEN }}
+          script: |
+            // Prepare issue body
+            let issue_body = '*This PR is automatically created by the Scheduled Model Regression Test workflow. Checkout the Github Action Run [here](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).* <br> --- <br> **Description of Problem:** <br>  Some test performance scores **decreased**. Please look at the following table for more details. <br>'
+            issue_body += `${{ steps.performance.outputs.report_description }}`
+
+            // Open issue
+            var issue = await github.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: '${{ env.GITHUB_ISSUE_TITLE }}',
+              labels: ${{ env.GITHUB_ISSUE_LABELS }},
+              body: issue_body
+            })
+            return issue.data.number
+
+      - name: Notify Slack when Performance Drops 💬
+        if: steps.performance.outputs.is_dropped == 'true' && steps.issue-exists.outputs.result == 'false'
+        uses: 8398a7/action-slack@fbd6aa58ba854a740e11a35d0df80cb5d12101d8 #v3
+        with:
+          status: custom
+          fields: workflow,job,commit,repo,ref,author,took
+          custom_payload: |
+            {
+              attachments: [{
+                fallback: 'fallback',
+                color: 'danger',
+                title: `${process.env.AS_WORKFLOW}`,
+                text: 'Scheduled model regression test performance drops :chart_with_downwards_trend:',
+                fields: [{
+                  title: 'GitHub Issue',
+                  value: `https://github.com/${{ github.repository }}/issues/${{ steps.create-issue.outputs.result }}`,
+                  short: false
+                },
+                {
+                  title: 'GitHub Action',
+                  value: `https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}`,
+                  short: false
+                }],
+                actions: [{
+                }]
+              }]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CI_MODEL_REGRESSION_TEST }}
+
+  remove_runner_gpu:
+    name: Delete Github Runner - GPU
+    if: always()
+    needs:
+      - deploy_runner_gpu
+      - model_regression_test_gpu
+    runs-on: ubuntu-24.04
+
+    steps:
+
+      - name: Setup Python
+        id: python
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.8' 
+
+      - name: Export CLOUDSDK_PYTHON env variable
+        run: |
+          echo "CLOUDSDK_PYTHON=${{ steps.python.outputs.python-path }}" >> $GITHUB_OUTPUT
+          export CLOUDSDK_PYTHON=${{ steps.python.outputs.python-path }}
+
+      # Setup gcloud auth
+      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
+        with:
+          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
+          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+
+      # Get the GKE credentials for the cluster
+      - name: Get GKE Cluster Credentials
+        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
+        with:
+          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
+          location: ${{ env.GKE_ZONE }}
+          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}
+
+      - name: Remove Github Runner
+        run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30

.github/workflows/ci-model-regression.yml

@@ -0,0 +1,923 @@
+# The docs:
+# - https://www.notion.so/rasa/The-CI-for-model-regression-tests-aa579d5524a544af992f97d132bcc2de
+# - https://www.notion.so/rasa/Datadog-Usage-Documentation-422099c9a3a24f5a99d92d904537dd0b
+name: CI - Model Regression
+
+on:
+  push:
+    branches:
+      - "[0-9]+.[0-9]+.x"
+    tags:
+      - "**"
+  pull_request:
+    types: [opened, synchronize, labeled]
+
+concurrency:
+  group: ci-model-regression-${{ github.ref }} # branch or tag name
+  cancel-in-progress: true
+
+env:
+  GKE_ZONE: us-central1
+  GCLOUD_VERSION: "318.0.0"
+  DD_PROFILING_ENABLED: false
+  TF_FORCE_GPU_ALLOW_GROWTH: true
+  NVML_INTERVAL_IN_SEC: 1
+
+jobs:
+  read_test_configuration:
+    name: Reads tests configuration
+    if: ${{ github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }}
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      matrix_length: ${{ steps.set-matrix.outputs.matrix_length }}
+      configuration_id: ${{ steps.fc_config.outputs.comment-id }}
+      dataset_branch: ${{ steps.set-dataset-branch.outputs.dataset_branch }}
+
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Find a comment with configuration
+        uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d
+        id: fc_config
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          issue-number: ${{ github.event.number }}
+          body-includes: "^/modeltest"
+
+      - run: echo ${{ steps.fc_config.outputs.comment-id }}
+
+      # This step has to happen before the other configuration details are read from
+      # the same PR comment, because we need to check out the correct branch to feed the
+      # dataset mapping and configs into the 'Read configuration from a PR comment' step
+      # which creates the experiments matrix
+      - name: Read dataset branch from a PR comment
+        if: steps.fc_config.outputs.comment-id != ''
+        id: set-dataset-branch
+        run: |-
+          source <(gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_read_dataset_branch.tmpl)
+          echo "dataset_branch=${DATASET_BRANCH}" >> $GITHUB_OUTPUT
+
+      - name: Checkout dataset
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repository: ${{ secrets.DATASET_REPOSITORY }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset"
+          ref: ${{ steps.set-dataset-branch.outputs.dataset_branch }}
+
+      - name: Render help description from template
+        id: get_help_description
+        run: |
+          OUTPUT=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/model_regression_test_config_comment.tmpl)
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          echo "help_description=$OUTPUT" >> $GITHUB_OUTPUT
+
+      - name: Create a comment with help description
+        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
+        with:
+          mode: "delete-previous"
+          id: comment_help_description
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          body: |
+            ${{ steps.get_help_description.outputs.help_description }}
+
+      - if: steps.fc_config.outputs.comment-id == ''
+        run: echo "::error::Cannot find a comment with the configuration"
+        name: Log a warning message if a configuration cannot be found
+
+      - name: Read configuration from a PR comment
+        if: steps.fc_config.outputs.comment-id != ''
+        id: set-matrix
+        run: |-
+          matrix=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl)
+
+          if [ $? -ne 0 ]; then
+            echo "::error::Cannot read config from PR. Please double check your config."
+            exit 1
+          fi
+
+          matrix_length=$(echo $matrix | jq '.[] | length')
+          echo "matrix_length=$matrix_length" >> $GITHUB_OUTPUT
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+      - name: Update the comment with the configuration
+        uses: peter-evans/create-or-update-comment@3383acd359705b10cb1eeef05c0e88c056ea4666
+        if: steps.fc_config.outputs.comment-id != ''
+        with:
+          comment-id: ${{ steps.fc_config.outputs.comment-id }}
+          body: |
+            <!-- comment-id:comment_configuration -->
+          reactions: eyes
+
+      - name: Re-create the comment with the configuration
+        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
+        if: steps.fc_config.outputs.comment-id != '' && steps.fc_config.outputs.comment-body != ''
+        with:
+          mode: "delete-previous"
+          id: comment_configuration
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          body: ${{ steps.fc_config.outputs.comment-body }}
+
+      - name: Find a comment with configuration - update
+        uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d
+        id: fc_config_update
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          issue-number: ${{ github.event.number }}
+          body-includes: "^/modeltest"
+
+      - name: Add reaction
+        uses: peter-evans/create-or-update-comment@3383acd359705b10cb1eeef05c0e88c056ea4666
+        if: steps.fc_config_update.outputs.comment-id != ''
+        with:
+          edit-mode: "replace"
+          comment-id: ${{ steps.fc_config_update.outputs.comment-id }}
+          reactions: heart, hooray, rocket
+
+      - name: Add a comment that the tests are in progress
+        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
+        if: steps.fc_config_update.outputs.comment-id != ''
+        with:
+          mode: "delete-previous"
+          id: comment_tests_in_progress
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          body: |
+            The model regression tests have started. It might take a while, please be patient.
+            As soon as results are ready you'll see a new comment with the results.
+
+            Used configuration can be found in [the comment.](https://github.com/${{ github.repository }}/pull/${{ github.event.number}}#issuecomment-${{ steps.fc_config_update.outputs.comment-id }})
+
+  deploy_runner_gpu:
+    name: Deploy Github Runner - GPU
+    needs: read_test_configuration
+    runs-on: ubuntu-24.04
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Get TensorFlow version
+        run: |-
+          # Read TF version from poetry.lock file
+          pip install toml
+          TF_VERSION=$(scripts/read_tensorflow_version.sh)
+          # Keep the first 3 characters, e.g. we keep 2.3 if TF_VERSION is 2.3.4
+          TF_VERSION=${TF_VERSION::3}
+          echo "TensorFlow version: $TF_VERSION"
+          echo TF_VERSION=$TF_VERSION >> $GITHUB_ENV
+
+      # Use compatible CUDA/cuDNN with the given TF version
+      - name: Prepare GitHub runner image tag
+        run: |-
+          GH_RUNNER_IMAGE_TAG=$(jq -r 'if (.config | any(.TF == "${{ env.TF_VERSION }}" )) then (.config[] | select(.TF == "${{ env.TF_VERSION }}") | .IMAGE_TAG) else .default_image_tag end' .github/configs/tf-cuda.json)
+          echo "GitHub runner image tag for TensorFlow ${{ env.TF_VERSION }} is ${GH_RUNNER_IMAGE_TAG}"
+          echo GH_RUNNER_IMAGE_TAG=$GH_RUNNER_IMAGE_TAG >> $GITHUB_ENV
+
+          num_max_replicas=3
+          matrix_length=${{ needs.read_test_configuration.outputs.matrix_length }}
+          if [[ $matrix_length -gt $num_max_replicas ]]; then
+            NUM_REPLICAS=$num_max_replicas
+          else
+            NUM_REPLICAS=$matrix_length
+          fi
+          echo NUM_REPLICAS=$NUM_REPLICAS >> $GITHUB_ENV
+
+      - name: Send warning if the current TF version does not have CUDA image tags configured
+        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
+        env:
+          TF_CUDA_FILE: ./github/config/tf-cuda.json
+        run: |-
+          echo "::warning file=${TF_CUDA_FILE},line=3,col=1,endColumn=3::Missing cuda config for tf ${{ env.TF_VERSION }}. If you are not sure how to config CUDA, please reach out to infrastructure."
+
+      - name: Notify slack on tf-cuda config updates
+        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: WARNING
+          color: warning
+
+      - name: Render deployment template
+        run: |-
+          export GH_RUNNER_IMAGE_TAG=${{ env.GH_RUNNER_IMAGE_TAG }}
+          export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }}
+          gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml
+
+      # Setup gcloud auth
+      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
+        with:
+          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
+          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+
+      # Get the GKE credentials for the cluster
+      - name: Get GKE Cluster Credentials
+        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
+        with:
+          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
+          location: ${{ env.GKE_ZONE }}
+          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}
+
+      - name: Deploy Github Runner
+        run: |-
+          kubectl apply -f runner_deployment.yaml
+          kubectl -n github-runner rollout status --timeout=15m deployment/github-runner-$GITHUB_RUN_ID
+
+  model_regression_test_gpu:
+    name: Model Regression Tests - GPU
+    needs:
+      - deploy_runner_gpu
+      - read_test_configuration
+    env:
+      # Determine where CUDA and Nvidia libraries are located. TensorFlow looks for libraries in the given paths
+      LD_LIBRARY_PATH: "/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
+      ACCELERATOR_TYPE: "GPU"
+    runs-on: [self-hosted, gpu, "${{ github.run_id }}"]
+    strategy:
+      # max-parallel: By default, GitHub will maximize the number of jobs run in parallel depending on the available runners on GitHub-hosted virtual machines.
+      matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}}
+      fail-fast: false
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Checkout dataset
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repository: ${{ secrets.DATASET_REPOSITORY }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset"
+          ref: ${{ needs.read_test_configuration.outputs.dataset_branch }}
+
+      - name: Set env variables
+        id: set_dataset_config_vars
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG_NAME: "${{ matrix.config }}"
+        run: |-
+          # determine extra environment variables
+          # - CONFIG
+          # - DATASET
+          # - IS_EXTERNAL
+          # - EXTERNAL_DATASET_REPOSITORY_BRANCH
+          # - TRAIN_DIR
+          # - TEST_DIR
+          # - DOMAIN_FILE
+          source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl)
+
+          # Not all configurations are available for all datasets.
+          # The job will fail and the workflow continues, if the configuration file doesn't exist
+          # for a given dataset
+
+          echo "is_dataset_exists=true" >> $GITHUB_OUTPUT
+          echo "is_config_exists=true" >> $GITHUB_OUTPUT
+          echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT
+
+          # Warn about job if dataset is Hermit and config is BERT + DIET(seq) + ResponseSelector(t2t) or Sparse + BERT + DIET(seq) + ResponseSelector(t2t)
+          if [[ "${{ matrix.dataset }}" == "Hermit" && "${{ matrix.config }}" =~ "BERT + DIET(seq) + ResponseSelector(t2t)" ]]; then
+            echo "::warning::This ${{ matrix.dataset }} dataset / ${{ matrix.config }} config is currently being skipped on scheduled tests due to OOM associated with the upgrade to TF 2.6. You may see OOM here."
+          fi
+
+          if [[ "${IS_EXTERNAL}" == "true" ]]; then
+            echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV
+          else
+            echo "DATASET_DIR=dataset" >> $GITHUB_ENV
+            test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0)
+          fi
+
+          # Skip job if a given type is not available for a given dataset
+          if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then
+            echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
+          fi
+
+          test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \
+              && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0)
+
+          echo "DATASET=${DATASET}" >> $GITHUB_ENV
+          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
+          echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV
+          echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV
+          echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV
+
+          if [[ -z "${TRAIN_DIR}" ]]; then
+            echo "TRAIN_DIR=train" >> $GITHUB_ENV
+          else
+            echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV
+          fi
+
+          if [[ -z "${TEST_DIR}" ]]; then
+            echo "TEST_DIR=test" >> $GITHUB_ENV
+          else
+            echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV
+          fi
+
+          HOST_NAME=`hostname`
+          echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout dataset - external
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        if: steps.set_dataset_config_vars.outputs.is_external == 'true'
+        with:
+          repository: ${{ env.DATASET }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset_external"
+          ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }}
+
+      - name: Set dataset commit
+        id: set-dataset-commit
+        working-directory: ${{ env.DATASET_DIR }}
+        run: |
+          DATASET_COMMIT=$(git rev-parse HEAD)
+          echo $DATASET_COMMIT
+          echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT
+
+      - name: Start Datadog Agent
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG: "${{ matrix.config }}"
+          DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}"
+          BRANCH: ${{ github.head_ref }}
+          GITHUB_SHA: "${{ github.sha }}"
+          PR_ID: "${{ github.event.number }}"
+          TYPE: "${{ matrix.type }}"
+          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
+          INDEX_REPETITION: "${{ matrix.index_repetition }}"
+        run: |
+          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
+          .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }}
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          python-version: '3.10'
+
+      - name: Read Poetry Version 🔢
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+
+      - name: Install Dependencies 📦
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          make install-full
+          poetry run python -m spacy download de_core_news_md
+
+      - name: Install datadog dependencies
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: poetry run pip install -U datadog-api-client ddtrace
+
+      - name: Validate that GPUs are working
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/validate_gpus.py
+
+      - name: Download pretrained models 💪
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG}
+
+      - name: Run test
+        id: run_test
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          TFHUB_CACHE_DIR: ~/.tfhub_cache/
+          OMP_NUM_THREADS: 1
+        run: |-
+          poetry run rasa --version
+
+          export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+          cd ${{ github.workspace }}
+
+          if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then
+            export DATASET=.
+          fi
+
+          if [[ "${{ matrix.type }}" == "nlu" ]]; then
+            poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG}
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+          elif [[ "${{ matrix.type }}" == "core" ]]; then
+            poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE}
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG}
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Generate a JSON file with a report / Publish results to Datadog
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          SUMMARY_FILE: "./report.json"
+          DATASET_NAME: ${{ matrix.dataset }}
+          RESULT_DIR: "${{ github.workspace }}/results"
+          CONFIG: ${{ matrix.config }}
+          TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }}
+          TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }}
+          TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }}
+          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
+          TYPE: ${{ matrix.type }}
+          INDEX_REPETITION: ${{ matrix.index_repetition }}
+          DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }}
+          BRANCH: ${{ github.head_ref }}
+          GITHUB_SHA: "${{ github.sha }}"
+          PR_ID: "${{ github.event.number }}"
+          DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_SITE: datadoghq.eu
+        run: |-
+          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
+          poetry run pip install analytics-python
+          poetry run python .github/scripts/mr_publish_results.py
+          cat $SUMMARY_FILE
+
+      - name: Upload an artifact with the report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }}
+          path: report.json
+
+      - name: Stop Datadog Agent
+        if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }}
+        run: |
+          sudo service datadog-agent stop
+
+  model_regression_test_cpu:
+    name: Model Regression Tests - CPU
+    needs:
+      - read_test_configuration
+    env:
+      ACCELERATOR_TYPE: "CPU"
+    runs-on: ubuntu-24.04
+    strategy:
+      max-parallel: 3
+      matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}}
+      fail-fast: false
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Checkout dataset
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repository: ${{ secrets.DATASET_REPOSITORY }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset"
+          ref: ${{ needs.read_test_configuration.outputs.dataset_branch }}
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Set env variables
+        id: set_dataset_config_vars
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG_NAME: "${{ matrix.config }}"
+        run: |-
+          # determine extra environment variables
+          # - CONFIG
+          # - DATASET
+          # - IS_EXTERNAL
+          # - EXTERNAL_DATASET_REPOSITORY_BRANCH
+          # - TRAIN_DIR
+          # - TEST_DIR
+          # - DOMAIN_FILE
+          source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl)
+
+          # Not all configurations are available for all datasets.
+          # The job will fail and the workflow continues, if the configuration file doesn't exist
+          # for a given dataset
+
+          echo "is_dataset_exists=true" >> $GITHUB_OUTPUT
+          echo "is_config_exists=true" >> $GITHUB_OUTPUT
+          echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT
+
+          if [[ "${IS_EXTERNAL}" == "true" ]]; then
+            echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV
+          else
+            echo "DATASET_DIR=dataset" >> $GITHUB_ENV
+            test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0)
+          fi
+
+          # Skip job if a given type is not available for a given dataset
+          if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then
+            echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \
+              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
+          fi
+
+          test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \
+              && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0)
+
+          echo "DATASET=${DATASET}" >> $GITHUB_ENV
+          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
+          echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV
+          echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV
+          echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV
+
+          if [[ -z "${TRAIN_DIR}" ]]; then
+            echo "TRAIN_DIR=train" >> $GITHUB_ENV
+          else
+            echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV
+          fi
+
+          if [[ -z "${TEST_DIR}" ]]; then
+            echo "TEST_DIR=test" >> $GITHUB_ENV
+          else
+            echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV
+          fi
+
+          HOST_NAME=`hostname`
+          echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout dataset - external
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        if: steps.set_dataset_config_vars.outputs.is_external == 'true'
+        with:
+          repository: ${{ env.DATASET }}
+          token: ${{ secrets.ML_TEST_SA_PAT }}
+          path: "dataset_external"
+          ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }}
+
+      - name: Set dataset commit
+        id: set-dataset-commit
+        working-directory: ${{ env.DATASET_DIR }}
+        run: |
+          DATASET_COMMIT=$(git rev-parse HEAD)
+          echo $DATASET_COMMIT
+          echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT
+
+      - name: Start Datadog Agent
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          DATASET_NAME: "${{ matrix.dataset }}"
+          CONFIG: "${{ matrix.config }}"
+          DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}"
+          BRANCH: ${{ github.head_ref }}
+          GITHUB_SHA: "${{ github.sha }}"
+          PR_ID: "${{ github.event.number }}"
+          TYPE: "${{ matrix.type }}"
+          DATASET_REPOSITORY_BRANCH: ${{ matrix.dataset_branch }}
+          INDEX_REPETITION: "${{ matrix.index_repetition }}"
+        run: |
+          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
+          .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }}
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          python-version: '3.10'
+
+      - name: Read Poetry Version 🔢
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+
+      - name: Install Dependencies 📦
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |
+          make install-full
+          poetry run python -m spacy download de_core_news_md
+
+      - name: Install datadog dependencies
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: poetry run pip install -U datadog-api-client ddtrace
+
+      - name: CPU run - Validate that no GPUs are available
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/validate_cpu.py
+
+      - name: Download pretrained models 💪
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        run: |-
+          poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG}
+
+      - name: Run test
+        id: run_test
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          TFHUB_CACHE_DIR: ~/.tfhub_cache/
+          OMP_NUM_THREADS: 1
+        run: |-
+          poetry run rasa --version
+
+          export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+          cd ${{ github.workspace }}
+
+          if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then
+            export DATASET=.
+          fi
+
+          if [[ "${{ matrix.type }}" == "nlu" ]]; then
+            poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG}
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+          elif [[ "${{ matrix.type }}" == "core" ]]; then
+            poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE}
+            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+
+            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
+            poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG}
+
+            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Generate a JSON file with a report / Publish results to Datadog
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        env:
+          SUMMARY_FILE: "./report.json"
+          DATASET_NAME: ${{ matrix.dataset }}
+          RESULT_DIR: "${{ github.workspace }}/results"
+          CONFIG: ${{ matrix.config }}
+          TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }}
+          TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }}
+          TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }}
+          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
+          TYPE: ${{ matrix.type }}
+          INDEX_REPETITION: ${{ matrix.index_repetition }}
+          DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }}
+          BRANCH: ${{ github.head_ref }}
+          GITHUB_SHA: "${{ github.sha }}"
+          PR_ID: "${{ github.event.number }}"
+          DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }}
+          DD_API_KEY: ${{ secrets.DD_API_KEY }}
+          DD_SITE: datadoghq.eu
+        run: |-
+          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
+          poetry run pip install analytics-python
+          poetry run python .github/scripts/mr_publish_results.py
+          cat $SUMMARY_FILE
+
+      - name: Upload an artifact with the report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
+        with:
+          name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }}
+          path: report.json
+
+      - name: Stop Datadog Agent
+        if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }}
+        run: |
+          sudo service datadog-agent stop
+
+  combine_reports:
+    name: Combine reports
+    runs-on: ubuntu-24.04
+    needs:
+      - model_regression_test_cpu
+      - model_regression_test_gpu
+    if: ${{ always() && ((needs.model_regression_test_cpu.result != 'skipped') != (needs.model_regression_test_gpu.result != 'skipped')) }}
+    outputs:
+      success_status: ${{ steps.set-success-status.outputs.success_status }}
+
+    steps:
+      - name: Set success status
+        id: set-success-status
+        run: |-
+          succeeded=${{ needs.model_regression_test_cpu.result == 'success' || needs.model_regression_test_gpu.result == 'success' }}
+          if [[ $succeeded == "false" ]]; then
+            success_status="Failed"
+          elif [[ $succeeded == "true" ]]; then
+            success_status="Succeeded"
+          else
+            success_status="Unknown"
+          fi
+          echo $success_status
+          echo "success_status=$success_status" >> $GITHUB_OUTPUT
+
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.10'
+
+      - name: Get reports
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          path: reports/
+
+      - name: Display structure of downloaded files
+        continue-on-error: true
+        run: ls -R
+        working-directory: reports/
+
+      - name: Merge all reports
+        env:
+          SUMMARY_FILE: "./report.json"
+          REPORTS_DIR: "reports/"
+        run: |
+          python .github/scripts/mr_generate_summary.py
+          cat $SUMMARY_FILE
+
+      - name: Upload an artifact with the overall report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: report.json
+          path: ./report.json
+
+  set_job_success_status:
+    name: Set job success status
+    runs-on: ubuntu-24.04
+    needs:
+      - combine_reports
+    if: ${{ always() && needs.combine_reports.result == 'success' }}
+    steps:
+      - name: Set return code
+        run: |
+          success_status=${{ needs.combine_reports.outputs.success_status }}
+          echo "Status: $success_status"
+          if [[ $success_status == "Succeeded" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+
+  add_comment_results:
+    name: Add a comment with the results
+    runs-on: ubuntu-24.04
+    needs:
+      - combine_reports
+    if: ${{ always() && needs.combine_reports.result == 'success' }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Download report from last on-schedule regression test
+        run: |
+          # Get ID of last on-schedule workflow
+          SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \
+            | jq -r '.workflows[] | select(.name == "CI - Model Regression on schedule") | select(.path | test("schedule")) | .id')
+
+          ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url)
+
+          DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \
+            | jq -r '.artifacts[] | select(.name == "report.json") | .archive_download_url')
+
+          # Download the artifact
+          curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL
+
+          # Unzip and change name
+          unzip report.json.zip && mv report.json report_main.json
+
+      - name: Download the report
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: report.json
+
+      - name: Download gomplate
+        run: |-
+          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
+          sudo chmod +x /usr/local/bin/gomplate
+
+      - name: Render a comment to add
+        id: get_results
+        run: |
+          OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)"
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          echo "result=$OUTPUT" >> $GITHUB_OUTPUT
+
+          # Get time of current commit as start time
+          TIME_ISO_COMMIT=$(gomplate -d github=https://api.github.com/repos/rasaHQ/rasa/commits/${{ github.sha }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -i '{{ (ds "github").commit.author.date }}')  # Example "2022-02-17T14:06:38Z"
+          TIME_UNIX_COMMIT=$(date -d "${TIME_ISO_COMMIT}" +%s%3N)  # Example: "1645106798"
+
+          # Get current time
+          TIME_ISO_NOW=$(gomplate -i '{{ (time.Now).UTC.Format time.RFC3339}}') # Example: "2022-02-17T14:50:54Z%"
+          TIME_UNIX_NOW=$(date -d "${TIME_ISO_NOW}" +%s%3N)  # Example: "1645118083"
+
+          echo "from_ts=$TIME_UNIX_COMMIT" >> $GITHUB_OUTPUT
+          echo "to_ts=$TIME_UNIX_NOW" >> $GITHUB_OUTPUT
+
+      - name: Publish results as a PR comment
+        uses: marocchino/sticky-pull-request-comment@f61b6cf21ef2fcc468f4345cdfcc9bda741d2343 # v2.6.2
+        if: ${{ always() }}
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          header: ${{ github.run_id }}
+          append: true
+          message: |-
+            Status of the run: ${{ needs.combine_reports.outputs.success_status }}
+
+            Commit: ${{ github.sha }}, [The full report is available as an artifact.](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
+
+            [Datadog dashboard link](https://app.datadoghq.eu/dashboard/mf4-2hu-x84?tpl_var_branch_baseline=${{ github.head_ref }}&from_ts=${{ steps.get_results.outputs.from_ts }}&to_ts=${{ steps.get_results.outputs.to_ts }}&live=false)
+
+            ${{ steps.get_results.outputs.result }}
+
+      - name: Remove 'status:model-regression-tests' label
+        continue-on-error: true
+        uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          label: "status:model-regression-tests"
+          type: remove
+
+      - name: Remove 'runner:gpu' label
+        continue-on-error: true
+        uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          label: "runner:gpu"
+          type: remove
+
+  remove_runner_gpu:
+    name: Delete Github Runner - GPU
+    needs:
+      - deploy_runner_gpu
+      - model_regression_test_gpu
+    runs-on: ubuntu-24.04
+    if: ${{ always() && needs.deploy_runner_gpu.result != 'skipped' && contains(github.event.pull_request.labels.*.name, 'runner:gpu') && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }}
+
+    steps:
+      # Setup gcloud auth
+      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
+        with:
+          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
+          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}
+
+      # Get the GKE credentials for the cluster
+      - name: Get GKE Cluster Credentials
+        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
+        with:
+          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
+          location: ${{ env.GKE_ZONE }}
+          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}
+
+      - name: Remove Github Runner
+        run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30

.github/workflows/dependabot-batch-updater.yml

@@ -0,0 +1,18 @@
+name: Check if dependencies can be updated and create a pull request with updated ones.
+on:
+  schedule:
+    # Run at 05:00 on Wednesday
+    - cron:  '0 5 * * 3'
+
+jobs:
+  update_dependencies:
+    runs-on: ubuntu-24.04
+    name: Update dependencies
+    steps:
+      - uses: RasaHQ/dependabot-batch-updater@f049cbb0bbd3754bcb5ab154a79f00cd780fc633 # v1.0
+        with:
+          repo-token: ${{ secrets.RASABOT_GITHUB_TOKEN }}
+          repository: RasaHQ/rasa
+          directory: /
+          package-manager: pip
+          batch-size: 5

.github/workflows/documentation.yml

@@ -0,0 +1,325 @@
+name: Publish Documentation
+
+on:
+  push:
+    branches:
+      - "main"
+      - "documentation"
+    tags:
+      - "**"
+  pull_request:
+
+concurrency:
+  # group workflow runs based on the branch or the tag ref
+  group: documentation-${{ github.ref }}
+  cancel-in-progress: true
+
+# SECRETS
+# - GH_DOCS_WRITE_KEY: generated locally, added to github repo (public key)
+#                    `ssh-keygen -t rsa -b 4096 -C "Github CI Docs Key" -N "" -f key`
+# - GITHUB_TOKEN: (default, from github actions)
+# - NETLIFY_AUTH_TOKEN: an access token to use when authenticating commands on Netlify
+# - NETLIFY_SITE_ID: the API ID of the Netlify site for the docs
+
+env:
+  DOCS_FOLDER: docs
+  DOCS_BRANCH: documentation
+  IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
+  IS_MAIN_BRANCH: ${{ github.ref == 'refs/heads/main' }}
+
+jobs:
+  changes:
+    name: Check for file changes
+    runs-on: ubuntu-24.04
+    # don't run this for pull requests of forks
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa'
+    outputs:
+      # Both of the outputs below are strings but only one exists at any given time
+      backend: ${{ steps.changed-files.outputs.backend || steps.run-all.outputs.backend }}
+      docker: ${{ steps.changed-files.outputs.docker || steps.run-all.outputs.docker }}
+      docs: ${{ steps.changed-files.outputs.docs || steps.run-all.outputs.docs }}
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
+        # Run the normal filters if the all-tests-required label is not set
+        id: changed-files
+        if: contains(github.event.pull_request.labels.*.name, 'status:all-tests-required') == false
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: .github/change_filters.yml
+      - name: Set all filters to true if all tests are required
+        # Set all filters to true if the all-tests-required label is set
+        # Bypasses all the change filters in change_filters.yml and forces all outputs to true
+        id: run-all
+        if: contains(github.event.pull_request.labels.*.name, 'status:all-tests-required')
+        run: |
+          echo "backend=true" >> $GITHUB_OUTPUT
+          echo "docker=true" >> $GITHUB_OUTPUT
+          echo "docs=true" >> $GITHUB_OUTPUT
+
+  evaluate_release_tag:
+    name: Evaluate release tag
+    runs-on: ubuntu-24.04
+    # don't run this for main branches of forks and on documentation branch
+    if: github.repository == 'RasaHQ/rasa' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request'
+    outputs:
+      build_docs: ${{ steps.check_tag.outputs.build_docs }}
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Install version library
+        run: |
+          python3 -m pip install pep440_version_utils
+
+      - name: Check if tag version is equal or higher than the latest tagged Rasa version
+        id: check_tag
+        if: env.IS_TAG_BUILD == 'true' || env.IS_MAIN_BRANCH == 'true'
+        run: |
+          if [[ "${IS_MAIN_BRANCH}" == "true" ]]; then
+            echo "Main branch: setting build_docs to true."
+            echo "build_docs=true" >> $GITHUB_OUTPUT
+          else
+            # Get latest tagged Rasa version
+            git describe --tags --match="3.6.[0-9]*" --abbrev=0 HEAD
+            # Fetch branch history
+            TAG_NAME=${GITHUB_REF#refs/tags/}
+            git fetch --prune --unshallow
+            python scripts/evaluate_release_tag.py $TAG_NAME
+            exit_status=$?
+
+            if [[ ${exit_status} -eq 0 ]]; then
+              echo "Setting build_docs to true."
+              echo "build_docs=true" >> $GITHUB_OUTPUT
+            else
+              echo "Setting build_docs to false."
+              echo "build_docs=false" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+  prebuild_docs:
+    name: Prebuild Docs
+    runs-on: ubuntu-24.04
+    needs: [evaluate_release_tag]
+    # don't run this for main branches of forks, would fail anyways
+    if: github.repository == 'RasaHQ/rasa' && needs.evaluate_release_tag.outputs.build_docs == 'true' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request'
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.10 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.10'
+
+      - name: Set up Node 12.x 🦙
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: "12.x"
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-3.9-non-full
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true'
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: steps.cache-poetry.outputs.cache-hit != 'true'
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        run: poetry config virtualenvs.in-project true
+
+      - name: Load Yarn Cached Packages ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: docs/node_modules
+          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-12.x
+
+      - name: Install Dependencies 📦
+        run: make install install-docs
+
+      - name: Pre-build Docs 🧶
+        run: make prepare-docs
+
+      - name: Push docs to documentation branch 🏃‍♀️
+        env:
+          GH_DOCS_WRITE_KEY: ${{ secrets.GH_DOCS_WRITE_KEY }}
+          TMP_DOCS_FOLDER: /tmp/documentation-${{ github.run_id }}
+          TMP_SSH_KEY_PATH: /tmp/docs_key
+        run: |
+          eval "$(ssh-agent -s)"; touch $TMP_SSH_KEY_PATH; chmod 0600 $TMP_SSH_KEY_PATH
+          echo "$GH_DOCS_WRITE_KEY" > $TMP_SSH_KEY_PATH
+          ssh-add $TMP_SSH_KEY_PATH
+
+          git config --global user.email "builds@github-ci.com"
+          git config --global user.name "GitHub CI"
+          git remote set-url --push origin "git@github.com:${{github.repository}}"
+
+          ./scripts/push_docs_to_branch.sh
+
+      - name: Notify slack on failure
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e # v1.6.0
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: FAILED
+          color: warning
+
+  preview_docs:
+    name: Preview Docs
+    runs-on: ubuntu-24.04
+    needs: [changes]
+    # don't run this for pull requests from forks
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa'
+
+    steps:
+      - name: Checkout git repository 🕝
+        if: needs.changes.outputs.docs == 'true'
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.10 🐍
+        if: needs.changes.outputs.docs == 'true'
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.10'
+
+      - name: Set up Node 12.x 🦙
+        if: needs.changes.outputs.docs == 'true'
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: "12.x"
+
+      - name: Read Poetry Version 🔢
+        if: needs.changes.outputs.docs == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        if: needs.changes.outputs.docs == 'true'
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        if: needs.changes.outputs.docs == 'true'
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-3.9-non-full
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.docs == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-preview-docs')
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-preview-docs')) && needs.changes.outputs.docs == 'true'
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        if: needs.changes.outputs.docs == 'true'
+        run: poetry config virtualenvs.in-project true
+
+      - name: Load Yarn Cached Packages ⬇
+        if: needs.changes.outputs.docs == 'true'
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: docs/node_modules
+          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-12.x
+
+      - name: Install Dependencies 📦
+        if: needs.changes.outputs.docs == 'true'
+        run: make install install-docs
+
+      - name: Pre-build Docs 🧶
+        if: needs.changes.outputs.docs == 'true'
+        run: make prepare-docs
+
+      - name: Preview draft build 🔬
+        if: needs.changes.outputs.docs == 'true'
+        id: preview_draft_build
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          DOCS_SITE_BASE_URL: /docs/rasa
+          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          make preview-docs
+          DEPLOY_URL="https://$PULL_REQUEST_NUMBER--rasahq-docs-rasa-v2.netlify.app${DOCS_SITE_BASE_URL}"
+          echo "preview_url=$DEPLOY_URL" >> $GITHUB_OUTPUT
+
+      - name: Create a comment with help description
+        if: needs.changes.outputs.docs == 'true'
+        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
+        with:
+          mode: "delete-previous"
+          id: comment_docs_previews
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          body: |
+            🚀 A preview of the docs have been deployed at the following URL: ${{ steps.preview_draft_build.outputs.preview_url }}
+
+  publish_docs:
+    name: Publish Docs
+    runs-on: ubuntu-24.04
+    # don't run this for main branches of forks; only run on documentation branch
+    if: github.repository == 'RasaHQ/rasa' && github.ref == 'refs/heads/documentation'
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Node 12.x 🦙
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c
+        with:
+          node-version: "12.x"
+
+      - name: Load Yarn Cached Packages ⬇
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: docs/node_modules
+          key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-12.x
+
+      - name: Install Dependencies 📦
+        run: make install-docs
+
+      - name: Publish production build ✅
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+        run: make publish-docs
+
+      - name: Notify slack on failure
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e # v1.6.0
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: FAILED
+          color: warning

.github/workflows/nightly_release.yml

@@ -0,0 +1,178 @@
+name: Nightly Builds
+on:
+  schedule:
+    # Runs every weekday at 1am
+    - cron: 0 1 * * 1-5
+  workflow_dispatch:
+
+
+jobs:
+  run_script_and_tag_nightly_release:
+    name: Run release script and tag a new nightly release
+    runs-on: ubuntu-24.04
+    outputs:
+      tag_name: ${{ steps.set_tagname.outputs.tag_name }}
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Python module
+        run: |
+          python3 -m pip install pluggy
+          python3 -m pip install ruamel.yaml
+
+      - name: Compose tag name
+        id: set_tagname
+        run: |
+          DATE=$(date +'%Y%m%d')
+
+          # Find latest rasa-oss version
+          echo "Trying to find the latest rasa-oss version..."
+          LATEST_RASA_MINOR=$(python -c "import sys; import os; sys.path.append('${{ github.workspace }}/rasa'); from rasa.version import __version__; print(__version__)")
+          echo "Current RASA version: ${LATEST_RASA_MINOR}"
+
+          LATEST_NIGHTLY_VERSION=$(echo ${LATEST_RASA_MINOR})
+
+          echo "Composing nightly build tag name..."
+          GH_TAG=${LATEST_NIGHTLY_VERSION}.dev${DATE}
+          echo "New nightly release version: ${GH_TAG}"
+          echo "tag_name=${GH_TAG}" >> $GITHUB_OUTPUT
+
+      - name: Tag latest main commit as nightly
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+
+          git tag -a ${{ steps.set_tagname.outputs.tag_name }} -m "This is an internal development build"
+          git push origin ${{ steps.set_tagname.outputs.tag_name }} --tags
+
+
+  deploy:
+    name: Deploy to PyPI
+    runs-on: ubuntu-24.04
+
+    # deploy will only be run when there is a tag available
+    needs: run_script_and_tag_nightly_release  # only run after all other stages succeeded
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.9 🐍
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: 3.9
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Copy Segment write key to the package
+        env:
+          RASA_TELEMETRY_WRITE_KEY: ${{ secrets.RASA_OSS_TELEMETRY_WRITE_KEY }}
+          RASA_EXCEPTION_WRITE_KEY: ${{ secrets.RASA_OSS_EXCEPTION_WRITE_KEY }}
+        run: |
+          ./scripts/write_keys_file.sh
+
+      - name: Update version (nightly releases) 🚀
+        run: |
+          poetry run pip install toml pep440_version_utils
+          poetry run python ./scripts/prepare_nightly_release.py --next_version "${{ needs.run_script_and_tag_nightly_release.outputs.tag_name }}"
+
+      - name: Build ⚒️ Distributions
+        run: |
+          poetry build
+
+      # Authenticate and push to the release registry
+      - id: 'auth-release'
+        name: Authenticate with gcloud for release registry 🎫
+        uses: 'google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf'
+        with:
+          credentials_json: '${{ secrets.RASA_OSS_RELEASE_ACCOUNT_KEY }}'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@62d4898025f6041e16b1068643bfc5a696863587'
+
+      - name: Release via GCP Artifact Registry
+        run: |
+          pip install keyring
+          pip install keyrings.google-artifactregistry-auth
+          pip install twine
+          gcloud artifacts print-settings python --project=rasa-releases --repository=rasa --location=europe-west3 > ~/.pypirc
+          twine upload --verbose --repository-url https://europe-west3-python.pkg.dev/rasa-releases/rasa/ ${{ format('{0}/dist/*', github.workspace) }} 
+
+  docker:
+    name: Build Docker
+    runs-on: ubuntu-24.04
+    needs: run_script_and_tag_nightly_release
+    env:
+      GCLOUD_VERSION: "297.0.1"
+      # Registry used to store Docker images used for release purposes
+      DEV_REGISTRY: "europe-west3-docker.pkg.dev/rasa-ci-cd/rasa"
+      IMAGE_TAG: ${{ needs.run_script_and_tag_nightly_release.outputs.tag_name }}
+
+    steps:
+      - name: Checkout git repository 🕝
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Free disk space
+        # tries to make sure we do not run out of disk space, see
+        # https://github.community/t5/GitHub-Actions/BUG-Strange-quot-No-space-left-on-device-quot-IOExceptions-on/td-p/46101
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+          df -h
+
+      - name: Read Poetry Version 🔢
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+        id: buildx
+        with:
+          version: v0.5.1
+          driver: docker
+
+      - name: Copy Segment write key to the package
+        env:
+          RASA_TELEMETRY_WRITE_KEY: ${{ secrets.RASA_OSS_TELEMETRY_WRITE_KEY }}
+          RASA_EXCEPTION_WRITE_KEY: ${{ secrets.RASA_OSS_EXCEPTION_WRITE_KEY }}
+        run: |
+          ./scripts/write_keys_file.sh
+
+      - name: Build Docker image
+        run: |
+          docker build . -t rasa/rasa:base-localdev -f docker/Dockerfile.base
+          docker build . -t rasa/rasa:base-builder-localdev -f docker/Dockerfile.base-builder --build-arg IMAGE_BASE_NAME=rasa/rasa --build-arg POETRY_VERSION=${{ env.POETRY_VERSION }}
+          docker build . -t rasa/rasa:base-poetry -f docker/Dockerfile.base-poetry --build-arg IMAGE_BASE_NAME=rasa/rasa --build-arg BASE_IMAGE_HASH=localdev
+          docker build . -t rasa/rasa:${IMAGE_TAG} -f Dockerfile --build-arg IMAGE_BASE_NAME=rasa/rasa --build-arg BASE_IMAGE_HASH=localdev --build-arg BASE_BUILDER_IMAGE_HASH=localdev
+          docker tag rasa/rasa:${IMAGE_TAG} ${{env.DEV_REGISTRY}}/rasa:${IMAGE_TAG}
+
+      # Authenticate and push to the release registry
+      - id: 'auth-dev'
+        name: Authenticate with gcloud for dev registry 🎫
+        uses: 'google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf'
+        with:
+          credentials_json: '${{ secrets.RASA_OSS_RELEASE_ACCOUNT_KEY }}'
+
+      - name: Authenticate docker for dev registry 🎫
+        run: |
+          # Set up docker to authenticate via gcloud command-line tool.
+          gcloud auth configure-docker europe-west3-docker.pkg.dev
+
+      - name: Push image to release registry
+        run: |
+          docker push ${{env.DEV_REGISTRY}}/rasa:${IMAGE_TAG}

.github/workflows/rasa-install-cron-check.yml

@@ -0,0 +1,48 @@
+# This is a installation test that we run daily. We attempt to install Rasa
+# on Ubuntu, Mac and Windows and try to run `rasa init`. The goal here is to
+# catch installation issues in an automated way.
+
+name: Cron Test to Check Pip Installation
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          [ubuntu-24.04, ubuntu-18.04, macos-latest, windows-2019, windows-2022]
+        python-version: [3.8, 3.9, '3.10']
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Try to install Rasa.
+        run: |
+          python -m pip install rasa
+      - name: Try to install Rasa[full].
+        run: |
+          python -m pip install rasa[full]
+      - name: Try to run Rasa.
+        run: |
+          python -m rasa --version
+          # Must create a folder first
+          mkdir rasa-demo
+          # Next we try to init. 
+          python -m rasa init --init-dir rasa-demo --no-prompt
+      - name: Notify slack on failure
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e
+        with:
+          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
+          status: FAILED
+          color: danger

.github/workflows/security-scans.yml

@@ -0,0 +1,131 @@
+name: Security Scans
+
+on:
+  pull_request:
+    types: [opened, synchronize, labeled]
+
+concurrency:
+  group: security-scans-${{ github.head_ref }} # head branch name
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    name: Check for file changes
+    runs-on: ubuntu-24.04
+    outputs:
+      backend: ${{ steps.filter.outputs.backend }}
+      docker: ${{ steps.filter.outputs.docker }}
+      docs: ${{ steps.filter.outputs.docs }}
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
+        id: filter
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: .github/change_filters.yml
+
+  trivy:
+    name: Detecting hardcoded secrets
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+      with:
+        # Fetch all history for all tags and branches
+        fetch-depth: '0'
+    - name: Run Trivy vulnerability scanner
+      id: trivy
+      uses: aquasecurity/trivy-action@e5f43133f6e8736992c9f3c1b3296e24b37e17f2
+      continue-on-error: true
+      with:
+        format: 'table'
+        scan-type: 'fs'
+        exit-code: '1'
+        scanners: 'secret'
+    - name: Alert on secret finding
+      if: steps.trivy.outcome == 'failure'
+      uses: slackapi/slack-github-action@007b2c3c751a190b6f0f040e47ed024deaa72844
+      with:
+        payload: |
+          {
+            "text": "*A secret was detected in a GitHub commit in the repo ${{ github.repository }}.*\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}",
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "*A secret was detected in a GitHub commit in the repo ${{ github.repository }}.*\n${{ github.event.pull_request.html_url || github.event.head_commit.url }}"
+                }
+              }
+            ]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CODESECURITY_WEBHOOK_URL }}
+        SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+    - name: Fail build if a secret is found
+      if: steps.trivy.outcome == 'failure'
+      run: |
+        echo "=========================================================="
+        echo "| This build has failed because Trivy detected a secret. |"
+        echo "=========================================================="
+        echo "1. Check the step 'Run Trivy vulnerability scanner' for output to help you find the secret."
+        echo "2. If the finding is a false positive, add it as an entry to trivy-secret.yaml in the root of the repo to suppress the finding."
+        echo "3. If the finding is valid, the security team can help advise your next steps."
+        exit 1
+
+
+  bandit:
+    name: Detect python security issues
+    runs-on: ubuntu-24.04
+    needs: [changes]
+
+    steps:
+      - name: Checkout git repository 🕝
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+      - name: Set up Python 3.10 🐍
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+        with:
+          python-version: '3.10'
+
+      - name: Read Poetry Version 🔢
+        if: needs.changes.outputs.backend == 'true'
+        run: |
+          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Install poetry 🦄
+        if: needs.changes.outputs.backend == 'true'
+        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
+        with:
+          poetry-version: ${{ env.POETRY_VERSION }}
+
+      - name: Load Poetry Cached Libraries ⬇
+        id: cache-poetry
+        if: needs.changes.outputs.backend == 'true'
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}
+          restore-keys: ${{ runner.os }}-poetry-3.9
+
+      - name: Clear Poetry cache
+        if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-security-scans')
+        run: rm -r .venv
+
+      - name: Create virtual environment
+        if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-security-scans')) && needs.changes.outputs.backend == 'true'
+        run: python -m venv create .venv
+
+      - name: Set up virtual environment
+        if: needs.changes.outputs.backend == 'true'
+        run: poetry config virtualenvs.in-project true
+
+      - name: Install Dependencies (Linux) 📦
+        if: needs.changes.outputs.backend == 'true'
+        run: make install
+
+      - name: Run Bandit 🔪
+        if: needs.changes.outputs.backend == 'true'
+        run: make lint-security

.github/workflows/semgrep-check.yml

@@ -0,0 +1,33 @@
+# Name of this GitHub Actions workflow.
+name: Semgrep
+
+on:
+  # Scan mainline branches and report all findings:
+  push:
+    branches:
+      - main
+  # Scan changed files in PRs (diff-aware scanning):
+  pull_request:
+
+jobs:
+  semgrep:
+    # User-definable name of this GitHub Actions job:
+    name: Semgrep Workflow Security Scan
+    # If you are self-hosting, change the following `runs-on` value:
+    runs-on: ubuntu-24.04
+
+    container:
+      # A Docker image with Semgrep installed. Do not change this.
+      image: returntocorp/semgrep@sha256:37736e4992c539f760e36e14d48924bd9fa70d0abbde39a6d86d93f66a1affd4
+
+    # To skip any PR created by dependabot to avoid permission issues:
+    if: (github.actor != 'dependabot[bot]')
+
+    steps:
+      # Fetch project source with GitHub Actions Checkout.
+      - uses: actions/checkout@v3
+      # Run the "semgrep ci" command on the command line of the docker image.
+      - run: semgrep ci
+        env:
+          # Add the rules that Semgrep uses by setting the SEMGREP_RULES environment variable.
+          SEMGREP_RULES: p/github-actions

.github/workflows/spellcheck.yml

@@ -0,0 +1,21 @@
+name: Typo CI
+
+on:
+  push:
+    branches-ignore:
+      - main
+jobs:
+  spellcheck:
+    name: Typo CI (GitHub Action)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 4
+    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+    steps:
+    - name: TypoCheck
+      uses: typoci/spellcheck-action@a63b1430c8f0ceed9fd0bd0f3ad8f7466e6c695d # v1.1.0
+      # with:
+        # A license can be purchased via:
+        # https://gumroad.com/l/MvvBE
+        # typo_ci_license_key: ${{ secrets.TYPO_CI_LICENSE_KEY }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}