chore: import upstream snapshot with attribution

.github/tests/test_data/bert_diet_response2t.yml

@@ -0,0 +1,23 @@
+# Configuration for Rasa NLU.
+# https://rasa.com/docs/rasa/nlu/components/
+language: en
+pipeline:
+  - name: WhitespaceTokenizer
+  - name: LanguageModelFeaturizer
+    alias: "lmf"
+  - name: RegexFeaturizer
+    alias: "rf"
+  - name: LexicalSyntacticFeaturizer
+    alias: "lsf"
+  - name: DIETClassifier
+    epochs: 50
+    random_seed: 42
+  - name: ResponseSelector
+    epochs: 100
+    num_transformer_layers: 2
+    transformer_size: 256
+    hidden_layers_size:
+      text: []
+      label: []
+    random_seed: 42
+    featurizers: ["lmf"]

.github/tests/test_data/comment_body.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ndataset_branch: \"test_dataset_branch\"\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

.github/tests/test_data/comment_body_no_dataset_branch.json

@@ -0,0 +1,3 @@
+{
+  "body": "/modeltest\r\n\r\n```yml\r\ninclude:\r\n - dataset: [\"financial-demo\"]\r\n   config: [\"TEST\"]\r\n ```\r\n\r\n<!-- comment-id:comment_configuration -->"
+}

.github/tests/test_data/intent_report.json

@@ -0,0 +1,120 @@
+{
+	"search_transactions": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"greet": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"out_of_scope": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"thankyou": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"help": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"inform": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"goodbye": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"affirm": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"pay_cc": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_balance": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 5,
+		"confused_with": {}
+	},
+	"deny": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"ask_transfer_charge": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 1,
+		"confused_with": {}
+	},
+	"transfer_money": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 3,
+		"confused_with": {}
+	},
+	"check_recipients": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"check_earnings": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 2,
+		"confused_with": {}
+	},
+	"accuracy": 1.0,
+	"macro avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	},
+	"weighted avg": {
+		"precision": 1.0,
+		"recall": 1.0,
+		"f1-score": 1.0,
+		"support": 28
+	}
+}

.github/tests/test_data/report-on-schedule-2022-02-02.json

@@ -0,0 +1,303 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "35s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m28s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "55s",
+      "total_run_time": "2m8s",
+      "train_run_time": "1m14s",
+      "type": "nlu"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 48,
+          "total": 48,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 317
+        }
+      },
+      "test_run_time": "51s",
+      "total_run_time": "8m15s",
+      "train_run_time": "7m24s",
+      "type": "core"
+    }]
+  },
+  "RasaHQ/retail-demo": {
+    "BERT + DIET(bow) + ResponseSelector(bow)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "macro avg": {
+          "f1-score": 0.8,
+          "precision": 0.8,
+          "recall": 0.85,
+          "support": 16
+        },
+        "micro avg": {
+          "f1-score": 0.8387096774193549,
+          "precision": 0.8666666666666667,
+          "recall": 0.8125,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.8125,
+          "precision": 0.875,
+          "recall": 0.8125,
+          "support": 16
+        }
+      },
+      "test_run_time": "29s",
+      "total_run_time": "1m16s",
+      "train_run_time": "47s",
+      "type": "nlu"
+    }],
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.25,
+          "precision": 0.25,
+          "recall": 0.25,
+          "support": 6
+        },
+        "micro avg": {
+          "f1-score": 0.2857142857142857,
+          "precision": 1.0,
+          "recall": 0.16666666666666666,
+          "support": 6
+        },
+        "weighted avg": {
+          "f1-score": 0.16666666666666666,
+          "precision": 0.16666666666666666,
+          "recall": 0.16666666666666666,
+          "support": 6
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 0.875,
+        "macro avg": {
+          "f1-score": 0.8300000000000001,
+          "precision": 0.8166666666666667,
+          "recall": 0.85,
+          "support": 16
+        },
+        "weighted avg": {
+          "f1-score": 0.85,
+          "precision": 0.8333333333333333,
+          "recall": 0.875,
+          "support": 16
+        }
+      },
+      "test_run_time": "56s",
+      "total_run_time": "2m2s",
+      "train_run_time": "1m6s",
+      "type": "nlu"
+    }],
+    "Rules + Memo": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "conversation_accuracy": {
+          "accuracy": 0.8888888888888888,
+          "correct": 8,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 0.9663698541747322,
+          "precision": 1.0,
+          "recall": 0.946007696007696,
+          "support": 67
+        },
+        "micro avg": {
+          "f1-score": 0.9692307692307692,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 0.9656317714563074,
+          "precision": 1.0,
+          "recall": 0.9402985074626866,
+          "support": 67
+        }
+      },
+      "test_run_time": "10s",
+      "total_run_time": "19s",
+      "train_run_time": "10s",
+      "type": "core"
+    }],
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    }]
+  }
+}

.github/tests/test_data/report_listformat_core.json

@@ -0,0 +1,70 @@
+{
+  "RasaHQ/retail-demo": {
+    "Rules + Memo + TED": [{
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "31s",
+      "total_run_time": "4m57s",
+      "train_run_time": "4m27s",
+      "type": "core"
+    },
+    {
+      "accelerator_type": "GPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "8226b51b4312aa4d3723098cf6d4028feea040b4",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "external_dataset_repository": true,
+      "story_prediction": {
+        "accuracy": 1.0,
+        "conversation_accuracy": {
+          "accuracy": 1.0,
+          "correct": 9,
+          "total": 9,
+          "with_warnings": 0
+        },
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 67
+        }
+      },
+      "test_run_time": "41s",
+      "total_run_time": "5m57s",
+      "train_run_time": "5m27s",
+      "type": "core"
+    }]
+  }
+}

.github/tests/test_data/report_listformat_nlu.json

@@ -0,0 +1,98 @@
+{
+  "RasaHQ/financial-demo": {
+    "BERT + DIET(seq) + ResponseSelector(t2t)": [{
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "1m29s",
+      "total_run_time": "4m24s",
+      "train_run_time": "2m55s",
+      "type": "nlu"
+    },
+    {
+      "accelerator_type": "CPU",
+      "config_repository": "training-data",
+      "config_repository_branch": "main",
+      "dataset_commit": "52a3ad3eb5292d56542687e23b06703431f15ead",
+      "dataset_repository_branch": "fix-model-regression-tests",
+      "entity_prediction": {
+        "macro avg": {
+          "f1-score": 0.7333333333333333,
+          "precision": 0.8,
+          "recall": 0.7,
+          "support": 14
+        },
+        "micro avg": {
+          "f1-score": 0.8333333333333333,
+          "precision": 1.0,
+          "recall": 0.7142857142857143,
+          "support": 14
+        },
+        "weighted avg": {
+          "f1-score": 0.738095238095238,
+          "precision": 0.7857142857142857,
+          "recall": 0.7142857142857143,
+          "support": 14
+        }
+      },
+      "external_dataset_repository": true,
+      "intent_classification": {
+        "accuracy": 1.0,
+        "macro avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        },
+        "weighted avg": {
+          "f1-score": 1.0,
+          "precision": 1.0,
+          "recall": 1.0,
+          "support": 28
+        }
+      },
+      "test_run_time": "2m29s",
+      "total_run_time": "5m24s",
+      "train_run_time": "3m55s",
+      "type": "nlu"
+    }]
+  }
+}

.github/tests/test_download_pretrained.py

@@ -0,0 +1,101 @@
+from copy import deepcopy
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+
+sys.path.append(".github/scripts")
+import download_pretrained  # noqa: E402
+
+CONFIG_FPATH = Path(__file__).parent / "test_data" / "bert_diet_response2t.yml"
+
+
+def test_download_pretrained_lmf_exists_no_params():
+    lmf_specs = download_pretrained.get_model_name_and_weights_from_config(CONFIG_FPATH)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "rasa/LaBSE"
+
+
+def test_download_pretrained_lmf_exists_with_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "roberta"
+    step["cache_dir"] = "/this/dir"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "roberta"
+    assert lmf_specs[0].model_weights == "roberta-base"
+    assert lmf_specs[0].cache_dir == "/this/dir"
+
+
+def test_download_pretrained_unknown_model_name():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "unknown"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        with pytest.raises(KeyError):
+            download_pretrained.get_model_name_and_weights_from_config(fp.name)
+
+
+def test_download_pretrained_multiple_model_names():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step_new = deepcopy(step)
+    step_new["model_name"] = "roberta"
+    steps.append(step_new)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 2
+    assert lmf_specs[1].model_name == "roberta"
+
+
+def test_download_pretrained_with_model_name_and_nondefault_weight():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    step["model_name"] = "bert"
+    step["model_weights"] = "bert-base-uncased"
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert lmf_specs[0].model_name == "bert"
+    assert lmf_specs[0].model_weights == "bert-base-uncased"
+
+
+def test_download_pretrained_lmf_doesnt_exists():
+    yaml = YAML(typ="safe")
+    config = yaml.load(CONFIG_FPATH)
+
+    steps = config.get("pipeline", [])
+    step = list(filter(lambda x: x["name"] == download_pretrained.COMP_NAME, steps))[0]
+    steps.remove(step)
+
+    with tempfile.NamedTemporaryFile("w+") as fp:
+        yaml.dump(config, fp)
+        fp.seek(0)
+        lmf_specs = download_pretrained.get_model_name_and_weights_from_config(fp.name)
+    assert len(lmf_specs) == 0

.github/tests/test_model_regression_test_read_dataset_branch_tmpl.py

@@ -0,0 +1,27 @@
+import pathlib
+import subprocess
+import pytest
+from typing import Text
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_read_dataset_branch.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+DEFAULT_DATASET_BRANCH = "main"
+
+
+@pytest.mark.parametrize(
+    "comment_body_file,expected_dataset_branch",
+    [
+        ("comment_body.json", "test_dataset_branch"),
+        ("comment_body_no_dataset_branch.json", DEFAULT_DATASET_BRANCH),
+    ],
+)
+def test_read_dataset_branch(comment_body_file: Text, expected_dataset_branch: Text):
+    cmd = (
+        "gomplate "
+        f"-d github={TEST_DATA_DIR}/{comment_body_file} "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8").strip()
+    assert output == f'export DATASET_BRANCH="{expected_dataset_branch}"'

.github/tests/test_model_regression_test_results_tmpl.py

@@ -0,0 +1,50 @@
+import pathlib
+import subprocess
+
+TEMPLATE_FPATH = ".github/templates/model_regression_test_results.tmpl"
+REPO_DIR = pathlib.Path("").absolute()
+TEST_DATA_DIR = str(pathlib.Path(__file__).parent / "test_data")
+
+
+def test_comment_nlu():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_nlu.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/financial-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `52a3ad3eb5292d56542687e23b06703431f15ead`
+Configuration repository branch: `main`
+| Configuration | Intent Classification Micro F1 | Entity Recognition Micro F1 | Response Selection Micro F1 |
+|---------------|-----------------|-----------------|-------------------|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `1m29s`, train: `2m55s`, total: `4m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+| `BERT + DIET(seq) + ResponseSelector(t2t)`<br> test: `2m29s`, train: `3m55s`, total: `5m24s`|1.0000 (0.00)|0.8333 (0.00)|`no data`|
+
+
+"""  # noqa E501
+    assert output == expected_output
+
+
+def test_comment_core():
+    cmd = (
+        "gomplate "
+        f"-d data={TEST_DATA_DIR}/report_listformat_core.json "
+        f"-d results_main={TEST_DATA_DIR}/report-on-schedule-2022-02-02.json "
+        f"-f {TEMPLATE_FPATH}"
+    )
+    output = subprocess.check_output(cmd.split(" "), cwd=REPO_DIR)
+    output = output.decode("utf-8")
+    expected_output = """
+Dataset: `RasaHQ/retail-demo`, Dataset repository branch: `fix-model-regression-tests` (external repository), commit: `8226b51b4312aa4d3723098cf6d4028feea040b4`
+Configuration repository branch: `main`
+
+| Dialog Policy Configuration | Action Level Micro Avg. F1 | Conversation Level Accuracy | Run Time Train | Run Time Test |
+|---------------|-----------------|-----------------|-------------------|-------------------|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`4m27s`| `31s`|
+| `Rules + Memo + TED` |1.0000 (0.00)|1.0000 (0.00)|`5m27s`| `41s`|
+
+"""  # noqa E501
+    assert output == expected_output

.github/tests/test_mr_generate_summary.py

@@ -0,0 +1,208 @@
+import sys
+
+sys.path.append(".github/scripts")
+from mr_generate_summary import combine_result  # noqa: E402
+
+
+RESULT1 = {
+    "financial-demo": {
+        "BERT + DIET(bow) + ResponseSelector(bow)": [
+            {
+                "Entity Prediction": {
+                    "macro avg": {
+                        "f1-score": 0.7333333333333333,
+                    }
+                },
+                "test_run_time": "47s",
+            }
+        ]
+    }
+}
+
+
+def test_same_ds_different_config():
+    result2 = {
+        "financial-demo": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_different_ds_same_config():
+    result2 = {
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ],
+        },
+        "Carbon Bot": {
+            "Sparse + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.88,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        },
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_start_empty():
+    result2 = {}
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                }
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, result2)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(result2, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    actual_combined = combine_result(RESULT1, RESULT1)
+    assert actual_combined == expected_combined
+
+
+def test_combine_result_repetition_3times():
+    expected_combined = {
+        "financial-demo": {
+            "BERT + DIET(bow) + ResponseSelector(bow)": [
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+                {
+                    "Entity Prediction": {
+                        "macro avg": {
+                            "f1-score": 0.7333333333333333,
+                        }
+                    },
+                    "test_run_time": "47s",
+                },
+            ]
+        }
+    }
+
+    tmp_combined = combine_result(RESULT1, RESULT1)
+    actual_combined = combine_result(tmp_combined, RESULT1)
+    assert actual_combined == expected_combined
+
+    actual_combined = combine_result(RESULT1, tmp_combined)
+    assert actual_combined == expected_combined

.github/tests/test_mr_publish_results.py

@@ -0,0 +1,132 @@
+import os
+from pathlib import Path
+import sys
+from unittest import mock
+
+sys.path.append(".github/scripts")
+from mr_publish_results import (  # noqa: E402
+    prepare_ml_metric,
+    prepare_ml_metrics,
+    transform_to_seconds,
+    generate_json,
+    prepare_datadog_tags,
+)
+
+EXAMPLE_CONFIG = "Sparse + BERT + DIET(seq) + ResponseSelector(t2t)"
+EXAMPLE_DATASET_NAME = "financial-demo"
+
+ENV_VARS = {
+    "BRANCH": "my-branch",
+    "PR_ID": "10927",
+    "PR_URL": "https://github.com/RasaHQ/rasa/pull/10856/",
+    "GITHUB_EVENT_NAME": "pull_request",
+    "GITHUB_RUN_ID": "1882718340",
+    "GITHUB_SHA": "abc",
+    "GITHUB_WORKFLOW": "CI - Model Regression",
+    "IS_EXTERNAL": "false",
+    "DATASET_REPOSITORY_BRANCH": "main",
+    "CONFIG": EXAMPLE_CONFIG,
+    "DATASET_NAME": EXAMPLE_DATASET_NAME,
+    "CONFIG_REPOSITORY_BRANCH": "main",
+    "DATASET_COMMIT": "52a3ad3eb5292d56542687e23b06703431f15ead",
+    "ACCELERATOR_TYPE": "CPU",
+    "TEST_RUN_TIME": "1m54s",
+    "TRAIN_RUN_TIME": "4m4s",
+    "TOTAL_RUN_TIME": "5m58s",
+    "TYPE": "nlu",
+    "INDEX_REPETITION": "0",
+    "HOST_NAME": "github-runner-2223039222-22df222fcd-2cn7d",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_generate_json():
+    f = Path(__file__).parent / "test_data" / "intent_report.json"
+    result = generate_json(f, task="intent_classification", data={})
+    assert isinstance(result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG], list)
+
+    actual = result[EXAMPLE_DATASET_NAME][EXAMPLE_CONFIG][0]["intent_classification"]
+    expected = {
+        "accuracy": 1.0,
+        "weighted avg": {
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1-score": 1.0,
+            "support": 28,
+        },
+        "macro avg": {"precision": 1.0, "recall": 1.0, "f1-score": 1.0, "support": 28},
+    }
+    assert expected == actual
+
+
+def test_transform_to_seconds():
+    assert 87.0 == transform_to_seconds("1m27s")
+    assert 87.3 == transform_to_seconds("1m27.3s")
+    assert 27.0 == transform_to_seconds("27s")
+    assert 3627.0 == transform_to_seconds("1h27s")
+    assert 3687.0 == transform_to_seconds("1h1m27s")
+
+
+def test_prepare_ml_model_perf_metrics():
+    results = [
+        {
+            "macro avg": {
+                "precision": 0.8,
+                "recall": 0.8,
+                "f1-score": 0.8,
+                "support": 14,
+            },
+            "micro avg": {
+                "precision": 1.0,
+                "recall": 0.7857142857142857,
+                "f1-score": 0.88,
+                "support": 14,
+            },
+            "file_name": "DIETClassifier_report.json",
+            "task": "Entity Prediction",
+        },
+        {
+            "accuracy": 1.0,
+            "weighted avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "macro avg": {
+                "precision": 1.0,
+                "recall": 1.0,
+                "f1-score": 1.0,
+                "support": 28,
+            },
+            "file_name": "intent_report.json",
+            "task": "Intent Classification",
+        },
+    ]
+    metrics_ml = prepare_ml_metrics(results)
+    assert len(metrics_ml) == 17
+
+
+def test_prepare_ml_model_perf_metrics_simple():
+    result = {
+        "accuracy": 1.0,
+        "weighted avg": {"precision": 1, "recall": 1.0, "f1-score": 1, "support": 28},
+        "task": "Intent Classification",
+    }
+    metrics_ml = prepare_ml_metric(result)
+    assert len(metrics_ml) == 5
+
+    for _, v in metrics_ml.items():
+        assert isinstance(v, float)
+
+    key, value = "Intent Classification.accuracy", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+    key, value = "Intent Classification.weighted avg.f1-score", 1.0
+    assert key in metrics_ml and value == metrics_ml[key]
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_prepare_datadog_tags():
+    tags_list = prepare_datadog_tags()
+    assert "dataset:financial-demo" in tags_list

.github/tests/test_validate_gpus.py

@@ -0,0 +1,27 @@
+import os
+import sys
+from unittest import mock
+
+import pytest
+
+sys.path.append(".github/scripts")
+import validate_cpu  # noqa: E402
+import validate_gpus  # noqa: E402
+
+ENV_VARS = {
+    "CUDA_VISIBLE_DEVICES": "-1",
+}
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_cpu_succeeds_when_there_are_no_gpus():
+    validate_cpu.check_gpu_not_available()
+
+
+@mock.patch.dict(os.environ, ENV_VARS, clear=True)
+def test_validate_gpus_exits_when_there_are_no_gpus():
+    # This unit test assumes that unit tests are run on a CPU
+    with pytest.raises(SystemExit) as pytest_wrapped_e:
+        validate_gpus.check_gpu_available()
+    assert pytest_wrapped_e.type == SystemExit
+    assert pytest_wrapped_e.value.code == 1