rasahq--rasa/tests/shared/utils/test_validation.py

from typing import Text
from threading import Thread

import pytest

from pep440_version_utils import Version

from rasa.shared.exceptions import YamlException, SchemaValidationError
import rasa.shared.utils.io
import rasa.shared.utils.validation as validation_utils
import rasa.utils.io as io_utils
import rasa.shared.nlu.training_data.schemas.data_schema as schema
from rasa.shared.constants import (
    CONFIG_SCHEMA_FILE,
    DOMAIN_SCHEMA_FILE,
    LATEST_TRAINING_DATA_FORMAT_VERSION,
)
from rasa.shared.nlu.training_data.formats.rasa_yaml import NLU_SCHEMA_FILE
from rasa.shared.utils.validation import KEY_TRAINING_DATA_FORMAT_VERSION


@pytest.mark.parametrize(
    "file, schema",
    [
        ("data/test_moodbot/domain.yml", DOMAIN_SCHEMA_FILE),
        ("data/test_config/config_defaults.yml", CONFIG_SCHEMA_FILE),
        ("data/test_config/config_supervised_embeddings.yml", CONFIG_SCHEMA_FILE),
        ("data/test_config/config_crf_custom_features.yml", CONFIG_SCHEMA_FILE),
    ],
)
def test_validate_yaml_schema(file, schema):
    # should raise no exception
    validation_utils.validate_yaml_schema(rasa.shared.utils.io.read_file(file), schema)


def test_validate_yaml_schema_with_package_name():
    # should raise no exception
    file = "data/test_moodbot/domain.yml"
    schema = DOMAIN_SCHEMA_FILE
    validation_utils.validate_yaml_schema(
        rasa.shared.utils.io.read_file(file), schema, package_name="rasa"
    )


def test_validate_yaml_schema_with_random_package_name_fails():
    # should raise no exception
    file = "data/test_moodbot/domain.yml"
    schema = DOMAIN_SCHEMA_FILE

    with pytest.raises(ModuleNotFoundError):
        validation_utils.validate_yaml_schema(
            rasa.shared.utils.io.read_file(file), schema, package_name="rasa_foo_bar_42"
        )


@pytest.mark.parametrize(
    "file, schema",
    [
        ("data/test_domains/valid_actions.yml", DOMAIN_SCHEMA_FILE),
    ],
)
def test_validate_yaml_schema_actions(file: Text, schema: Text):
    # should raise no exception
    validation_utils.validate_yaml_schema(rasa.shared.utils.io.read_file(file), schema)


@pytest.mark.parametrize(
    "content, schema",
    [
        (
            """
        intents:
            - greet

        entities:
            - name

        responses:
            utter_greet:
                - text: hey there!

        actions:
          - utter_default: {send_domain: 1}
        """,
            DOMAIN_SCHEMA_FILE,
        ),
        (
            """
        intents:
            - greet

        entities:
            - name

        responses:
            utter_greet:
                - text: hey there!

        actions:
          - utter_default: {send_domain: 0}
        """,
            DOMAIN_SCHEMA_FILE,
        ),
        (
            """
        intents:
            - greet

        entities:
            - name

        responses:
            utter_greet:
                - text: hey there!

        actions:
          - utter_default: {send_domain: Ttrue}
        """,
            DOMAIN_SCHEMA_FILE,
        ),
        (
            """
        intents:
            - greet

        entities:
            - name

        responses:
            utter_greet:
                - text: hey there!

        actions:
          - utter_default: {send_domain: ""}
        """,
            DOMAIN_SCHEMA_FILE,
        ),
    ],
)
def test_invalid_send_domain_value_in_actions(content: Text, schema: Text):
    with pytest.raises(validation_utils.YamlValidationException):
        validation_utils.validate_yaml_schema(content, schema)


@pytest.mark.parametrize(
    "file, schema",
    [
        ("data/test_domains/invalid_format.yml", DOMAIN_SCHEMA_FILE),
        ("data/test_domains/wrong_response_format.yml", DOMAIN_SCHEMA_FILE),
        ("data/test_domains/wrong_custom_response_format.yml", DOMAIN_SCHEMA_FILE),
        ("data/test_domains/empty_response_format.yml", DOMAIN_SCHEMA_FILE),
    ],
)
def test_validate_yaml_schema_raise_exception(file: Text, schema: Text):
    with pytest.raises(YamlException):
        validation_utils.validate_yaml_schema(
            rasa.shared.utils.io.read_file(file), schema
        )


def test_validate_yaml_schema_raise_exception_null_text():
    domain = f"""
    version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"
    responses:
      utter_ask_email:
      - text: What is your email ID?
      utter_ask_name:
      - text: null
    """
    with pytest.raises(validation_utils.YamlValidationException) as e:
        validation_utils.validate_yaml_schema(domain, DOMAIN_SCHEMA_FILE)

    assert (
        "Missing 'text' or 'custom' key in response or null 'text' value in response."
        in str(e.value)
    )


def test_validate_yaml_schema_raise_exception_extra_hyphen_for_image():
    domain = f"""
        version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"
        responses:
          utter_cheer_up:
          - image: https://i.imgur.com/nGF1K8f.jpg
          - text: Here is something to cheer you up
        """
    with pytest.raises(validation_utils.YamlValidationException) as e:
        validation_utils.validate_yaml_schema(domain, DOMAIN_SCHEMA_FILE)

    assert (
        "Missing 'text' or 'custom' key in response or null 'text' value in response."
        in str(e.value)
    )


def test_example_training_data_is_valid():
    demo_json = "data/examples/rasa/demo-rasa.json"
    data = rasa.shared.utils.io.read_json_file(demo_json)
    validation_utils.validate_training_data(data, schema.rasa_nlu_data_schema())


@pytest.mark.parametrize(
    "invalid_data",
    [
        {"wrong_top_level": []},
        ["this is not a toplevel dict"],
        {
            "rasa_nlu_data": {
                "common_examples": [
                    {
                        "text": "mytext",
                        "entities": [{"start": "INVALID", "end": 0, "entity": "x"}],
                    }
                ]
            }
        },
    ],
)
def test_validate_training_data_is_throwing_exceptions(invalid_data):
    with pytest.raises(SchemaValidationError):
        validation_utils.validate_training_data(
            invalid_data, schema.rasa_nlu_data_schema()
        )


def test_url_data_format():
    data = """
    {
      "rasa_nlu_data": {
        "entity_synonyms": [
          {
            "value": "nyc",
            "synonyms": ["New York City", "nyc", "the big apple"]
          }
        ],
        "common_examples" : [
          {
            "text": "show me flights to New York City",
            "intent": "unk",
            "entities": [
              {
                "entity": "destination",
                "start": 19,
                "end": 32,
                "value": "NYC"
              }
            ]
          }
        ]
      }
    }"""
    fname = io_utils.create_temporary_file(
        data.encode(rasa.shared.utils.io.DEFAULT_ENCODING),
        suffix="_tmp_training_data.json",
        mode="w+b",
    )
    data = rasa.shared.utils.io.read_json_file(fname)
    assert data is not None
    validation_utils.validate_training_data(data, schema.rasa_nlu_data_schema())


@pytest.mark.parametrize(
    "invalid_data",
    [
        {"group": "a", "role": "c", "value": "text"},
        ["this is not a toplevel dict"],
        {"entity": 1, "role": "c", "value": "text"},
        {"entity": "e", "role": None, "value": "text"},
    ],
)
def test_validate_entity_dict_is_throwing_exceptions(invalid_data):
    with pytest.raises(SchemaValidationError):
        validation_utils.validate_training_data(
            invalid_data, schema.entity_dict_schema()
        )


@pytest.mark.parametrize(
    "data",
    [
        {"entity": "e", "group": "a", "role": "c", "value": "text"},
        {"entity": "e"},
        {"entity": "e", "value": "text"},
        {"entity": "e", "group": "a"},
        {"entity": "e", "role": "c"},
        {"entity": "e", "role": "c", "value": "text"},
        {"entity": "e", "group": "a", "value": "text"},
        {"entity": "e", "group": "a", "role": "c"},
        {"entity": "e", "value": 3},
        {"entity": "e", "value": "3"},
    ],
)
def test_entity_dict_is_valid(data):
    validation_utils.validate_training_data(data, schema.entity_dict_schema())


async def test_future_training_data_format_version_not_compatible():

    next_minor = str(Version(LATEST_TRAINING_DATA_FORMAT_VERSION).next_minor())

    incompatible_version = {KEY_TRAINING_DATA_FORMAT_VERSION: next_minor}

    with pytest.warns(UserWarning):
        assert not validation_utils.validate_training_data_format_version(
            incompatible_version, ""
        )


async def test_compatible_training_data_format_version():

    prev_major = str(Version("1.0"))

    compatible_version_1 = {KEY_TRAINING_DATA_FORMAT_VERSION: prev_major}
    compatible_version_2 = {
        KEY_TRAINING_DATA_FORMAT_VERSION: LATEST_TRAINING_DATA_FORMAT_VERSION
    }

    for version in [compatible_version_1, compatible_version_2]:
        with pytest.warns(None):
            assert validation_utils.validate_training_data_format_version(version, "")


async def test_invalid_training_data_format_version_warns():

    invalid_version_1 = {KEY_TRAINING_DATA_FORMAT_VERSION: 2.0}
    invalid_version_2 = {KEY_TRAINING_DATA_FORMAT_VERSION: "Rasa"}

    for version in [invalid_version_1, invalid_version_2]:
        with pytest.warns(UserWarning):
            assert validation_utils.validate_training_data_format_version(version, "")


def test_concurrent_schema_validation():
    successful_results = []

    def validate() -> None:
        payload = f"""
version: "{LATEST_TRAINING_DATA_FORMAT_VERSION}"
nlu:
- intent: greet
  examples: |
    - hey
    - hello
    - hi
    - hello there
    - good morning
    - good evening
    - moin
    - hey there
    - let's go
    - hey dude
    - goodmorning
    - goodevening
    - good afternoon
- intent: goodbye
  examples: |
    - good afternoon
    - cu
    - good by
    - cee you later
    - good night
    - bye
    - goodbye
    - have a nice day
    - see you around
    - bye bye
    - see you later
        """
        rasa.shared.utils.validation.validate_yaml_schema(payload, NLU_SCHEMA_FILE)
        successful_results.append(True)

    threads = []
    for i in range(10):
        threads.append(Thread(target=validate))

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()

    assert len(successful_results) == len(threads)