rasahq--rasa/rasa/shared/nlu/training_data/formats/luis.py

import logging
from typing import Any, Dict, List, Text

from rasa.shared.nlu.constants import (
    TEXT,
    INTENT,
    ENTITIES,
    ENTITY_ATTRIBUTE_TYPE,
    ENTITY_ATTRIBUTE_ROLE,
    ENTITY_ATTRIBUTE_VALUE,
    ENTITY_ATTRIBUTE_START,
    ENTITY_ATTRIBUTE_END,
    NO_ENTITY_TAG,
)
from rasa.shared.nlu.training_data.formats.readerwriter import JsonTrainingDataReader
import rasa.shared.utils.io

from rasa.shared.nlu.training_data.training_data import TrainingData
from rasa.shared.nlu.training_data.message import Message

logger = logging.getLogger(__name__)


class LuisReader(JsonTrainingDataReader):
    """Reads LUIS training data."""

    @staticmethod
    def _extract_regex_features(js: Dict[Text, Any]) -> List[Dict[Text, Any]]:
        regex_features: List[Dict[Text, Any]] = []

        for r in js.get("regex_features", []):
            if r.get("activated", False):
                regex_features.append(
                    {"name": r.get("name"), "pattern": r.get("pattern")}
                )

        # LUIS removed `regex_features` and exports regular expressions in `
        # regex_entities` now:
        # https://stackoverflow.com/questions/48170631/what-happened-to-the-regex-features # noqa: E501
        for r in js.get("regex_entities", []):
            regex_features.append(
                {"name": r.get("name"), "pattern": r.get("regexPattern")}
            )

        return regex_features

    def read_from_json(self, js: Dict[Text, Any], **kwargs: Any) -> "TrainingData":
        """Loads training data stored in the LUIS.ai data format."""
        training_examples = []

        max_tested_luis_schema_version = 7
        major_version = int(js["luis_schema_version"].split(".")[0])
        if major_version > max_tested_luis_schema_version:
            rasa.shared.utils.io.raise_warning(
                f"Your luis data schema version {js['luis_schema_version']} "
                f"is higher than 7.x.x. "
                f"Training may not be performed correctly. "
            )

        for s in js["utterances"]:
            text = s.get("text")
            intent = s.get("intent")
            entities = []
            for e in s.get("entities") or []:
                start, end = e["startPos"], e["endPos"] + 1
                val = text[start:end]

                entities.append(
                    {
                        ENTITY_ATTRIBUTE_TYPE: e.get("entity"),
                        ENTITY_ATTRIBUTE_VALUE: val,
                        ENTITY_ATTRIBUTE_START: start,
                        ENTITY_ATTRIBUTE_END: end,
                        ENTITY_ATTRIBUTE_ROLE: e.get("role", NO_ENTITY_TAG),
                    }
                )

            data = {ENTITIES: entities}
            if intent:
                data[INTENT] = intent
            data[TEXT] = text

            training_examples.append(Message(data=data))

        return TrainingData(
            training_examples, regex_features=self._extract_regex_features(js)
        )