chore: import upstream snapshot with attribution

data/test_classes/custom_graph_components/nlu_dense.py

@@ -0,0 +1,148 @@
+import numpy as np
+import logging
+from bpemb import BPEmb
+from typing import Any, Text, Dict, List, Type
+
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
+from rasa.nlu.tokenizers.tokenizer import Tokenizer
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.features import Features
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.constants import (
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+    FEATURIZER_CLASS_ALIAS,
+)
+from rasa.shared.nlu.constants import (
+    TEXT,
+    TEXT_TOKENS,
+    FEATURE_TYPE_SENTENCE,
+    FEATURE_TYPE_SEQUENCE,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=False
+)
+class BytePairFeaturizer(DenseFeaturizer, GraphComponent):
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [Tokenizer]
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["bpemb"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            **DenseFeaturizer.get_default_config(),
+            # specifies the language of the subword segmentation model
+            "lang": None,
+            # specifies the dimension of the subword embeddings
+            "dim": None,
+            # specifies the vocabulary size of the segmentation model
+            "vs": None,
+            # if set to True and the given vocabulary size can't be loaded for the given
+            # model, the closest size is chosen
+            "vs_fallback": True,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        name: Text,
+    ) -> None:
+        """Constructs a new byte pair vectorizer."""
+        super().__init__(name, config)
+        # The configuration dictionary is saved in `self._config` for reference.
+        self.model = BPEmb(
+            lang=self._config["lang"],
+            dim=self._config["dim"],
+            vs=self._config["vs"],
+            vs_fallback=self._config["vs_fallback"],
+        )
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new component (see parent class for full docstring)."""
+        return cls(config, execution_context.node_name)
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Processes incoming messages and computes and sets features."""
+        for message in messages:
+            for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
+                self._set_features(message, attribute)
+        return messages
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Processes the training examples in the given training data in-place."""
+        self.process(training_data.training_examples)
+        return training_data
+
+    def _create_word_vector(self, document: Text) -> np.ndarray:
+        """Creates a word vector from a text. Utility method."""
+        encoded_ids = self.model.encode_ids(document)
+        if encoded_ids:
+            return self.model.vectors[encoded_ids[0]]
+
+        return np.zeros((self.component_config["dim"],), dtype=np.float32)
+
+    def _set_features(self, message: Message, attribute: Text = TEXT) -> None:
+        """Sets the features on a single message. Utility method."""
+        tokens = message.get(TEXT_TOKENS)
+
+        # If the message doesn't have tokens, we can't create features.
+        if not tokens:
+            return None
+
+        # We need to reshape here such that the shape is equivalent to that of sparsely
+        # generated features. Without it, it'd be a 1D tensor. We need 2D (n_utterance, n_dim).
+        text_vector = self._create_word_vector(document=message.get(TEXT)).reshape(
+            1, -1
+        )
+        word_vectors = np.array(
+            [self._create_word_vector(document=t.text) for t in tokens]
+        )
+
+        final_sequence_features = Features(
+            word_vectors,
+            FEATURE_TYPE_SEQUENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sequence_features)
+        final_sentence_features = Features(
+            text_vector,
+            FEATURE_TYPE_SENTENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sentence_features)
+
+    @classmethod
+    def validate_config(cls, config: Dict[Text, Any]) -> None:
+        """Validates that the component is configured properly."""
+        if not config["lang"]:
+            raise ValueError("BytePairFeaturizer needs language setting via `lang`.")
+        if not config["dim"]:
+            raise ValueError(
+                "BytePairFeaturizer needs dimensionality setting via `dim`."
+            )
+        if not config["vs"]:
+            raise ValueError("BytePairFeaturizer needs a vector size setting via `vs`.")

data/test_classes/custom_graph_components/nlu_meta_fallback.py

@@ -0,0 +1,42 @@
+from typing import Dict, Text, Any, List
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.nlu.classifiers.fallback_classifier import FallbackClassifier
+
+
+@DefaultV1Recipe.register(
+    [DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER], is_trainable=True
+)
+class MetaFallback(FallbackClassifier):
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> None:
+        super().__init__(config)
+
+        self._model_storage = model_storage
+        self._resource = resource
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> FallbackClassifier:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource, execution_context)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        # Do something here with the messages
+        return self._resource

data/test_classes/custom_graph_components/nlu_meta_intent_featurizer.py

@@ -0,0 +1,15 @@
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.nlu.classifiers.diet_classifier import DIETClassifier
+
+
+@DefaultV1Recipe.register(
+    [DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER,
+     DefaultV1Recipe.ComponentType.ENTITY_EXTRACTOR,
+     DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER], is_trainable=True
+)
+class DIETFeaturizer(DIETClassifier):
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        # classify and add the attributes to the messages on the training data
+        return training_data

data/test_classes/custom_graph_components/nlu_sparse.py

@@ -0,0 +1,164 @@
+import logging
+from typing import Any, Text, Dict, List, Type
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
+from rasa.nlu.tokenizers.tokenizer import Tokenizer
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.features import Features
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.constants import (
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+    FEATURIZER_CLASS_ALIAS,
+)
+from joblib import dump, load
+from rasa.shared.nlu.constants import (
+    TEXT,
+    TEXT_TOKENS,
+    FEATURE_TYPE_SENTENCE,
+    FEATURE_TYPE_SEQUENCE,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True
+)
+class TfIdfFeaturizer(SparseFeaturizer, GraphComponent):
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [Tokenizer]
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["sklearn"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            **SparseFeaturizer.get_default_config(),
+            "analyzer": "word",
+            "min_ngram": 1,
+            "max_ngram": 1,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        name: Text,
+        model_storage: ModelStorage,
+        resource: Resource,
+    ) -> None:
+        """Constructs a new tf/idf vectorizer using the sklearn framework."""
+        super().__init__(name, config)
+        # Initialize the tfidf sklearn component
+        self.tfm = TfidfVectorizer(
+            analyzer=config["analyzer"],
+            ngram_range=(config["min_ngram"], config["max_ngram"]),
+        )
+
+        # We need to use these later when saving the trained component.
+        self._model_storage = model_storage
+        self._resource = resource
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Trains the component from training data."""
+        texts = [e.get(TEXT) for e in training_data.training_examples if e.get(TEXT)]
+        self.tfm.fit(texts)
+        self.persist()
+        return self._resource
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, execution_context.node_name, model_storage, resource)
+
+    def _set_features(self, message: Message, attribute: Text = TEXT) -> None:
+        """Sets the features on a single message. Utility method."""
+        tokens = message.get(TEXT_TOKENS)
+
+        # If the message doesn't have tokens, we can't create features.
+        if not tokens:
+            return None
+
+        # Make distinction between sentence and sequence features
+        text_vector = self.tfm.transform([message.get(TEXT)])
+        word_vectors = self.tfm.transform([t.text for t in tokens])
+
+        final_sequence_features = Features(
+            word_vectors,
+            FEATURE_TYPE_SEQUENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sequence_features)
+        final_sentence_features = Features(
+            text_vector,
+            FEATURE_TYPE_SENTENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sentence_features)
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Processes incoming message and compute and set features."""
+        for message in messages:
+            for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
+                self._set_features(message, attribute)
+        return messages
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Processes the training examples in the given training data in-place."""
+        self.process(training_data.training_examples)
+        return training_data
+
+    def persist(self) -> None:
+        """
+        Persist this model into the passed directory.
+
+        Returns the metadata necessary to load the model again. In this case; `None`.
+        """
+        with self._model_storage.write_to(self._resource) as model_dir:
+            dump(self.tfm, model_dir / "tfidfvectorizer.joblib")
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Loads trained component from disk."""
+        try:
+            with model_storage.read_from(resource) as model_dir:
+                tfidfvectorizer = load(model_dir / "tfidfvectorizer.joblib")
+                component = cls(
+                    config, execution_context.node_name, model_storage, resource
+                )
+                component.tfm = tfidfvectorizer
+        except (ValueError, FileNotFoundError):
+            logger.debug(
+                f"Couldn't load metadata for component '{cls.__name__}' as the persisted "
+                f"model data couldn't be loaded."
+            )
+        return component
+
+    @classmethod
+    def validate_config(cls, config: Dict[Text, Any]) -> None:
+        """Validates that the component is configured properly."""
+        pass

data/test_classes/custom_slots.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Any, Dict, List, Text, Optional
+
+from rasa.shared.core.slots import Slot
+
+logger = logging.getLogger(__name__)
+
+
+class LimitSlot(Slot):
+    """
+    A slot for featurizing an amount as greater than or equal to vs. less than a given value.
+
+    Example of configuration in the domain.yml file:
+    slots:
+        my_slot:
+          type: custom.slots.LimitSlot
+          limit: 100
+    """
+
+    type_name = "limit"
+
+    def __init__(
+        self,
+        name: Text,
+        limit: int,
+        mappings: List[Dict[Text, Any]],
+        initial_value: Any = None,
+        value_reset_delay: Optional[int] = None,
+        influence_conversation: bool = True,
+    ) -> None:
+
+        super().__init__(
+            name=name,
+            initial_value=initial_value,
+            mappings=mappings,
+            value_reset_delay=value_reset_delay,
+            influence_conversation=influence_conversation,
+        )
+        self.limit = limit
+
+    def _as_feature(self) -> List[float]:
+        try:
+            greater_than_limit = float(self.value >= self.limit)
+            return [1.0, greater_than_limit]
+        except (TypeError, ValueError):
+            return [0.0, 0.0]
+
+    def persistence_info(self) -> Dict[Text, Any]:
+        """Returns relevant information to persist this slot."""
+        d = super().persistence_info()
+        d["limit"] = self.limit
+        return d
+
+    def _feature_dimensionality(self) -> int:
+        return len(self.as_feature())

data/test_classes/graph_component_interface.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import List, Type, Dict, Text, Any, Optional
+
+from rasa.engine.graph import ExecutionContext
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+
+
+class GraphComponent(ABC):
+    """Interface for any component which will run in a graph."""
+
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return []
+
+    @classmethod
+    @abstractmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new `GraphComponent`.
+
+        Args:
+            config: This config overrides the `default_config`.
+            model_storage: Storage which graph components can use to persist and load
+                themselves.
+            resource: Resource locator for this component which can be used to persist
+                and load itself from the `model_storage`.
+            execution_context: Information about the current graph run.
+
+        Returns: An instantiated `GraphComponent`.
+        """
+        ...
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+        **kwargs: Any,
+    ) -> GraphComponent:
+        """Creates a component using a persisted version of itself.
+
+        If not overridden this method merely calls `create`.
+
+        Args:
+            config: The config for this graph component. This is the default config of
+                the component merged with config specified by the user.
+            model_storage: Storage which graph components can use to persist and load
+                themselves.
+            resource: Resource locator for this component which can be used to persist
+                and load itself from the `model_storage`.
+            execution_context: Information about the current graph run.
+            kwargs: Output values from previous nodes might be passed in as `kwargs`.
+
+        Returns:
+            An instantiated, loaded `GraphComponent`.
+        """
+        return cls.create(config, model_storage, resource, execution_context)
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config.
+
+        Default config and user config are merged by the `GraphNode` before the
+        config is passed to the `create` and `load` method of the component.
+
+        Returns:
+            The default config of the component.
+        """
+        return {}
+
+    @staticmethod
+    def supported_languages() -> Optional[List[Text]]:
+        """Determines which languages this component can work with.
+
+        Returns: A list of supported languages, or `None` to signify all are supported.
+        """
+        return None
+
+    @staticmethod
+    def not_supported_languages() -> Optional[List[Text]]:
+        """Determines which languages this component cannot work with.
+
+        Returns: A list of not supported languages, or
+            `None` to signify all are supported.
+        """
+        return None
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return []
+
+    @classmethod
+    def fingerprint_addon(cls, config: Dict[str, Any]) -> Optional[str]:
+        """Adds additional data to the fingerprint calculation.
+
+        This is useful if a component uses external data that is not provided
+        by the graph.
+        """
+        return None

data/test_classes/nlu_component_skeleton.py

@@ -0,0 +1,41 @@
+from typing import Dict, Text, Any, List
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
+
+# TODO: Correctly register your component with its type
+@DefaultV1Recipe.register(
+    [DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER], is_trainable=True
+)
+class CustomNLUComponent(GraphComponent):
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        # TODO: Implement this
+        ...
+
+    def train(self, training_data: TrainingData) -> Resource:
+        # TODO: Implement this if your component requires training
+        ...
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        # TODO: Implement this if your component augments the training data with
+        #       tokens or message features which are used by other components
+        #       during training.
+        ...
+
+        return training_data
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        # TODO: This is the method which Rasa Open Source will call during inference.
+        ...
+        return messages

data/test_classes/registered_component.py

@@ -0,0 +1,11 @@
+from rasa.engine.graph import GraphComponent
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+
+
+@DefaultV1Recipe.register(
+    component_types=[DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER],
+    is_trainable=True,
+    model_from="SpacyNLP",
+)
+class MyComponent(GraphComponent):
+    ...