chore: import upstream snapshot with attribution

data/test_classes/custom_graph_components/nlu_dense.py

@@ -0,0 +1,148 @@
+import numpy as np
+import logging
+from bpemb import BPEmb
+from typing import Any, Text, Dict, List, Type
+
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
+from rasa.nlu.tokenizers.tokenizer import Tokenizer
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.features import Features
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.constants import (
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+    FEATURIZER_CLASS_ALIAS,
+)
+from rasa.shared.nlu.constants import (
+    TEXT,
+    TEXT_TOKENS,
+    FEATURE_TYPE_SENTENCE,
+    FEATURE_TYPE_SEQUENCE,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=False
+)
+class BytePairFeaturizer(DenseFeaturizer, GraphComponent):
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [Tokenizer]
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["bpemb"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            **DenseFeaturizer.get_default_config(),
+            # specifies the language of the subword segmentation model
+            "lang": None,
+            # specifies the dimension of the subword embeddings
+            "dim": None,
+            # specifies the vocabulary size of the segmentation model
+            "vs": None,
+            # if set to True and the given vocabulary size can't be loaded for the given
+            # model, the closest size is chosen
+            "vs_fallback": True,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        name: Text,
+    ) -> None:
+        """Constructs a new byte pair vectorizer."""
+        super().__init__(name, config)
+        # The configuration dictionary is saved in `self._config` for reference.
+        self.model = BPEmb(
+            lang=self._config["lang"],
+            dim=self._config["dim"],
+            vs=self._config["vs"],
+            vs_fallback=self._config["vs_fallback"],
+        )
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new component (see parent class for full docstring)."""
+        return cls(config, execution_context.node_name)
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Processes incoming messages and computes and sets features."""
+        for message in messages:
+            for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
+                self._set_features(message, attribute)
+        return messages
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Processes the training examples in the given training data in-place."""
+        self.process(training_data.training_examples)
+        return training_data
+
+    def _create_word_vector(self, document: Text) -> np.ndarray:
+        """Creates a word vector from a text. Utility method."""
+        encoded_ids = self.model.encode_ids(document)
+        if encoded_ids:
+            return self.model.vectors[encoded_ids[0]]
+
+        return np.zeros((self.component_config["dim"],), dtype=np.float32)
+
+    def _set_features(self, message: Message, attribute: Text = TEXT) -> None:
+        """Sets the features on a single message. Utility method."""
+        tokens = message.get(TEXT_TOKENS)
+
+        # If the message doesn't have tokens, we can't create features.
+        if not tokens:
+            return None
+
+        # We need to reshape here such that the shape is equivalent to that of sparsely
+        # generated features. Without it, it'd be a 1D tensor. We need 2D (n_utterance, n_dim).
+        text_vector = self._create_word_vector(document=message.get(TEXT)).reshape(
+            1, -1
+        )
+        word_vectors = np.array(
+            [self._create_word_vector(document=t.text) for t in tokens]
+        )
+
+        final_sequence_features = Features(
+            word_vectors,
+            FEATURE_TYPE_SEQUENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sequence_features)
+        final_sentence_features = Features(
+            text_vector,
+            FEATURE_TYPE_SENTENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sentence_features)
+
+    @classmethod
+    def validate_config(cls, config: Dict[Text, Any]) -> None:
+        """Validates that the component is configured properly."""
+        if not config["lang"]:
+            raise ValueError("BytePairFeaturizer needs language setting via `lang`.")
+        if not config["dim"]:
+            raise ValueError(
+                "BytePairFeaturizer needs dimensionality setting via `dim`."
+            )
+        if not config["vs"]:
+            raise ValueError("BytePairFeaturizer needs a vector size setting via `vs`.")

data/test_classes/custom_graph_components/nlu_meta_fallback.py

@@ -0,0 +1,42 @@
+from typing import Dict, Text, Any, List
+
+from rasa.engine.graph import GraphComponent, ExecutionContext
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.shared.nlu.training_data.message import Message
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.nlu.classifiers.fallback_classifier import FallbackClassifier
+
+
+@DefaultV1Recipe.register(
+    [DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER], is_trainable=True
+)
+class MetaFallback(FallbackClassifier):
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> None:
+        super().__init__(config)
+
+        self._model_storage = model_storage
+        self._resource = resource
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> FallbackClassifier:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, model_storage, resource, execution_context)
+
+    def train(self, training_data: TrainingData) -> Resource:
+        # Do something here with the messages
+        return self._resource

data/test_classes/custom_graph_components/nlu_meta_intent_featurizer.py

@@ -0,0 +1,15 @@
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.nlu.classifiers.diet_classifier import DIETClassifier
+
+
+@DefaultV1Recipe.register(
+    [DefaultV1Recipe.ComponentType.INTENT_CLASSIFIER,
+     DefaultV1Recipe.ComponentType.ENTITY_EXTRACTOR,
+     DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER], is_trainable=True
+)
+class DIETFeaturizer(DIETClassifier):
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        # classify and add the attributes to the messages on the training data
+        return training_data

data/test_classes/custom_graph_components/nlu_sparse.py

@@ -0,0 +1,164 @@
+import logging
+from typing import Any, Text, Dict, List, Type
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from rasa.engine.recipes.default_recipe import DefaultV1Recipe
+from rasa.engine.graph import ExecutionContext, GraphComponent
+from rasa.engine.storage.resource import Resource
+from rasa.engine.storage.storage import ModelStorage
+from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer
+from rasa.nlu.tokenizers.tokenizer import Tokenizer
+from rasa.shared.nlu.training_data.training_data import TrainingData
+from rasa.shared.nlu.training_data.features import Features
+from rasa.shared.nlu.training_data.message import Message
+from rasa.nlu.constants import (
+    DENSE_FEATURIZABLE_ATTRIBUTES,
+    FEATURIZER_CLASS_ALIAS,
+)
+from joblib import dump, load
+from rasa.shared.nlu.constants import (
+    TEXT,
+    TEXT_TOKENS,
+    FEATURE_TYPE_SENTENCE,
+    FEATURE_TYPE_SEQUENCE,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@DefaultV1Recipe.register(
+    DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True
+)
+class TfIdfFeaturizer(SparseFeaturizer, GraphComponent):
+    @classmethod
+    def required_components(cls) -> List[Type]:
+        """Components that should be included in the pipeline before this component."""
+        return [Tokenizer]
+
+    @staticmethod
+    def required_packages() -> List[Text]:
+        """Any extra python dependencies required for this component to run."""
+        return ["sklearn"]
+
+    @staticmethod
+    def get_default_config() -> Dict[Text, Any]:
+        """Returns the component's default config."""
+        return {
+            **SparseFeaturizer.get_default_config(),
+            "analyzer": "word",
+            "min_ngram": 1,
+            "max_ngram": 1,
+        }
+
+    def __init__(
+        self,
+        config: Dict[Text, Any],
+        name: Text,
+        model_storage: ModelStorage,
+        resource: Resource,
+    ) -> None:
+        """Constructs a new tf/idf vectorizer using the sklearn framework."""
+        super().__init__(name, config)
+        # Initialize the tfidf sklearn component
+        self.tfm = TfidfVectorizer(
+            analyzer=config["analyzer"],
+            ngram_range=(config["min_ngram"], config["max_ngram"]),
+        )
+
+        # We need to use these later when saving the trained component.
+        self._model_storage = model_storage
+        self._resource = resource
+
+    def train(self, training_data: TrainingData) -> Resource:
+        """Trains the component from training data."""
+        texts = [e.get(TEXT) for e in training_data.training_examples if e.get(TEXT)]
+        self.tfm.fit(texts)
+        self.persist()
+        return self._resource
+
+    @classmethod
+    def create(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Creates a new untrained component (see parent class for full docstring)."""
+        return cls(config, execution_context.node_name, model_storage, resource)
+
+    def _set_features(self, message: Message, attribute: Text = TEXT) -> None:
+        """Sets the features on a single message. Utility method."""
+        tokens = message.get(TEXT_TOKENS)
+
+        # If the message doesn't have tokens, we can't create features.
+        if not tokens:
+            return None
+
+        # Make distinction between sentence and sequence features
+        text_vector = self.tfm.transform([message.get(TEXT)])
+        word_vectors = self.tfm.transform([t.text for t in tokens])
+
+        final_sequence_features = Features(
+            word_vectors,
+            FEATURE_TYPE_SEQUENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sequence_features)
+        final_sentence_features = Features(
+            text_vector,
+            FEATURE_TYPE_SENTENCE,
+            attribute,
+            self._config[FEATURIZER_CLASS_ALIAS],
+        )
+        message.add_features(final_sentence_features)
+
+    def process(self, messages: List[Message]) -> List[Message]:
+        """Processes incoming message and compute and set features."""
+        for message in messages:
+            for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
+                self._set_features(message, attribute)
+        return messages
+
+    def process_training_data(self, training_data: TrainingData) -> TrainingData:
+        """Processes the training examples in the given training data in-place."""
+        self.process(training_data.training_examples)
+        return training_data
+
+    def persist(self) -> None:
+        """
+        Persist this model into the passed directory.
+
+        Returns the metadata necessary to load the model again. In this case; `None`.
+        """
+        with self._model_storage.write_to(self._resource) as model_dir:
+            dump(self.tfm, model_dir / "tfidfvectorizer.joblib")
+
+    @classmethod
+    def load(
+        cls,
+        config: Dict[Text, Any],
+        model_storage: ModelStorage,
+        resource: Resource,
+        execution_context: ExecutionContext,
+    ) -> GraphComponent:
+        """Loads trained component from disk."""
+        try:
+            with model_storage.read_from(resource) as model_dir:
+                tfidfvectorizer = load(model_dir / "tfidfvectorizer.joblib")
+                component = cls(
+                    config, execution_context.node_name, model_storage, resource
+                )
+                component.tfm = tfidfvectorizer
+        except (ValueError, FileNotFoundError):
+            logger.debug(
+                f"Couldn't load metadata for component '{cls.__name__}' as the persisted "
+                f"model data couldn't be loaded."
+            )
+        return component
+
+    @classmethod
+    def validate_config(cls, config: Dict[Text, Any]) -> None:
+        """Validates that the component is configured properly."""
+        pass