dc6079821b
Docs Tests / Check for file changes (push) Has been cancelled
Docs Tests / Test Documentation (push) Has been cancelled
Docs Tests / Documentation Linting Checks (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-policies) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.8, test-policies) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.9, test-policies) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.10, test-policies) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.8, test-policies) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-performance) (push) Has been cancelled
Continuous Integration / Run Tests (windows-2022, 3.9, test-policies) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (ubuntu-24.04, 3.10) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (ubuntu-24.04, 3.8) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (ubuntu-24.04, 3.9) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (windows-2022, 3.10) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (windows-2022, 3.8) (push) Has been cancelled
Continuous Integration / Run Flaky Tests (windows-2022, 3.9) (push) Has been cancelled
Continuous Integration / Check for file changes (push) Has been cancelled
Continuous Integration / Wait for docs tests (push) Has been cancelled
Continuous Integration / Code Quality (push) Has been cancelled
Continuous Integration / Check for changelog (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-cli) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-core-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-full-model-training) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-nlu-featurizers) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-nlu-predictors) (push) Has been cancelled
Continuous Integration / Run Tests (ubuntu-24.04, 3.10, test-other-unit-tests) (push) Has been cancelled
Continuous Integration / Upload coverage reports to codeclimate (push) Has been cancelled
Continuous Integration / Run Non-Sequential Integration Tests (push) Has been cancelled
Continuous Integration / Run Broker Integration Tests (push) Has been cancelled
Continuous Integration / Run Sequential Integration Tests (push) Has been cancelled
Continuous Integration / Build Docker base images and setup environment (push) Has been cancelled
Continuous Integration / Build Docker (default) (push) Has been cancelled
Continuous Integration / Build Docker (full) (push) Has been cancelled
Continuous Integration / Build Docker (mitie-en) (push) Has been cancelled
Continuous Integration / Build Docker (spacy-de) (push) Has been cancelled
Continuous Integration / Build Docker (spacy-en) (push) Has been cancelled
Continuous Integration / Build Docker (spacy-it) (push) Has been cancelled
Continuous Integration / Deploy to PyPI (push) Has been cancelled
Continuous Integration / Notify Slack & Publish Release Notes (push) Has been cancelled
Publish Documentation / Evaluate release tag (push) Has been cancelled
Publish Documentation / Prebuild Docs (push) Has been cancelled
Publish Documentation / Preview Docs (push) Has been cancelled
Publish Documentation / Check for file changes (push) Has been cancelled
Publish Documentation / Publish Docs (push) Has been cancelled
Automatic PR Merger / mergepal (push) Has been cancelled
CI Github Actions / Run Tests (push) Has been cancelled
Semgrep / Semgrep Workflow Security Scan (push) Has been cancelled
837 lines
29 KiB
Python
837 lines
29 KiB
Python
import json
|
|
import logging
|
|
from collections import deque, defaultdict
|
|
|
|
import uuid
|
|
import typing
|
|
from typing import (
|
|
List,
|
|
Text,
|
|
Deque,
|
|
Dict,
|
|
Optional,
|
|
Tuple,
|
|
Any,
|
|
Set,
|
|
ValuesView,
|
|
Union,
|
|
Sequence,
|
|
)
|
|
|
|
import rasa.shared.utils.io
|
|
from rasa.shared.core.constants import (
|
|
ACTION_LISTEN_NAME,
|
|
ACTION_SESSION_START_NAME,
|
|
ACTION_UNLIKELY_INTENT_NAME,
|
|
)
|
|
from rasa.shared.core.conversation import Dialogue
|
|
from rasa.shared.core.domain import Domain
|
|
from rasa.shared.core.events import (
|
|
UserUttered,
|
|
ActionExecuted,
|
|
Event,
|
|
SessionStarted,
|
|
SlotSet,
|
|
)
|
|
from rasa.shared.core.trackers import DialogueStateTracker
|
|
from rasa.shared.exceptions import RasaCoreException
|
|
|
|
|
|
if typing.TYPE_CHECKING:
|
|
import networkx as nx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Checkpoint id used to identify story starting blocks
|
|
STORY_START = "STORY_START"
|
|
|
|
# Checkpoint id used to identify story end blocks
|
|
STORY_END = None
|
|
|
|
# need abbreviations otherwise they are not visualized well
|
|
GENERATED_CHECKPOINT_PREFIX = "GENR_"
|
|
CHECKPOINT_CYCLE_PREFIX = "CYCL_"
|
|
|
|
GENERATED_HASH_LENGTH = 5
|
|
|
|
FORM_PREFIX = "form: "
|
|
# prefix for storystep ID to get reproducible sorting results
|
|
# will get increased with each new instance
|
|
STEP_COUNT = 1
|
|
|
|
|
|
class EventTypeError(RasaCoreException, ValueError):
|
|
"""Represents an error caused by a Rasa Core event not being of the expected
|
|
type.
|
|
|
|
"""
|
|
|
|
|
|
class Checkpoint:
|
|
"""Represents places where trackers split.
|
|
|
|
This currently happens if
|
|
- users place manual checkpoints in their stories
|
|
- have `or` statements for intents in their stories.
|
|
"""
|
|
|
|
def __init__(
|
|
self, name: Text, conditions: Optional[Dict[Text, Any]] = None
|
|
) -> None:
|
|
"""Creates `Checkpoint`.
|
|
|
|
Args:
|
|
name: Name of the checkpoint.
|
|
conditions: Slot conditions for this checkpoint.
|
|
"""
|
|
self.name = name
|
|
self.conditions = conditions if conditions else {}
|
|
|
|
def as_story_string(self) -> Text:
|
|
dumped_conds = json.dumps(self.conditions) if self.conditions else ""
|
|
return f"{self.name}{dumped_conds}"
|
|
|
|
def filter_trackers(
|
|
self, trackers: List[DialogueStateTracker]
|
|
) -> List[DialogueStateTracker]:
|
|
"""Filters out all trackers that do not satisfy the conditions."""
|
|
if not self.conditions:
|
|
return trackers
|
|
|
|
for slot_name, slot_value in self.conditions.items():
|
|
trackers = [t for t in trackers if t.get_slot(slot_name) == slot_value]
|
|
return trackers
|
|
|
|
def __repr__(self) -> Text:
|
|
return "Checkpoint(name={!r}, conditions={})".format(
|
|
self.name, json.dumps(self.conditions)
|
|
)
|
|
|
|
|
|
class StoryStep:
|
|
"""A StoryStep is a section of a story block between two checkpoints.
|
|
|
|
NOTE: Checkpoints are not only limited to those manually written
|
|
in the story file, but are also implicitly created at points where
|
|
multiple intents are separated in one line by chaining them with "OR"s.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
block_name: Text,
|
|
start_checkpoints: Optional[List[Checkpoint]] = None,
|
|
end_checkpoints: Optional[List[Checkpoint]] = None,
|
|
events: Optional[List[Union[Event, List[Event]]]] = None,
|
|
source_name: Optional[Text] = None,
|
|
) -> None:
|
|
"""Initialise `StoryStep` default attributes."""
|
|
self.end_checkpoints = end_checkpoints if end_checkpoints else []
|
|
self.start_checkpoints = start_checkpoints if start_checkpoints else []
|
|
self.events = events if events else []
|
|
self.block_name = block_name
|
|
self.source_name = source_name
|
|
# put a counter prefix to uuid to get reproducible sorting results
|
|
global STEP_COUNT
|
|
self.id = "{}_{}".format(STEP_COUNT, uuid.uuid4().hex)
|
|
STEP_COUNT += 1
|
|
|
|
def create_copy(self, use_new_id: bool) -> "StoryStep":
|
|
copied = StoryStep(
|
|
self.block_name,
|
|
self.start_checkpoints,
|
|
self.end_checkpoints,
|
|
self.events[:],
|
|
self.source_name,
|
|
)
|
|
if not use_new_id:
|
|
copied.id = self.id
|
|
return copied
|
|
|
|
def add_user_message(self, user_message: UserUttered) -> None:
|
|
self.add_event(user_message)
|
|
|
|
def add_event(self, event: Event) -> None:
|
|
self.events.append(event)
|
|
|
|
def add_events(self, events: List[Event]) -> None:
|
|
self.events.append(events)
|
|
|
|
@staticmethod
|
|
def _checkpoint_string(story_step_element: Checkpoint) -> Text:
|
|
return f"> {story_step_element.as_story_string()}\n"
|
|
|
|
@staticmethod
|
|
def _user_string(story_step_element: UserUttered, e2e: bool) -> Text:
|
|
return f"* {story_step_element.as_story_string(e2e)}\n"
|
|
|
|
@staticmethod
|
|
def _bot_string(story_step_element: Event) -> Text:
|
|
return f" - {story_step_element.as_story_string()}\n"
|
|
|
|
@staticmethod
|
|
def _event_to_story_string(event: Event, e2e: bool) -> Optional[Text]:
|
|
if isinstance(event, UserUttered):
|
|
return event.as_story_string(e2e=e2e)
|
|
return event.as_story_string()
|
|
|
|
@staticmethod
|
|
def _or_string(story_step_element: Sequence[Event], e2e: bool) -> Optional[Text]:
|
|
for event in story_step_element:
|
|
# OR statement can also contain `slot_was_set`, and
|
|
# we're going to ignore this events when representing
|
|
# the story as a string
|
|
if not isinstance(event, UserUttered) and not isinstance(event, SlotSet):
|
|
raise EventTypeError(
|
|
"OR statement events must be of type `UserUttered` or `SlotSet`."
|
|
)
|
|
|
|
event_as_strings = [
|
|
StoryStep._event_to_story_string(element, e2e)
|
|
for element in story_step_element
|
|
]
|
|
result = " OR ".join([event for event in event_as_strings if event is not None])
|
|
|
|
return f"* {result}\n"
|
|
|
|
def as_story_string(self, flat: bool = False, e2e: bool = False) -> Text:
|
|
"""Returns a story as a string."""
|
|
# if the result should be flattened, we
|
|
# will exclude the caption and any checkpoints.
|
|
if flat:
|
|
result = ""
|
|
else:
|
|
result = f"\n## {self.block_name}\n"
|
|
for checkpoint in self.start_checkpoints:
|
|
if checkpoint.name != STORY_START:
|
|
result += self._checkpoint_string(checkpoint)
|
|
|
|
for event in self.events:
|
|
if (
|
|
self.is_action_listen(event)
|
|
or self.is_action_session_start(event)
|
|
or self.is_action_unlikely_intent(event)
|
|
or isinstance(event, SessionStarted)
|
|
):
|
|
continue
|
|
|
|
if isinstance(event, UserUttered):
|
|
result += self._user_string(event, e2e)
|
|
elif isinstance(event, Event):
|
|
converted = event.as_story_string()
|
|
if converted:
|
|
result += self._bot_string(event)
|
|
elif isinstance(event, list):
|
|
# The story reader classes support reading stories in
|
|
# conversion mode. When this mode is enabled, OR statements
|
|
# are represented as lists of events.
|
|
or_string = self._or_string(event, e2e)
|
|
if or_string:
|
|
result += or_string
|
|
else:
|
|
raise Exception(f"Unexpected element in story step: {event}")
|
|
|
|
if not flat:
|
|
for checkpoint in self.end_checkpoints:
|
|
result += self._checkpoint_string(checkpoint)
|
|
return result
|
|
|
|
@staticmethod
|
|
def is_action_listen(event: Event) -> bool:
|
|
# this is not an `isinstance` because
|
|
# we don't want to allow subclasses here
|
|
return type(event) == ActionExecuted and event.action_name == ACTION_LISTEN_NAME
|
|
|
|
@staticmethod
|
|
def is_action_unlikely_intent(event: Event) -> bool:
|
|
"""Checks if the executed action is a `action_unlikely_intent`."""
|
|
return (
|
|
type(event) == ActionExecuted
|
|
and event.action_name == ACTION_UNLIKELY_INTENT_NAME
|
|
)
|
|
|
|
@staticmethod
|
|
def is_action_session_start(event: Event) -> bool:
|
|
"""Checks if the executed action is a `action_session_start`."""
|
|
# this is not an `isinstance` because
|
|
# we don't want to allow subclasses here
|
|
return (
|
|
type(event) == ActionExecuted
|
|
and event.action_name == ACTION_SESSION_START_NAME
|
|
)
|
|
|
|
def _add_action_listen(self, events: List[Event]) -> None:
|
|
if not events or not self.is_action_listen(events[-1]):
|
|
# do not add second action_listen
|
|
events.append(ActionExecuted(ACTION_LISTEN_NAME))
|
|
|
|
def explicit_events(
|
|
self, domain: Domain, should_append_final_listen: bool = True
|
|
) -> List[Event]:
|
|
"""Returns events contained in the story step including implicit events.
|
|
|
|
Not all events are always listed in the story dsl. This
|
|
includes listen actions as well as implicitly
|
|
set slots. This functions makes these events explicit and
|
|
returns them with the rest of the steps events.
|
|
"""
|
|
events: List[Event] = []
|
|
|
|
for e in self.events:
|
|
if isinstance(e, UserUttered):
|
|
self._add_action_listen(events)
|
|
events.append(e)
|
|
events.extend(domain.slots_for_entities(e.entities))
|
|
else:
|
|
events.append(e)
|
|
|
|
if not self.end_checkpoints and should_append_final_listen:
|
|
self._add_action_listen(events)
|
|
|
|
return events
|
|
|
|
def __repr__(self) -> Text:
|
|
return (
|
|
"StoryStep("
|
|
"block_name={!r}, "
|
|
"start_checkpoints={!r}, "
|
|
"end_checkpoints={!r}, "
|
|
"events={!r})".format(
|
|
self.block_name,
|
|
self.start_checkpoints,
|
|
self.end_checkpoints,
|
|
self.events,
|
|
)
|
|
)
|
|
|
|
|
|
class RuleStep(StoryStep):
|
|
"""A Special type of StoryStep representing a Rule."""
|
|
|
|
def __init__(
|
|
self,
|
|
block_name: Optional[Text] = None,
|
|
start_checkpoints: Optional[List[Checkpoint]] = None,
|
|
end_checkpoints: Optional[List[Checkpoint]] = None,
|
|
events: Optional[List[Union[Event, List[Event]]]] = None,
|
|
source_name: Optional[Text] = None,
|
|
condition_events_indices: Optional[Set[int]] = None,
|
|
) -> None:
|
|
super().__init__(
|
|
block_name, start_checkpoints, end_checkpoints, events, source_name
|
|
)
|
|
self.condition_events_indices = (
|
|
condition_events_indices if condition_events_indices else set()
|
|
)
|
|
|
|
def create_copy(self, use_new_id: bool) -> "StoryStep":
|
|
copied = RuleStep(
|
|
self.block_name,
|
|
self.start_checkpoints,
|
|
self.end_checkpoints,
|
|
self.events[:],
|
|
self.source_name,
|
|
self.condition_events_indices,
|
|
)
|
|
if not use_new_id:
|
|
copied.id = self.id
|
|
return copied
|
|
|
|
def __repr__(self) -> Text:
|
|
return (
|
|
"RuleStep("
|
|
"block_name={!r}, "
|
|
"start_checkpoints={!r}, "
|
|
"end_checkpoints={!r}, "
|
|
"events={!r})".format(
|
|
self.block_name,
|
|
self.start_checkpoints,
|
|
self.end_checkpoints,
|
|
self.events,
|
|
)
|
|
)
|
|
|
|
def get_rules_condition(self) -> List[Union[Event, List[Event]]]:
|
|
"""Returns a list of events forming a condition of the Rule."""
|
|
return [
|
|
event
|
|
for event_id, event in enumerate(self.events)
|
|
if event_id in self.condition_events_indices
|
|
]
|
|
|
|
def get_rules_events(self) -> List[Union[Event, List[Event]]]:
|
|
"""Returns a list of events forming the Rule, that are not conditions."""
|
|
return [
|
|
event
|
|
for event_id, event in enumerate(self.events)
|
|
if event_id not in self.condition_events_indices
|
|
]
|
|
|
|
def add_event_as_condition(self, event: Event) -> None:
|
|
"""Adds event to the Rule as part of its condition.
|
|
|
|
Args:
|
|
event: The event to be added.
|
|
"""
|
|
self.condition_events_indices.add(len(self.events))
|
|
self.events.append(event)
|
|
|
|
|
|
class Story:
|
|
def __init__(
|
|
self, story_steps: List[StoryStep] = None, story_name: Optional[Text] = None
|
|
) -> None:
|
|
self.story_steps = story_steps if story_steps else []
|
|
self.story_name = story_name
|
|
|
|
@staticmethod
|
|
def from_events(events: List[Event], story_name: Optional[Text] = None) -> "Story":
|
|
"""Create a story from a list of events."""
|
|
story_step = StoryStep(story_name)
|
|
for event in events:
|
|
story_step.add_event(event)
|
|
return Story([story_step], story_name)
|
|
|
|
def as_dialogue(self, sender_id: Text, domain: Domain) -> Dialogue:
|
|
events = []
|
|
for step in self.story_steps:
|
|
events.extend(
|
|
step.explicit_events(domain, should_append_final_listen=False)
|
|
)
|
|
|
|
events.append(ActionExecuted(ACTION_LISTEN_NAME))
|
|
return Dialogue(sender_id, events)
|
|
|
|
def as_story_string(self, flat: bool = False, e2e: bool = False) -> Text:
|
|
story_content = ""
|
|
for step in self.story_steps:
|
|
story_content += step.as_story_string(flat, e2e)
|
|
|
|
if flat:
|
|
if self.story_name:
|
|
name = self.story_name
|
|
else:
|
|
name = "Generated Story {}".format(hash(story_content))
|
|
return f"## {name}\n{story_content}"
|
|
else:
|
|
return story_content
|
|
|
|
|
|
class StoryGraph:
|
|
"""Graph of the story-steps pooled from all stories in the training data."""
|
|
|
|
def __init__(
|
|
self,
|
|
story_steps: List[StoryStep],
|
|
story_end_checkpoints: Optional[Dict[Text, Text]] = None,
|
|
) -> None:
|
|
self.story_steps = story_steps
|
|
self.step_lookup = {s.id: s for s in self.story_steps}
|
|
ordered_ids, cyclic_edges = StoryGraph.order_steps(story_steps)
|
|
self.ordered_ids = ordered_ids
|
|
self.cyclic_edge_ids = cyclic_edges
|
|
if story_end_checkpoints:
|
|
self.story_end_checkpoints = story_end_checkpoints
|
|
else:
|
|
self.story_end_checkpoints = {}
|
|
|
|
def __hash__(self) -> int:
|
|
"""Return hash for the story step.
|
|
|
|
Returns:
|
|
Hash of the story step.
|
|
"""
|
|
return int(self.fingerprint(), 16)
|
|
|
|
def fingerprint(self) -> Text:
|
|
"""Returns a unique hash for the stories which is stable across python runs.
|
|
|
|
Returns:
|
|
fingerprint of the stories
|
|
"""
|
|
from rasa.shared.core.training_data.story_writer.yaml_story_writer import (
|
|
YAMLStoryWriter,
|
|
)
|
|
|
|
stories_as_yaml = YAMLStoryWriter().stories_to_yaml(self.story_steps)
|
|
return rasa.shared.utils.io.deep_container_fingerprint(stories_as_yaml)
|
|
|
|
def ordered_steps(self) -> List[StoryStep]:
|
|
"""Returns the story steps ordered by topological order of the DAG."""
|
|
return [self._get_step(step_id) for step_id in self.ordered_ids]
|
|
|
|
def cyclic_edges(self) -> List[Tuple[Optional[StoryStep], Optional[StoryStep]]]:
|
|
"""Returns the story steps ordered by topological order of the DAG."""
|
|
return [
|
|
(self._get_step(source), self._get_step(target))
|
|
for source, target in self.cyclic_edge_ids
|
|
]
|
|
|
|
def merge(self, other: Optional["StoryGraph"]) -> "StoryGraph":
|
|
"""Merge two StoryGraph together."""
|
|
if not other:
|
|
return self
|
|
|
|
steps = self.story_steps.copy() + other.story_steps
|
|
story_end_checkpoints = self.story_end_checkpoints.copy().update(
|
|
other.story_end_checkpoints
|
|
)
|
|
return StoryGraph(steps, story_end_checkpoints)
|
|
|
|
@staticmethod
|
|
def overlapping_checkpoint_names(
|
|
cps: List[Checkpoint], other_cps: List[Checkpoint]
|
|
) -> Set[Text]:
|
|
"""Find overlapping checkpoints names."""
|
|
return {cp.name for cp in cps} & {cp.name for cp in other_cps}
|
|
|
|
def with_cycles_removed(self) -> "StoryGraph":
|
|
"""Create a graph with the cyclic edges removed from this graph."""
|
|
story_end_checkpoints = self.story_end_checkpoints.copy()
|
|
cyclic_edge_ids = self.cyclic_edge_ids
|
|
# we need to remove the start steps and replace them with steps ending
|
|
# in a special end checkpoint
|
|
|
|
story_steps = {s.id: s for s in self.story_steps}
|
|
|
|
# collect all overlapping checkpoints
|
|
# we will remove unused start ones
|
|
all_overlapping_cps = set()
|
|
|
|
if self.cyclic_edge_ids:
|
|
# we are going to do this in a recursive way. we are going to
|
|
# remove one cycle and then we are going to
|
|
# let the cycle detection run again
|
|
# this is not inherently necessary so if this becomes a performance
|
|
# issue, we can change it. It is actually enough to run the cycle
|
|
# detection only once and then remove one cycle after another, but
|
|
# since removing the cycle is done by adding / removing edges and
|
|
# nodes
|
|
# the logic is a lot easier if we only need to make sure the
|
|
# change is consistent if we only change one compared to
|
|
# changing all of them.
|
|
|
|
for s, e in cyclic_edge_ids:
|
|
cid = generate_id(max_chars=GENERATED_HASH_LENGTH)
|
|
prefix = GENERATED_CHECKPOINT_PREFIX + CHECKPOINT_CYCLE_PREFIX
|
|
# need abbreviations otherwise they are not visualized well
|
|
sink_cp_name = prefix + "SINK_" + cid
|
|
connector_cp_name = prefix + "CONN_" + cid
|
|
source_cp_name = prefix + "SRC_" + cid
|
|
story_end_checkpoints[sink_cp_name] = source_cp_name
|
|
|
|
overlapping_cps = self.overlapping_checkpoint_names(
|
|
story_steps[s].end_checkpoints, story_steps[e].start_checkpoints
|
|
)
|
|
|
|
all_overlapping_cps.update(overlapping_cps)
|
|
|
|
# change end checkpoints of starts
|
|
start = story_steps[s].create_copy(use_new_id=False)
|
|
start.end_checkpoints = [
|
|
cp for cp in start.end_checkpoints if cp.name not in overlapping_cps
|
|
]
|
|
start.end_checkpoints.append(Checkpoint(sink_cp_name))
|
|
story_steps[s] = start
|
|
|
|
needs_connector = False
|
|
|
|
for k, step in list(story_steps.items()):
|
|
additional_ends = []
|
|
for original_cp in overlapping_cps:
|
|
for cp in step.start_checkpoints:
|
|
if cp.name == original_cp:
|
|
if k == e:
|
|
cp_name = source_cp_name
|
|
else:
|
|
cp_name = connector_cp_name
|
|
needs_connector = True
|
|
|
|
if not self._is_checkpoint_in_list(
|
|
cp_name, cp.conditions, step.start_checkpoints
|
|
):
|
|
# add checkpoint only if it was not added
|
|
additional_ends.append(
|
|
Checkpoint(cp_name, cp.conditions)
|
|
)
|
|
|
|
if additional_ends:
|
|
updated = step.create_copy(use_new_id=False)
|
|
updated.start_checkpoints.extend(additional_ends)
|
|
story_steps[k] = updated
|
|
|
|
if needs_connector:
|
|
start.end_checkpoints.append(Checkpoint(connector_cp_name))
|
|
|
|
# the process above may generate unused checkpoints
|
|
# we need to find them and remove them
|
|
self._remove_unused_generated_cps(
|
|
story_steps, all_overlapping_cps, story_end_checkpoints
|
|
)
|
|
|
|
return StoryGraph(list(story_steps.values()), story_end_checkpoints)
|
|
|
|
@staticmethod
|
|
def _checkpoint_difference(
|
|
cps: List[Checkpoint], cp_name_to_ignore: Set[Text]
|
|
) -> List[Checkpoint]:
|
|
"""Finds checkpoints which names are
|
|
different form names of checkpoints to ignore.
|
|
"""
|
|
return [cp for cp in cps if cp.name not in cp_name_to_ignore]
|
|
|
|
def _remove_unused_generated_cps(
|
|
self,
|
|
story_steps: Dict[Text, StoryStep],
|
|
overlapping_cps: Set[Text],
|
|
story_end_checkpoints: Dict[Text, Text],
|
|
) -> None:
|
|
"""Finds unused generated checkpoints
|
|
and remove them from story steps.
|
|
"""
|
|
unused_cps = self._find_unused_checkpoints(
|
|
story_steps.values(), story_end_checkpoints
|
|
)
|
|
|
|
unused_overlapping_cps = unused_cps.intersection(overlapping_cps)
|
|
|
|
unused_genr_cps = {
|
|
cp_name
|
|
for cp_name in unused_cps
|
|
if cp_name is not None and cp_name.startswith(GENERATED_CHECKPOINT_PREFIX)
|
|
}
|
|
|
|
k_to_remove = set()
|
|
for k, step in story_steps.items():
|
|
# changed all ends
|
|
updated = step.create_copy(use_new_id=False)
|
|
updated.start_checkpoints = self._checkpoint_difference(
|
|
updated.start_checkpoints, unused_overlapping_cps
|
|
)
|
|
|
|
# remove generated unused end checkpoints
|
|
updated.end_checkpoints = self._checkpoint_difference(
|
|
updated.end_checkpoints, unused_genr_cps
|
|
)
|
|
|
|
if (
|
|
step.start_checkpoints
|
|
and not updated.start_checkpoints
|
|
or step.end_checkpoints
|
|
and not updated.end_checkpoints
|
|
):
|
|
# remove story step if the generated checkpoints
|
|
# were the only ones
|
|
k_to_remove.add(k)
|
|
|
|
story_steps[k] = updated
|
|
|
|
# remove unwanted story steps
|
|
for k in k_to_remove:
|
|
del story_steps[k]
|
|
|
|
@staticmethod
|
|
def _is_checkpoint_in_list(
|
|
checkpoint_name: Text, conditions: Dict[Text, Any], cps: List[Checkpoint]
|
|
) -> bool:
|
|
"""Checks if checkpoint with name and conditions is
|
|
already in the list of checkpoints.
|
|
"""
|
|
for cp in cps:
|
|
if checkpoint_name == cp.name and conditions == cp.conditions:
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def _find_unused_checkpoints(
|
|
story_steps: ValuesView[StoryStep], story_end_checkpoints: Dict[Text, Text]
|
|
) -> Set[Optional[Text]]:
|
|
"""Finds all unused checkpoints."""
|
|
collected_start = {STORY_END, STORY_START}
|
|
collected_end = {STORY_END, STORY_START}
|
|
|
|
for step in story_steps:
|
|
for start in step.start_checkpoints:
|
|
collected_start.add(start.name)
|
|
for end in step.end_checkpoints:
|
|
start_name = story_end_checkpoints.get(end.name, end.name)
|
|
collected_end.add(start_name)
|
|
|
|
return collected_end.symmetric_difference(collected_start)
|
|
|
|
def _get_step(self, step_id: Text) -> StoryStep:
|
|
"""Looks a story step up by its id."""
|
|
return self.step_lookup[step_id]
|
|
|
|
@staticmethod
|
|
def order_steps(
|
|
story_steps: List[StoryStep],
|
|
) -> Tuple[deque, List[Tuple[Text, Text]]]:
|
|
"""Topological sort of the steps returning the ids of the steps."""
|
|
checkpoints = StoryGraph._group_by_start_checkpoint(story_steps)
|
|
graph = {
|
|
s.id: {
|
|
other.id for end in s.end_checkpoints for other in checkpoints[end.name]
|
|
}
|
|
for s in story_steps
|
|
}
|
|
return StoryGraph.topological_sort(graph)
|
|
|
|
@staticmethod
|
|
def _group_by_start_checkpoint(
|
|
story_steps: List[StoryStep],
|
|
) -> Dict[Text, List[StoryStep]]:
|
|
"""Returns all the start checkpoint of the steps."""
|
|
checkpoints = defaultdict(list)
|
|
for step in story_steps:
|
|
for start in step.start_checkpoints:
|
|
checkpoints[start.name].append(step)
|
|
return checkpoints
|
|
|
|
@staticmethod
|
|
def topological_sort(
|
|
graph: Dict[Text, Set[Text]]
|
|
) -> Tuple[deque, List[Tuple[Text, Text]]]:
|
|
"""Creates a top sort of a directed graph. This is an unstable sorting!
|
|
|
|
The function returns the sorted nodes as well as the edges that need
|
|
to be removed from the graph to make it acyclic (and hence, sortable).
|
|
|
|
The graph should be represented as a dictionary, e.g.:
|
|
|
|
>>> example_graph = {
|
|
... "a": set("b", "c", "d"),
|
|
... "b": set(),
|
|
... "c": set("d"),
|
|
... "d": set(),
|
|
... "e": set("f"),
|
|
... "f": set()}
|
|
>>> StoryGraph.topological_sort(example_graph)
|
|
(deque([u'e', u'f', u'a', u'c', u'd', u'b']), [])
|
|
"""
|
|
# noinspection PyPep8Naming
|
|
GRAY, BLACK = 0, 1
|
|
|
|
ordered: Deque = deque()
|
|
unprocessed = sorted(set(graph))
|
|
visited_nodes = {}
|
|
|
|
removed_edges = set()
|
|
|
|
def dfs(node: Text) -> None:
|
|
visited_nodes[node] = GRAY
|
|
for k in sorted(graph.get(node, set())):
|
|
sk = visited_nodes.get(k, None)
|
|
if sk == GRAY:
|
|
removed_edges.add((node, k))
|
|
continue
|
|
if sk == BLACK:
|
|
continue
|
|
unprocessed.remove(k)
|
|
dfs(k)
|
|
ordered.appendleft(node)
|
|
visited_nodes[node] = BLACK
|
|
|
|
while unprocessed:
|
|
dfs(unprocessed.pop())
|
|
|
|
return ordered, sorted(removed_edges)
|
|
|
|
def visualize(self, output_file: Optional[Text] = None) -> "nx.MultiDiGraph":
|
|
import networkx as nx
|
|
from rasa.shared.core.training_data import visualization
|
|
from colorhash import ColorHash
|
|
|
|
graph = nx.MultiDiGraph()
|
|
next_node_idx = [0]
|
|
nodes = {"STORY_START": 0, "STORY_END": -1}
|
|
|
|
def ensure_checkpoint_is_drawn(cp: Checkpoint) -> None:
|
|
if cp.name not in nodes:
|
|
next_node_idx[0] += 1
|
|
nodes[cp.name] = next_node_idx[0]
|
|
|
|
if cp.name.startswith(GENERATED_CHECKPOINT_PREFIX):
|
|
# colors generated checkpoints based on their hash
|
|
color = ColorHash(cp.name[-GENERATED_HASH_LENGTH:]).hex
|
|
graph.add_node(
|
|
next_node_idx[0],
|
|
label=_cap_length(cp.name),
|
|
style="filled",
|
|
fillcolor=color,
|
|
)
|
|
else:
|
|
graph.add_node(next_node_idx[0], label=_cap_length(cp.name))
|
|
|
|
graph.add_node(
|
|
nodes["STORY_START"], label="START", fillcolor="green", style="filled"
|
|
)
|
|
graph.add_node(nodes["STORY_END"], label="END", fillcolor="red", style="filled")
|
|
|
|
for step in self.story_steps:
|
|
next_node_idx[0] += 1
|
|
step_idx = next_node_idx[0]
|
|
|
|
graph.add_node(
|
|
next_node_idx[0],
|
|
label=_cap_length(step.block_name),
|
|
style="filled",
|
|
fillcolor="lightblue",
|
|
shape="rect",
|
|
)
|
|
|
|
for c in step.start_checkpoints:
|
|
ensure_checkpoint_is_drawn(c)
|
|
graph.add_edge(nodes[c.name], step_idx)
|
|
for c in step.end_checkpoints:
|
|
ensure_checkpoint_is_drawn(c)
|
|
graph.add_edge(step_idx, nodes[c.name])
|
|
|
|
if not step.end_checkpoints:
|
|
graph.add_edge(step_idx, nodes["STORY_END"])
|
|
|
|
if output_file:
|
|
visualization.persist_graph(graph, output_file)
|
|
|
|
return graph
|
|
|
|
def is_empty(self) -> bool:
|
|
"""Checks if `StoryGraph` is empty."""
|
|
return not self.story_steps
|
|
|
|
def __repr__(self) -> Text:
|
|
"""Returns text representation of object."""
|
|
return f"{self.__class__.__name__}: {len(self.story_steps)} story steps"
|
|
|
|
|
|
def generate_id(prefix: Text = "", max_chars: Optional[int] = None) -> Text:
|
|
"""Generate a random UUID.
|
|
|
|
Args:
|
|
prefix: String to prefix the ID with.
|
|
max_chars: Maximum number of characters.
|
|
|
|
Returns:
|
|
Generated random UUID.
|
|
"""
|
|
import uuid
|
|
|
|
gid = uuid.uuid4().hex
|
|
if max_chars:
|
|
gid = gid[:max_chars]
|
|
|
|
return f"{prefix}{gid}"
|
|
|
|
|
|
def _cap_length(s: Text, char_limit: int = 20, append_ellipsis: bool = True) -> Text:
|
|
"""Makes sure the string doesn't exceed the passed char limit.
|
|
|
|
Appends an ellipsis if the string is too long.
|
|
"""
|
|
if len(s) > char_limit:
|
|
if append_ellipsis:
|
|
return s[: char_limit - 3] + "..."
|
|
else:
|
|
return s[:char_limit]
|
|
else:
|
|
return s
|