chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,15 @@
|
||||
from .screenwriter import Screenwriter
|
||||
from .storyboard_artist import StoryboardArtist
|
||||
from .camera_image_generator import CameraImageGenerator
|
||||
from .character_extractor import CharacterExtractor
|
||||
from .character_portraits_generator import CharacterPortraitsGenerator
|
||||
from .reference_image_selector import ReferenceImageSelector
|
||||
|
||||
__all__ = [
|
||||
"Screenwriter",
|
||||
"StoryboardArtist",
|
||||
"CameraImageGenerator",
|
||||
"CharacterExtractor",
|
||||
"CharacterPortraitsGenerator",
|
||||
"ReferenceImageSelector",
|
||||
]
|
||||
@@ -0,0 +1,147 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from utils.image import image_path_to_b64
|
||||
|
||||
|
||||
|
||||
system_prompt_template_select_most_consistent_image = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional visual assessment expert. Your expertise includes identifying Character Consistency and Spatial Consistency between candidate image and reference image, and assessing semantic consistency between candidate image and text description.
|
||||
|
||||
[Task]
|
||||
Based on the reference image provided by the user, the text description of the target image, and several candidate images, evaluate which candidate image performs best in the following aspects:
|
||||
- Character Consistency: Whether the character features (a. gender, b.ethnicity, c.age, d.facial features, e.body shape, f.outlook, g. hairstyle) in the candidate image align with those of the character in the reference image.
|
||||
- Spatial Consistency: Whether the relative positions between characters (e.g. Character A is on the left, character B is on the right, scene layout, perspective, and other spatial relationships) in the candidate image are consistent with those in the reference image.
|
||||
- Description Accuracy: Whether the candidate image accurately reflects the content described in the text (Note: The text description describes the target image we want, which is not an editing instruction).
|
||||
|
||||
[Input]
|
||||
The user will provide the following content:
|
||||
- Reference images: These include images of characters or other perspectives, each along with a brief text description. For example, "Reference Image 0: A young girl with long brown hair wearing a red dress." then follow the corresponding image. The index starts from 0.
|
||||
- Candidate images: The candidate images to be evaluated. For example, "Generated Image 0", then follow a generated image. The index starts from 0.
|
||||
- Text description for target image: This describes what the generated image should contain. It is enclosed <TARGET_DESCRIPTION_START> and <TARGET_DESCRIPTION_END> tags.
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
- Prioritize Character Consistency: Ensure that the characters in the generated image are highly consistent with those in the reference image in terms of visual features (e.g., a. gender b.ethnicity, c.age, d.facial features, e.body shape, f.outlook, g. hairstyle etc.).
|
||||
- Focus on Spatial Consistency: Verify whether the relative positions of characters, object arrangements, and perspectives align logically with the reference image (e.g., if Character A is on the left and Character B is on the right in the reference image, the generated image should not reverse this).
|
||||
- Strictly Compare with Text Description: The generated image must adhere to key elements in the text description (e.g., actions, scenes, objects, etc.), while disregarding parts related to editing instructions (as the input description reflects the expected outcome rather than directives).
|
||||
- If multiple images partially meet the criteria, select the one with the highest overall consistency; if none are ideal, choose the relatively best option and explain its shortcomings.
|
||||
- Ensure the key elements described in the text are present in the selected image.
|
||||
- Avoid subjective preferences; base all analysis on objective comparisons.
|
||||
- Prioritize images without white borders, black edges, or any additional framing.
|
||||
"""
|
||||
|
||||
human_prompt_template_select_most_consistent_image = \
|
||||
"""
|
||||
<TARGET_DESCRIPTION_START>
|
||||
{target_description}
|
||||
<TARGET_DESCRIPTION_END>
|
||||
"""
|
||||
|
||||
|
||||
class BestImageResponse(BaseModel):
|
||||
best_image_index: int = Field(
|
||||
...,
|
||||
description="The index of the best image."
|
||||
)
|
||||
reason: str = Field(
|
||||
...,
|
||||
description="The reason why the image is the best."
|
||||
)
|
||||
|
||||
|
||||
class BestImageSelector:
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
chat_model: str,
|
||||
):
|
||||
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
model_provider="openai",
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=lambda retry_state: logging.warning(f"Retrying best image selection due to {retry_state.outcome.exception()}"),
|
||||
)
|
||||
async def __call__(
|
||||
self,
|
||||
reference_image_path_and_text_pairs: List[Tuple[str, str]],
|
||||
target_description: str,
|
||||
candidate_image_paths: List[str],
|
||||
) -> str:
|
||||
"""
|
||||
Args:
|
||||
ref_image_path_and_text_pairs:
|
||||
A list of tuples containing reference image paths and their descriptions.
|
||||
|
||||
target_description:
|
||||
The description of the target image.
|
||||
|
||||
candidate_image_paths:
|
||||
A list of paths to the candidate images to be evaluated.
|
||||
"""
|
||||
|
||||
if not candidate_image_paths:
|
||||
logging.warning("No candidate images provided; skipping best image selection")
|
||||
raise ValueError("No candidate images to select from")
|
||||
|
||||
logging.info(f"Selecting the best image from candidates: {candidate_image_paths}")
|
||||
|
||||
human_content = []
|
||||
for idx, (ref_image_path, text) in enumerate(reference_image_path_and_text_pairs):
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": f"Reference Image {idx}: {text}"
|
||||
})
|
||||
human_content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_path_to_b64(ref_image_path, mime=True)}
|
||||
})
|
||||
|
||||
for idx, candidate_image_path in enumerate(candidate_image_paths):
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": f"Candidate Image {idx}"
|
||||
})
|
||||
human_content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_path_to_b64(candidate_image_path, mime=True)}
|
||||
})
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": human_prompt_template_select_most_consistent_image.format(target_description=target_description)
|
||||
})
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=BestImageResponse)
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt_template_select_most_consistent_image.format(format_instructions=parser.get_format_instructions())),
|
||||
HumanMessage(content=human_content)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
|
||||
response = await chain.ainvoke(messages)
|
||||
idx = response.best_image_index
|
||||
if not isinstance(idx, int) or idx < 0 or idx >= len(candidate_image_paths):
|
||||
logging.warning(f"Received invalid best_image_index={idx}; defaulting to 0")
|
||||
idx = 0
|
||||
best_image_path = candidate_image_paths[idx]
|
||||
logging.info(f"Best image selected: {best_image_path}")
|
||||
logging.info(f"Selection reason: {response.reason}")
|
||||
return best_image_path
|
||||
@@ -0,0 +1,272 @@
|
||||
import os
|
||||
import logging
|
||||
import cv2
|
||||
from typing import List, Tuple, Union, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from scenedetect import open_video, SceneManager, split_video_ffmpeg
|
||||
from scenedetect.detectors import ContentDetector
|
||||
|
||||
from interfaces import ShotDescription, ShotBriefDescription, Camera, ImageOutput, VideoOutput
|
||||
|
||||
|
||||
from moviepy import VideoFileClip
|
||||
from PIL import Image
|
||||
|
||||
|
||||
system_prompt_template_select_reference_camera = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional video editing expert specializing in multi-camera shot analysis and scene structure modeling. You have deep knowledge of cinematic language, enabling you to understand shot sizes (e.g., wide shot, medium shot, close-up) and content inclusion relationships. You can infer hierarchical structures between camera positions based on corresponding shot descriptions.
|
||||
|
||||
[Task]
|
||||
Your task is to analyze the input camera position data to construct a "camera position tree". This tree structure represents a relationship where a parent camera's content encompasses that of a child camera. Specifically, you need to identify the parent camera for each camera position (if one exists) and determine the dependent shot indices (i.e., the specific shots within the parent camera's footage that contain the child camera's content). If a camera position has no parent, output None.
|
||||
|
||||
[Input]
|
||||
The input is a sequence of cameras. The sequence will be enclosed within <CAMERA_SEQ> and </CAMERA_SEQ>.
|
||||
Each camera contains a sequence of shots filmed by the camera, which will be enclosed within <CAMERA_N> and </CAMERA_N>, where N is the index of the camera.
|
||||
|
||||
Below is an example of the input format:
|
||||
|
||||
<CAMERA_SEQ>
|
||||
<CAMERA_0>
|
||||
Shot 0: Medium shot of the street. Alice and Bob are walking towards each other.
|
||||
Shot 2: Medium shot of the street. Alice and Bob hug each other.
|
||||
</CAMERA_0>
|
||||
<CAMERA_1>
|
||||
Shot 1: Close-up of the Alice's face. Her expression shifts from surprise to delight as she recognizes Bob.
|
||||
</CAMERA_1>
|
||||
</CAMERA_SEQ>
|
||||
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
- The language of all output values (not include keys) should be consistent with the language of the input.
|
||||
- Content Inclusion Check: The parent camera should as fully as possible contain the child camera's content in certain shots (e.g., a parent medium two-shot encompasses a child over-the-shoulder reverse shot). Analyze shot descriptions by comparing keywords (e.g., characters, actions, setting) to ensure the parent shot's field of view covers the child shot's.
|
||||
- Transition Smoothness Priority: Larger shot size as parent camera is preferred, such as Wide Shot -> Medium Shot or Medium Shot -> Close-up. The shot sizes of adjacent parent and child nodes should be as similar as possible. A direct transition from a long shot to a close-up is not allowed unless absolutely necessary.
|
||||
- Temporal Proximity: Each camera is described by its corresponding first shot, and the parent camera is located based on the description of the first shot. The shot index of the parent camera should be as close as possible to the first shot index of the child camera.
|
||||
- Logical Consistency: The camera tree should be acyclic, avoid circular dependencies. If a camera is contained by multiple potential parents, select the best match (based on shot size and content). If there is no suitable parent camera, output None.
|
||||
- When a broader perspective is not available, choose the shot with the largest overlapping field of view as the parent (the one with the most information overlap), or a shot can also serve as the parent of a reverse shot. When two cameras can be the parent of each other, choose the one with the smaller index as the parent of the camera with the larger index.
|
||||
- Only one camera can exist without a parent.
|
||||
- When describing the elements lost in a shot, carefully compare the details between the parent shot and the child shot. For example, the parent shot is a medium shot of Character A and Character B facing each other (both in profile to the camera), while the child shot is a close-up of Character A (with Character A facing the camera directly). In this case, the child shot lacks the frontal view information of Character A.
|
||||
- The first camera must be the root of the camera tree.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_select_reference_camera = \
|
||||
"""
|
||||
<CAMERA_SEQ>
|
||||
{camera_seq_str}
|
||||
</CAMERA_SEQ>
|
||||
"""
|
||||
|
||||
|
||||
class CameraParentItem(BaseModel):
|
||||
parent_cam_idx: Optional[int] = Field(
|
||||
default=None,
|
||||
description="The index of the parent camera. Set to None if the camera has no parent (e.g., for a root camera).",
|
||||
examples=[0, 1, None],
|
||||
)
|
||||
parent_shot_idx: Optional[int] = Field(
|
||||
default=None,
|
||||
description="The index of the dependent shot. Set to None if the camera has no parent (e.g., for a root camera).",
|
||||
examples=[0, 3, None],
|
||||
)
|
||||
reason: str = Field(
|
||||
description="The reason for the selection of the parent camera. If the camera has no parent, it should explain why it's a root camera.",
|
||||
examples=[
|
||||
"The parent shot's field of view covers the child shot's field of view (from medium shot to close-up)",
|
||||
"The parent shot and the child shot have a shot/reverse shot relationship.",
|
||||
"CAMERA_0 (Shot 0) establishes the entire scene and contains all characters and the setting. It is the root camera." # 补充 LLM 实际输出的例子
|
||||
],
|
||||
)
|
||||
is_parent_fully_covers_child: Optional[bool] = Field(
|
||||
default=None,
|
||||
description="Whether the parent camera fully covers the child camera's content. Set to None if the camera has no parent.",
|
||||
examples=[True, False, None],
|
||||
)
|
||||
missing_info: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The missing elements in the child shot that are not covered by the parent shot. If the parent shot fully covers the child shot, set this to None.",
|
||||
examples=[
|
||||
"The frontal view of Alice.",
|
||||
None,
|
||||
],
|
||||
)
|
||||
|
||||
class CameraTreeResponse(BaseModel):
|
||||
camera_parent_items: List[Optional[CameraParentItem]] = Field(
|
||||
description="The parent camera items for each camera. If a camera has no parent, set this to None. The length of the list should be the same as the number of cameras.",
|
||||
)
|
||||
|
||||
|
||||
|
||||
class CameraImageGenerator:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
chat_model,
|
||||
image_generator,
|
||||
video_generator,
|
||||
):
|
||||
self.chat_model = chat_model
|
||||
self.image_generator = image_generator
|
||||
self.video_generator = video_generator
|
||||
|
||||
|
||||
async def construct_camera_tree(
|
||||
self,
|
||||
cameras: List[Camera],
|
||||
shot_descs: List[Union[ShotDescription, ShotBriefDescription]],
|
||||
) -> List[Camera]:
|
||||
parser = PydanticOutputParser(pydantic_object=CameraTreeResponse)
|
||||
shot_desc_by_idx = {shot.idx: shot for shot in shot_descs}
|
||||
|
||||
camera_seq_str = "<CAMERA_SEQ>\n"
|
||||
for cam in cameras:
|
||||
camera_seq_str += f"<CAMERA_{cam.idx}>\n"
|
||||
for shot_idx in cam.active_shot_idxs:
|
||||
shot_desc = shot_desc_by_idx.get(shot_idx)
|
||||
if shot_desc is None:
|
||||
raise ValueError(f"Camera {cam.idx} references missing shot {shot_idx}")
|
||||
camera_seq_str += f"Shot {shot_idx}: {shot_desc.visual_desc}\n"
|
||||
camera_seq_str += f"</CAMERA_{cam.idx}>\n"
|
||||
camera_seq_str += "</CAMERA_SEQ>"
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt_template_select_reference_camera.format(format_instructions=parser.get_format_instructions())),
|
||||
HumanMessage(content=human_prompt_template_select_reference_camera.format(camera_seq_str=camera_seq_str)),
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
response: CameraTreeResponse = await chain.ainvoke(messages)
|
||||
parent_items = response.camera_parent_items
|
||||
if len(parent_items) != len(cameras):
|
||||
raise ValueError(f"Camera tree response length mismatch: expected {len(cameras)}, got {len(parent_items)}")
|
||||
|
||||
valid_camera_idxs = {cam.idx for cam in cameras}
|
||||
valid_shot_idxs = set(shot_desc_by_idx)
|
||||
parent_by_camera = {}
|
||||
for cam, parent_cam_item in zip(cameras, parent_items):
|
||||
parent_cam_idx = parent_cam_item.parent_cam_idx if parent_cam_item is not None else None
|
||||
parent_shot_idx = parent_cam_item.parent_shot_idx if parent_cam_item is not None else None
|
||||
if parent_cam_idx is not None and parent_cam_idx not in valid_camera_idxs:
|
||||
raise ValueError(f"Camera {cam.idx} has invalid parent camera {parent_cam_idx}")
|
||||
if parent_cam_idx == cam.idx:
|
||||
raise ValueError(f"Camera {cam.idx} cannot be its own parent")
|
||||
if parent_shot_idx is not None and parent_shot_idx not in valid_shot_idxs:
|
||||
raise ValueError(f"Camera {cam.idx} has invalid parent shot {parent_shot_idx}")
|
||||
parent_by_camera[cam.idx] = parent_cam_idx
|
||||
|
||||
for cam in cameras:
|
||||
seen = set()
|
||||
current = cam.idx
|
||||
while parent_by_camera.get(current) is not None:
|
||||
current = parent_by_camera[current]
|
||||
if current in seen:
|
||||
raise ValueError(f"Camera tree contains a cycle involving camera {cam.idx}")
|
||||
seen.add(current)
|
||||
|
||||
for cam, parent_cam_item in zip(cameras, parent_items):
|
||||
cam.parent_cam_idx = parent_cam_item.parent_cam_idx if parent_cam_item is not None else None
|
||||
cam.parent_shot_idx = parent_cam_item.parent_shot_idx if parent_cam_item is not None else None
|
||||
cam.reason = parent_cam_item.reason if parent_cam_item is not None else None
|
||||
cam.is_parent_fully_covers_child = parent_cam_item.is_parent_fully_covers_child if parent_cam_item is not None else None
|
||||
cam.missing_info = parent_cam_item.missing_info if parent_cam_item is not None else None
|
||||
return cameras
|
||||
|
||||
|
||||
async def generate_transition_video(
|
||||
self,
|
||||
first_shot_visual_desc: str,
|
||||
second_shot_visual_desc: str,
|
||||
first_shot_ff_path: str,
|
||||
progress=None,
|
||||
) -> VideoOutput:
|
||||
|
||||
prompt = f"Two shots. The transition between the shots is a cut to. The style of the two shots should be consistent."
|
||||
prompt += f"\nThe first shot description: {first_shot_visual_desc}."
|
||||
prompt += f"\nThe second shot description: {second_shot_visual_desc}."
|
||||
reference_image_paths = [first_shot_ff_path]
|
||||
video_output = await self.video_generator.generate_single_video(
|
||||
prompt=prompt,
|
||||
reference_image_paths=reference_image_paths,
|
||||
progress=progress,
|
||||
)
|
||||
return video_output
|
||||
|
||||
|
||||
def get_new_camera_image(
|
||||
self,
|
||||
transition_video_path: str,
|
||||
) -> ImageOutput:
|
||||
video = open_video(transition_video_path)
|
||||
scene_manager = SceneManager()
|
||||
scene_manager.add_detector(ContentDetector())
|
||||
scene_manager.detect_scenes(video, show_progress=False)
|
||||
scene_list = scene_manager.get_scene_list()
|
||||
output_dir = os.path.join(os.path.dirname(transition_video_path), "cache")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
split_video_ffmpeg(transition_video_path, scene_list, output_dir, show_progress=True)
|
||||
|
||||
|
||||
video_name = os.path.basename(transition_video_path).split('.')[0]
|
||||
second_video_path = os.path.join(output_dir, f"{video_name}-Scene-002.mp4")
|
||||
if os.path.exists(second_video_path):
|
||||
# use first frame of second shot as new camera image
|
||||
clip = VideoFileClip(second_video_path)
|
||||
ff = clip.get_frame(0)
|
||||
ff = Image.fromarray(ff.astype('uint8'), 'RGB')
|
||||
return ImageOutput(fmt="pil", ext="png", data=ff)
|
||||
else:
|
||||
# use last frame of transition video to instead
|
||||
clip = VideoFileClip(transition_video_path)
|
||||
lf_time = clip.duration - (1 / clip.fps)
|
||||
lf_time = max(0, lf_time)
|
||||
lf = clip.get_frame(lf_time)
|
||||
lf = Image.fromarray(lf.astype('uint8'), 'RGB')
|
||||
return ImageOutput(fmt="pil", ext="png", data=lf)
|
||||
|
||||
|
||||
async def generate_first_frame(
|
||||
self,
|
||||
shot_desc: ShotDescription,
|
||||
character_portrait_path_and_text_pairs: List[Tuple[str, str]],
|
||||
) -> ImageOutput:
|
||||
prompt = ""
|
||||
reference_image_paths = []
|
||||
for i,(path, text )in enumerate(character_portrait_path_and_text_pairs):
|
||||
prompt += f"Image {i}: {text}\n"
|
||||
reference_image_paths.append(path)
|
||||
prompt += f"Generate an image based on the following description: {shot_desc.ff_desc}."
|
||||
image_output = await self.image_generator.generate_single_image(
|
||||
prompt=prompt,
|
||||
reference_image_paths=reference_image_paths,
|
||||
size="1600x900",
|
||||
)
|
||||
return image_output
|
||||
|
||||
|
||||
|
||||
def _validate_camera_tree(cameras: List[Camera]) -> None:
|
||||
"""Reject parent assignments that would deadlock frame generation."""
|
||||
by_idx = {cam.idx: cam for cam in cameras}
|
||||
for cam in cameras:
|
||||
if cam.parent_cam_idx is None:
|
||||
continue
|
||||
if cam.parent_cam_idx == cam.idx:
|
||||
raise ValueError(f"Camera {cam.idx} lists itself as its parent.")
|
||||
if cam.parent_cam_idx not in by_idx:
|
||||
raise ValueError(f"Camera {cam.idx} references unknown parent camera {cam.parent_cam_idx}.")
|
||||
for cam in cameras:
|
||||
seen = set()
|
||||
current = cam
|
||||
while current.parent_cam_idx is not None:
|
||||
if current.idx in seen:
|
||||
raise ValueError(f"Cycle detected in camera parent graph involving camera {current.idx}.")
|
||||
seen.add(current.idx)
|
||||
current = by_idx[current.parent_cam_idx]
|
||||
@@ -0,0 +1,89 @@
|
||||
import logging
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models.base import BaseChatModel
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List
|
||||
from tenacity import retry, stop_after_attempt
|
||||
from interfaces import CharacterInScene
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from utils.retry import after_func
|
||||
|
||||
|
||||
system_prompt_template_extract_characters = \
|
||||
"""
|
||||
[Role]
|
||||
You are a top-tier movie script analysis expert.
|
||||
|
||||
[Task]
|
||||
Your task is to analyze the provided script and extract all relevant character information.
|
||||
|
||||
[Input]
|
||||
You will receive a script enclosed within <SCRIPT> and </SCRIPT>.
|
||||
|
||||
Below is a simple example of the input:
|
||||
|
||||
<SCRIPT>
|
||||
A young woman sits alone at a table, staring out the window. She takes a sip of her coffee and sighs. The liquid is no longer warm, just a bitter reminder of the time that has passed. Outside, the world moves in a blur of hurried footsteps and distant car horns, but inside the quiet café, time feels thick and heavy.
|
||||
Her finger traces the rim of the ceramic mug, following the imperfect circle over and over. The decision she had to make was supposed to be simple—a mere checkbox on the form of her life. Yesor No. Stayor Go. Yet, it had rooted itself in her chest, a tangled knot of fear and longing.
|
||||
</SCRIPT>
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
|
||||
[Guidelines]
|
||||
- Ensure that the language of all output values(not include keys) matches that used in the script.
|
||||
- Group all names referring to the same entity under one character. Select the most appropriate name as the character's identifier. If the person is a real famous person, the real person's name should be retained (e.g., Elon Musk, Bill Gates)
|
||||
- If the character's name is not mentioned, you can use reasonable pronouns to refer to them, including using their occupation or notable physical traits. For example, "the young woman" or "the barista".
|
||||
- For background characters in the script, you do not need to consider them as individual characters.
|
||||
- If a character's traits are not described or only partially outlined in the script, you need to design plausible features based on the context to make their characteristics more complete and detailed, ensuring they are vivid and evocative.
|
||||
- In static features, you need to describe the character's physical appearance, physique, and other relatively unchanging features. In dynamic features, you need to describe the character's attire, accessories, key items they carry, and other easily changeable features.
|
||||
- Don't include any information about the character's personality, role, or relationships with others in either static or dynamic features.
|
||||
- When designing character features, within reasonable limits, different character appearances should be made more distinct from each other.
|
||||
- The description of characters should be detailed, avoiding the use of abstract terms. Instead, employ descriptions that can be visualized—such as specific clothing colors and concrete physical traits (e.g., large eyes, a high nose bridge).
|
||||
"""
|
||||
|
||||
human_prompt_template_extract_characters = \
|
||||
"""
|
||||
<SCRIPT>
|
||||
{script}
|
||||
</SCRIPT>
|
||||
"""
|
||||
|
||||
|
||||
class ExtractCharactersResponse(BaseModel):
|
||||
characters: List[CharacterInScene] = Field(
|
||||
..., description="A list of characters extracted from the script."
|
||||
)
|
||||
|
||||
|
||||
|
||||
class CharacterExtractor:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model,
|
||||
):
|
||||
self.chat_model = chat_model
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=after_func,
|
||||
)
|
||||
async def extract_characters(self, script: str) -> List[CharacterInScene]:
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=ExtractCharactersResponse)
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt_template_extract_characters.format(format_instructions=parser.get_format_instructions())),
|
||||
HumanMessage(content=human_prompt_template_extract_characters.format(script=script)),
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
|
||||
response: ExtractCharactersResponse = await chain.ainvoke(messages)
|
||||
|
||||
return response.characters
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models.base import BaseChatModel
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Dict
|
||||
from interfaces import CharacterInScene, ImageOutput
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
|
||||
|
||||
prompt_template_front = \
|
||||
"""
|
||||
Generate a full-body, front-view portrait of character {identifier} based on the following description, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. Gazing straight ahead. Standing with arms relaxed at sides. Natural expression.
|
||||
Features: {features}
|
||||
Style: {style}
|
||||
"""
|
||||
|
||||
prompt_template_side = \
|
||||
"""
|
||||
Generate a full-body, side-view portrait of character {identifier} based on the provided front-view portrait, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. Facing left. Standing with arms relaxed at sides.
|
||||
"""
|
||||
|
||||
prompt_template_back = \
|
||||
"""
|
||||
Generate a full-body, back-view portrait of character {identifier} based on the provided front-view portrait, with a pure white background. Use a wide 16:9 landscape canvas, not a vertical portrait canvas. The character should be centered in the image, occupying the middle of the wide frame with enough horizontal empty space. No facial features should be visible.
|
||||
"""
|
||||
|
||||
|
||||
class CharacterPortraitsGenerator:
|
||||
def __init__(
|
||||
self,
|
||||
image_generator,
|
||||
):
|
||||
self.image_generator = image_generator
|
||||
|
||||
|
||||
async def generate_front_portrait(
|
||||
self,
|
||||
character: CharacterInScene,
|
||||
style: str,
|
||||
) -> ImageOutput:
|
||||
features = "(static) " + character.static_features + "; (dynamic) " + character.dynamic_features
|
||||
prompt = prompt_template_front.format(
|
||||
identifier=character.identifier_in_scene,
|
||||
features=features,
|
||||
style=style,
|
||||
)
|
||||
image_output = await self.image_generator.generate_single_image(
|
||||
prompt=prompt,
|
||||
# size="512x512",
|
||||
)
|
||||
return image_output
|
||||
|
||||
async def generate_side_portrait(
|
||||
self,
|
||||
character: CharacterInScene,
|
||||
front_image_path: str,
|
||||
) -> ImageOutput:
|
||||
prompt = prompt_template_side.format(
|
||||
identifier=character.identifier_in_scene,
|
||||
)
|
||||
image_output = await self.image_generator.generate_single_image(
|
||||
prompt=prompt,
|
||||
reference_image_paths=[front_image_path],
|
||||
# size="1024x1024",
|
||||
)
|
||||
return image_output
|
||||
|
||||
|
||||
async def generate_back_portrait(
|
||||
self,
|
||||
character: CharacterInScene,
|
||||
front_image_path: str,
|
||||
) -> ImageOutput:
|
||||
prompt = prompt_template_back.format(
|
||||
identifier=character.identifier_in_scene,
|
||||
)
|
||||
image_output = await self.image_generator.generate_single_image(
|
||||
prompt=prompt,
|
||||
reference_image_paths=[front_image_path],
|
||||
# size="512x512",
|
||||
)
|
||||
return image_output
|
||||
@@ -0,0 +1,155 @@
|
||||
import os
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import List
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt
|
||||
|
||||
from interfaces import Event
|
||||
|
||||
system_prompt_template_extract_events = \
|
||||
"""
|
||||
You are a highly skilled Literary Analyst AI. Your expertise is in narrative structure, plot deconstruction, and thematic analysis. You meticulously read and interpret prose to break down a story into its fundamental sequential events.
|
||||
|
||||
**TASK**
|
||||
Extract the next event from the provided novel, following the sequence of the story and building upon the partially extracted events.
|
||||
|
||||
**INPUT**
|
||||
1. The full text of the novel, which is enclosed within <NOVEL_TEXT_START> and <NOVEL_TEXT_END> tags
|
||||
2. A sequence of already-extracted events (in order), which is enclosed within <EXTRACTED_EVENTS_START> and <EXTRACTED_EVENTS_END> tags. The sequence may be empty. Each event contains multiple processes and constitutes a complete causal chain.
|
||||
|
||||
Below is an example input:
|
||||
|
||||
<NOVEL_TEXT_START>
|
||||
The night was as dark as ink when the piercing alarm of the city museum suddenly shattered the silence. A thief, moving with phantom-like agility, had just pried open the display case and snatched the blue gem known as the "Heart of the Ocean" when the blaring alarm echoed through the hall.
|
||||
... (more novel text) ...
|
||||
<NOVEL_TEXT_END>
|
||||
|
||||
<EXTRACTED_EVENTS_START>
|
||||
<Event 0>
|
||||
Description: A thief who stole a gem from a museum was caught after a rooftop chase with guards, and the gem was recovered.
|
||||
Process Chain:
|
||||
- A thief steals a gem from a museum, triggering the alarm. Guards notice and begin the chase.
|
||||
- The thief rushes out the museum's back door and dashes through narrow alleys, with guards closely pursuing and calling for backup.
|
||||
- ... (more processes) ...
|
||||
|
||||
<Event 1>
|
||||
Description: ... (more description) ...
|
||||
Process Chain:
|
||||
- ... (more processes) ...
|
||||
|
||||
<EXTRACTED_EVENTS_END>
|
||||
|
||||
|
||||
**OUTPUT**
|
||||
{format_instructions}
|
||||
|
||||
**GUIDELINES**
|
||||
1. Focus on events that are critical to the plot, character development, or thematic depth.
|
||||
2. Ensure the event is logically distinct from previous and subsequent events.
|
||||
3. If the event spans multiple scenes, unify them under a single dramatic goal. For example, a chase sequence might begin in a city market, continue through back alleys, and conclude on a rooftop—all comprising a single event because they collectively achieve the dramatic purpose of "the protagonist evading capture."
|
||||
4. Maintain objectivity: describe events based on the text without interpretation or judgment.
|
||||
5. For the process field, provide a detailed, step-by-step account of the event's progression, including key actions, decisions, and turning points. Each step should be clear and concise, illustrating how the event unfolds over time.
|
||||
Below is an example:
|
||||
Timeframe: The following morning, after acquiring the information about the Temple.
|
||||
Characters: Elara (protagonist) and Kaelen (her rival treasure hunter).
|
||||
Cause: Both seek the same artifact and are determined to reach it first.
|
||||
Process: The event begins with Elara hastily purchasing supplies in the port town (scene 1), where she spots Kaelen already hiring a crew, raising the stakes. It continues as she races to secure her own ship and captain, negotiating fiercely under time pressure (scene 2). The event culminates in a direct confrontation on the docks (scene 3), where Kaelen attempts to sabotage her vessel, leading to a brief but intense sword fight between the two rivals.
|
||||
Outcome: Elara successfully defends her ship and sets sail, but the conflict solidifies a bitter personal rivalry with Kaelen, ensuring their race to the temple will be fraught with direct opposition and danger.
|
||||
6. Every detail in your event description must be directly supported by the input novel. Do not add, assume, or invent any information.
|
||||
7. The language of outputs in values should be same as the input text.
|
||||
"""
|
||||
|
||||
human_prompt_template_extract_next_event = \
|
||||
"""
|
||||
<NOVEL_TEXT_START>
|
||||
{novel_text}
|
||||
<NOVEL_TEXT_END>
|
||||
|
||||
<EXTRACTED_EVENTS_START>
|
||||
{extracted_events}
|
||||
<EXTRACTED_EVENTS_END>
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class EventExtractor:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
chat_model: str,
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
model_provider="openai",
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
self.parser = PydanticOutputParser(pydantic_object=Event)
|
||||
|
||||
|
||||
# Cap on extracted events: is_last is asserted by the LLM only, so without a
|
||||
# bound a model that never sets it would loop (and spend tokens) forever.
|
||||
max_events = 50
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
novel_text: str,
|
||||
):
|
||||
logging.info("Extracting events from novel...")
|
||||
|
||||
events = []
|
||||
while True:
|
||||
if len(events) >= self.max_events:
|
||||
raise RuntimeError(
|
||||
f"Event extraction exceeded the maximum of {self.max_events} events "
|
||||
"without an is_last marker; aborting to avoid unbounded LLM calls."
|
||||
)
|
||||
event = self.extract_next_event(novel_text, events)
|
||||
|
||||
events.append(event)
|
||||
logging.info(f"Extracted event: \n{event}")
|
||||
if event.is_last:
|
||||
break
|
||||
|
||||
return events
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=lambda retry_state: logging.warning(f"Retrying extract_next_event due to error: {retry_state.outcome.exception()}"),
|
||||
)
|
||||
def extract_next_event(
|
||||
self,
|
||||
novel_text: str,
|
||||
extracted_events: List[Event]
|
||||
) -> Event:
|
||||
|
||||
extracted_events_str = "\n\n".join([str(e) for e in extracted_events])
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_extract_events.format(format_instructions=self.parser.get_format_instructions()),
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_extract_next_event.format(
|
||||
novel_text=novel_text,
|
||||
extracted_events=extracted_events_str,
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
chain = self.chat_model | self.parser
|
||||
|
||||
event: Event = chain.invoke(messages)
|
||||
|
||||
assert event.index == len(extracted_events), f"Extracted event index {event.index} does not match the expected index {len(extracted_events)}"
|
||||
|
||||
return event
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
import os
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import List, Tuple, Dict, Optional
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from langchain.output_parsers import PydanticOutputParser
|
||||
from interfaces import Event, Scene
|
||||
from interfaces import CharacterInScene, CharacterInEvent, CharacterInNovel
|
||||
from tenacity import retry, stop_after_attempt
|
||||
|
||||
|
||||
system_prompt_template_merge_characters_across_scenes_in_event = \
|
||||
"""
|
||||
You are an expert script analysis and character fusion specialist. Your role is to intelligently analyze multiple script scenes, identify characters that represent the same entity across different scenes, and merge them into a unified character list with consistent identifiers.
|
||||
|
||||
**TASK**
|
||||
Process the input scenes, each containing a script and characters with their names and features. Identify and merge characters that are logically the same across scenes, even if they have different names or slight variations in description. Output a consolidated list of characters for the entire event. Each character in the list must have a unique identifier, along with the scene numbers where they appear and the name used in each scene. You also need to aggregate the static features of the same characters together.
|
||||
|
||||
**INPUT**
|
||||
A sequence of scenes. Each scene is enclosed within <SCENE_N_START> and <SCENE_N_END> tags, where N is the scene number(starting from 0).
|
||||
Each scene includes a screnplay script and a sequence of character names.
|
||||
The screenplay script is enclosed within <SCRIPT_START> and <SCRIPT_END> tags.
|
||||
The sequence of character is enclosed within <CHARACTERS_START> and <CHARACTERS_END> tags. Each character in the list is enclosed within <CHARACTER_M_START> and <CHARACTER_M_END> tags, where M is the character number(starting from 0).
|
||||
|
||||
Below is an example of one scene:
|
||||
|
||||
<SCENE_0_START>
|
||||
|
||||
<SCRIPT_START>
|
||||
John enters the room and sees Mary.
|
||||
John: Hi Mary, how are you?
|
||||
Mary: I'm good, John. Thanks for asking!
|
||||
<SCRIPT_END>
|
||||
|
||||
<CHARACTERS_START>
|
||||
|
||||
<CHARACTER_0_START>
|
||||
John [visible]
|
||||
static features: John is a tall man with short black hair and brown eyes.
|
||||
dynamic features: Wearing a blue shirt and black pants.
|
||||
<CHARACTER_0_END>
|
||||
|
||||
<CHARACTER_1_START>
|
||||
Mary [visible]
|
||||
static features: Mary is a young woman with long brown hair and green eyes.
|
||||
dynamic features: Wearing a floral dress and a denim jacket.
|
||||
<CHARACTER_1_END>
|
||||
|
||||
<CHARACTERS_END>
|
||||
|
||||
<SCENE_0_END>
|
||||
|
||||
|
||||
|
||||
**OUTPUT**
|
||||
{format_instructions}
|
||||
|
||||
**GUIDELINES**
|
||||
1. Character Fusion: Analyze contextual clues (e.g., dialogue style, role in plot, relationships, descriptions) to determine if characters from different scenes are the same person, even if names vary.
|
||||
2. Unique Identifier: Assign a consistent, unique ID (e.g., primary/canonical name) to each merged character. Use the most frequent or contextually appropriate name as the identifier, if possible.
|
||||
3. Scene Mapping: For each character, list all scenes they appear in and the exact name used in each scene.
|
||||
4. Completeness: Ensure all characters from all scenes are included in the final list. No duplicate, omitted, or extraneous characters.
|
||||
5. If a character undergoes significant changes across different scenes, it is necessary to split them into separate roles. For example, if Character A is a child in Scene 0 but an adult in Scene 1, they should be divided into two distinct characters (meaning two different actors are required to portray them).
|
||||
6. The language of outputs in values should be same as the input text.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_merge_characters_across_scenes_in_event = \
|
||||
"""
|
||||
{scenes_sequence}
|
||||
"""
|
||||
|
||||
class MergeCharactersAcrossScenesInEventResponse(BaseModel):
|
||||
characters: List[CharacterInEvent] = Field(
|
||||
description="List of merged characters with their identifiers",
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
system_prompt_template_merge_characters_to_existing_characters_in_novel = \
|
||||
"""
|
||||
You are an information integration expert skilled in accurately identifying, matching, and merging character information. Your responsibility is to ensure consistency in character attributes and efficiently maintain and update the global character list.
|
||||
|
||||
**TASK**
|
||||
Merge the character list extracted from the current event (which may include new or existing characters) into the global character list. For existing characters, ensure their feature descriptions remain consistent; for new characters, add them to the global list.
|
||||
|
||||
**INPUT**
|
||||
1. Existing Characters in the Novel: A list of characters already present in the novel, each with a unique index, identifier, and static features. The list is enclosed within <EXISTING_CHARACTERS_START> and <EXISTING_CHARACTERS_END> tags. Each character in the list is enclosed within <CHARACTER_P_START> and <CHARACTER_P_END> tags, where P is the character number(starting from 0).
|
||||
2. Characters in the Current Event: A list of characters identified in the current event, each with an index, identifier, active scenes, and static features. The list is enclosed within <EVENT_CHARACTERS_START> and <EVENT_CHARACTERS_END> tags. Each character in the list is enclosed within <CHARACTER_Q_START> and <CHARACTER_Q_END> tags, where Q is the character number(starting from 0).
|
||||
|
||||
|
||||
**OUTPUT**
|
||||
{format_instructions}
|
||||
|
||||
**GUIDELINES**
|
||||
1. Feature Consistency: Strictly compare the features of the current event characters with those of existing characters. Some character's identifier may be the same as existing role identifier, but their features differ, such as youth and old age. You need to distinguish them as two separate characters.
|
||||
2. Efficient Merging: Avoid duplicate characters to ensure the list remains concise.
|
||||
3. Feature Update: If an existing character's features are expanded or modified based on new information from the current event, update their description accordingly.
|
||||
"""
|
||||
|
||||
human_prompt_template_merge_characters_to_existing_characters_in_novel = \
|
||||
"""
|
||||
<EXISTING_CHARACTERS_START>
|
||||
{existing_characters_in_novel}
|
||||
<EXISTING_CHARACTERS_END>
|
||||
|
||||
<EVENT_CHARACTERS_START>
|
||||
{characters_in_event}
|
||||
<EVENT_CHARACTERS_END>
|
||||
"""
|
||||
|
||||
|
||||
class CharacterForMergingToNovel(BaseModel):
|
||||
index_in_event: int = Field(
|
||||
description="The index of the character in the list of characters in the current event.",
|
||||
examples=[0, 1, 2],
|
||||
)
|
||||
index_in_novel: int = Field(
|
||||
description="The index of the character in the list of existing characters in the novel. If this is a new character, set it to -1.",
|
||||
examples=[0, 7, -1],
|
||||
)
|
||||
identifier_in_novel: str = Field(
|
||||
description="The unique identifier for the character in the novel. If this is a new character, ensure the name does not conflict with existing characters. If this is not a new character, this should match the identifier in the existing characters list.",
|
||||
examples=["Alice", "Bob the Builder"],
|
||||
)
|
||||
modified_features: str = Field(
|
||||
description="The modified static features of the character after merging. If the character is new, this should be the full static features. If the character is existing and their features are expanded or modified, this should be filled in the complete modified features. If the character is existing and their features remain unchanged, this should be the same as the existing character's static features.",
|
||||
)
|
||||
|
||||
class MergeCharactersToExistingCharactersInNovelResponse(BaseModel):
|
||||
characters: List[CharacterForMergingToNovel] = Field(
|
||||
description="List of characters in the event with their corresponding index in the existing characters in the novel. If the character is new, the index_in_novel should be -1. The number of characters in this list should be the same as the number of characters in the event.",
|
||||
)
|
||||
|
||||
|
||||
|
||||
class GlobalInformationPlanner:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
chat_model: str,
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
model_provider="openai",
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=lambda retry_state: logging.warning(f"Retrying due to {retry_state.outcome.exception()}"),
|
||||
)
|
||||
async def merge_characters_across_scenes_in_event(
|
||||
self,
|
||||
event_idx: int,
|
||||
scenes: List[Scene], # Scene.characters is List[CharacterInScene]
|
||||
) -> List[CharacterInEvent]:
|
||||
scenes_sequence_str = ""
|
||||
for scene in scenes:
|
||||
scene_str = f"<SCENE_{scene.idx}_START>\n"
|
||||
scene_str += "<SCRIPT_START>\n"
|
||||
scene_str += scene.script + "\n"
|
||||
scene_str += "<SCRIPT_END>\n\n"
|
||||
scene_str += "<CHARACTERS_START>\n"
|
||||
for character in scene.characters:
|
||||
scene_str += f"<CHARACTER_{character.idx}_START>\n"
|
||||
scene_str += str(character)
|
||||
scene_str += f"<CHARACTER_{character.idx}_END>\n"
|
||||
scene_str += "<CHARACTERS_END>\n"
|
||||
scene_str += f"<SCENE_{scene.idx}_END>\n"
|
||||
scenes_sequence_str += scene_str
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=MergeCharactersAcrossScenesInEventResponse)
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_merge_characters_across_scenes_in_event.format(
|
||||
format_instructions=parser.get_format_instructions(),
|
||||
),
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_merge_characters_across_scenes_in_event.format(
|
||||
scenes_sequence=scenes_sequence_str,
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
response: MergeCharactersAcrossScenesInEventResponse = await chain.ainvoke(messages)
|
||||
characters_in_event = response.characters
|
||||
|
||||
# check the output is valid
|
||||
flags = [{c.identifier_in_scene: False for c in s.characters} for s in scenes]
|
||||
|
||||
# check if all character identifiers can be found in the scenes
|
||||
for character in characters_in_event:
|
||||
for scene_idx, identifier_in_scene in character.active_scenes.items():
|
||||
if identifier_in_scene not in [c.identifier_in_scene for c in scenes[scene_idx].characters]:
|
||||
raise ValueError(f"Character {identifier_in_scene} not found in scene {scene_idx} of event {event_idx}")
|
||||
else:
|
||||
flags[scene_idx][identifier_in_scene] = True
|
||||
|
||||
# check if all characters are included
|
||||
for scene_idx, flag in enumerate(flags):
|
||||
for identifier_in_scene, included in flag.items():
|
||||
if not included:
|
||||
raise ValueError(f"Character {identifier_in_scene} in scene {scene_idx} of event {event_idx} not included in the merged characters")
|
||||
|
||||
return characters_in_event
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=lambda retry_state: logging.warning(f"Retrying due to {retry_state.outcome.exception()}"),
|
||||
)
|
||||
def merge_characters_to_existing_characters_in_novel(
|
||||
self,
|
||||
event_idx: int,
|
||||
existing_characters_in_novel: List[CharacterInNovel],
|
||||
characters_in_event: List[CharacterInEvent],
|
||||
) -> List[CharacterInNovel]:
|
||||
existing_characters_str = ""
|
||||
for character in existing_characters_in_novel:
|
||||
existing_characters_str += f"<CHARACTER_{character.index}_START>\n"
|
||||
existing_characters_str += str(character)
|
||||
existing_characters_str += f"<CHARACTER_{character.index}_END>\n"
|
||||
|
||||
characters_in_event_str = ""
|
||||
for character in characters_in_event:
|
||||
characters_in_event_str += f"<CHARACTER_{character.index}_START>\n"
|
||||
characters_in_event_str += character.identifier_in_event + "\n"
|
||||
characters_in_event_str += "Static features: " + character.static_features + "\n"
|
||||
characters_in_event_str += f"<CHARACTER_{character.index}_END>\n"
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=MergeCharactersToExistingCharactersInNovelResponse)
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_merge_characters_to_existing_characters_in_novel.format(
|
||||
format_instructions=parser.get_format_instructions(),
|
||||
),
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_merge_characters_to_existing_characters_in_novel.format(
|
||||
existing_characters_in_novel=existing_characters_str,
|
||||
characters_in_event=characters_in_event_str,
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
response: MergeCharactersToExistingCharactersInNovelResponse = chain.invoke(messages)
|
||||
|
||||
for character in response.characters:
|
||||
if character.index_in_novel == -1:
|
||||
# new character, add to existing characters
|
||||
new_character = CharacterInNovel(
|
||||
index=len(existing_characters_in_novel),
|
||||
identifier_in_novel=character.identifier_in_novel,
|
||||
static_features=character.modified_features,
|
||||
active_events={event_idx: characters_in_event[character.index_in_event].identifier_in_event},
|
||||
)
|
||||
existing_characters_in_novel.append(new_character)
|
||||
else:
|
||||
existing_characters_in_novel[character.index_in_novel].static_features = character.modified_features
|
||||
existing_characters_in_novel[character.index_in_novel].active_events.update({event_idx: characters_in_event[character.index_in_event].identifier_in_event})
|
||||
|
||||
return existing_characters_in_novel
|
||||
|
||||
|
||||
# # TODO: 如果是长篇小说，事件太多，很容易报错，出场的角色会分不清在哪个事件里，也很容易漏，需要想办法解决
|
||||
# @retry(
|
||||
# stop=stop_after_attempt(3),
|
||||
# after=lambda retry_state: logging.warning(f"Retrying due to {retry_state.outcome.exception()}"),
|
||||
# )
|
||||
# def merge_characters_across_events_in_novel(
|
||||
# self,
|
||||
# events: List[Event],
|
||||
# characters_in_event: List[List[CharacterInEvent]],
|
||||
# ) -> List[CharacterInNovelWithoutStaticFeatures]:
|
||||
# events_sequence_str = ""
|
||||
# for event, characters in zip(events, characters_in_event):
|
||||
# event_str = f"<EVENT_{event.index}_START>\n\n"
|
||||
# event_str += "<DESCRIPTION_START>\n"
|
||||
# event_str += event.description + "\n"
|
||||
# event_str += "<DESCRIPTION_END>\n\n"
|
||||
# event_str += "<PROCESS_CHAIN_START>\n"
|
||||
# for process in event.process_chain:
|
||||
# event_str += process + "\n"
|
||||
# event_str += "<PROCESS_CHAIN_END>\n\n"
|
||||
# event_str += "<CHARACTERS_START>\n"
|
||||
# for i, character in enumerate(characters):
|
||||
# event_str += f"<CHARACTER_{i}_START>{character.identifier_in_event}<CHARACTER_{i}_END>\n"
|
||||
# event_str += "<CHARACTERS_END>\n\n"
|
||||
# event_str += f"<EVENT_{event.index}_END>\n\n"
|
||||
# events_sequence_str += event_str
|
||||
|
||||
# parser = PydanticOutputParser(pydantic_object=MergeCharactersAcrossEventsInNovelResponse)
|
||||
|
||||
# messages = [
|
||||
# SystemMessage(
|
||||
# content=system_prompt_template_merge_characters_across_events.format(
|
||||
# format_instructions=parser.get_format_instructions(),
|
||||
# ),
|
||||
# ),
|
||||
# HumanMessage(
|
||||
# content=human_prompt_template_merge_characters_across_events.format(
|
||||
# events_sequence=events_sequence_str,
|
||||
# )
|
||||
# )
|
||||
# ]
|
||||
|
||||
# chain = self.chat_model | parser
|
||||
# response: MergeCharactersAcrossEventsInNovelResponse = chain.invoke(messages)
|
||||
# characters_in_novel = response.characters
|
||||
|
||||
# # check the output is valid
|
||||
# flags = [{c.identifier_in_event: False for c in characters} for characters in characters_in_event]
|
||||
|
||||
# # check if all character identifiers can be found in the events
|
||||
# for character in characters_in_novel:
|
||||
# for event_idx, identifier_in_event in character.active_events.items():
|
||||
# if identifier_in_event not in [c.identifier_in_event for c in characters_in_event[event_idx]]:
|
||||
# raise ValueError(f"Character {identifier_in_event} not found in event {event_idx}")
|
||||
# else:
|
||||
# flags[event_idx][identifier_in_event] = True
|
||||
|
||||
# # check if all characters are included
|
||||
# # for event_idx, flag in enumerate(flags):
|
||||
# # for identifier_in_event, included in flag.items():
|
||||
# # if not included:
|
||||
# # raise ValueError(f"Character {identifier_in_event} in event {event_idx} not included in the merged characters")
|
||||
|
||||
# return characters_in_novel
|
||||
|
||||
|
||||
|
||||
# async def extract_static_feature_for_character_in_novel(
|
||||
# self,
|
||||
# relevant_chunks: List[str],
|
||||
# character: CharacterInNovelWithoutStaticFeatures,
|
||||
# ) -> str:
|
||||
# context_fragments_str = ""
|
||||
# for i, chunk in enumerate(relevant_chunks):
|
||||
# context_fragments_str += f"<CONTEXT_FRAGMENT_{i}_START>\n"
|
||||
# context_fragments_str += chunk + "\n"
|
||||
# context_fragments_str += f"<CONTEXT_FRAGMENT_{i}_END>\n"
|
||||
|
||||
# parser = None # no need to parse the output, just return the text
|
||||
|
||||
# messages = [
|
||||
# SystemMessage(
|
||||
# content=system_prompt_template_extract_static_feature_for_character_in_novel,
|
||||
# ),
|
||||
# HumanMessage(
|
||||
# content=human_prompt_template_extract_static_feature_for_character_in_novel.format(
|
||||
# character_name=character.identifier_in_novel,
|
||||
# context_fragments=context_fragments_str,
|
||||
# )
|
||||
# )
|
||||
# ]
|
||||
|
||||
# base_features = await self.chat_model.ainvoke(messages)
|
||||
# return base_features.content
|
||||
@@ -0,0 +1,171 @@
|
||||
import os
|
||||
import logging
|
||||
import asyncio
|
||||
from typing import List, Tuple
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain.chat_models import init_chat_model
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
|
||||
|
||||
|
||||
system_prompt_template_compress_novel_chunk = \
|
||||
"""
|
||||
You are an expert text compression assistant specialized in literary content. Your goal is to condense novels or story excerpts while preserving core narrative elements, key details, character development, and plot coherence.
|
||||
|
||||
|
||||
**TASK**
|
||||
Compress the provided input text to reduce its length significantly, eliminating redundancies, overly descriptive passages, and minor details—but without losing essential story arcs, dialogue, or emotional impact. Aim for clarity and readability in the compressed output.
|
||||
|
||||
|
||||
**INPUT**
|
||||
A segment of a novel (possibly truncated due to context length constraints). It is enclosed within <NOVEL_CHUNK_START> and <NOVEL_CHUNK_END> tags.
|
||||
|
||||
|
||||
**OUTPUT**
|
||||
A compressed version of the input text, retaining the core narrative, critical events, and character interactions.
|
||||
|
||||
**GUIDELINES**
|
||||
1. Fidelity to the Plot: Absolutely preserve all major plot points, twists, revelations, and the sequence of key events. Do not omit crucial story elements.
|
||||
2. Character Consistency: Maintain character actions, decisions, and development. Important dialogue that reveals plot or character can be condensed or paraphrased but its meaning must be kept intact.
|
||||
3. Streamline Description: Reduce lengthy descriptions of settings, characters, or objects to their most essential and evocative elements. Capture the mood and critical details without the elaborate prose.
|
||||
4. Condense Internal Monologue: Paraphrase characters' extended internal thoughts and reflections, focusing on the key realizations or decisions they lead to.
|
||||
5. Simplify Language: Use more direct and concise language. Combine sentences, eliminate redundant adverbs and adjectives, and avoid repetitive phrasing.
|
||||
6. Cohesion and Flow: Ensure the compressed text is smooth, readable, and maintains a logical narrative flow. It should not feel like a fragmented list of events.
|
||||
7. Discard any non-narrative text (e.g., "Please follow my account!", "Background setting:...", personal opinions).
|
||||
8. Produce a seamless paragraph (or paragraphs if necessary) without markers (e.g., "Chapter 1") or section breaks.
|
||||
9. The language of output should be consistent with the original text.
|
||||
"""
|
||||
|
||||
human_prompt_template_compress_novel_chunk = \
|
||||
"""
|
||||
<NOVEL_CHUNK_START>
|
||||
{novel_chunk}
|
||||
<NOVEL_CHUNK_END>
|
||||
"""
|
||||
|
||||
|
||||
system_prompt_template_aggregate = \
|
||||
"""
|
||||
You are a professional text processing assistant specializing in the aggregation and refinement of segmented text chunks. Your expertise lies in seamlessly merging sequential text fragments while intelligently handling overlapping or duplicated content expressed in different ways.
|
||||
|
||||
**TASK**
|
||||
Aggregate the provided text chunks into a coherent and continuous short story. Carefully identify and resolve overlaps where the end of one chunk and the beginning of the next chunk contain semantically similar content but with different expressions. Remove redundant repetitions while preserving the original meaning, style, and flow of the text. Ensure all non-overlapping content remains unchanged and intact.
|
||||
|
||||
|
||||
**INPUT**
|
||||
A sequence of text chunks (ordered from first to last), where each chunk may have an overlapping segment with the next chunk. The overlapping segments might vary in wording but convey similar meaning. Each chunk is enclosed within <CHUNK_N_START> and <CHUNK_N_END> tags, where N is the chunk index starting from 0.
|
||||
|
||||
**OUTPUT**
|
||||
A single, consolidated text of the short story without unnatural repetitions or disruptions. The output should maintain the original narrative structure, tone, and details, with smooth transitions between originally adjacent chunks.
|
||||
|
||||
**GUIDELINES**
|
||||
1. Analyze the input chunks sequentially. For each adjacent pair (e.g., Chunk N and Chunk N+1), compare the end of Chunk N and the beginning of Chunk N+1 to detect overlapping content.
|
||||
2. If the overlapping segments are semantically equivalent but phrased differently, merge them by retaining the most natural or contextually appropriate version (prioritize the version from the later chunk if both are equally valid, but avoid introducing inconsistency).
|
||||
3. If the overlapping segments are not perfectly equivalent (e.g., one contains additional details), integrate the meaningful information without duplication, ensuring no loss of content.
|
||||
4. Preserve all non-overlapping text exactly as it appears in the original chunks. Do not modify, paraphrase, or omit any unique content.
|
||||
5. Ensure the merged text is fluent and coherent, without abrupt jumps or redundant phrases.
|
||||
6. If no overlap is detected between two chunks, concatenate them directly without changes.
|
||||
7. Do not invent new content or alter the original narrative beyond handling the overlaps.
|
||||
8. The language of output should be consistent with the original text.
|
||||
"""
|
||||
|
||||
human_prompt_template_aggregate = \
|
||||
"""
|
||||
{chunks}
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
class NovelCompressor:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
chat_model: str,
|
||||
chunk_size: int = 65536,
|
||||
chunk_overlap: int = 8192,
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
model_provider="openai",
|
||||
)
|
||||
|
||||
self.splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
)
|
||||
|
||||
|
||||
def split(
|
||||
self,
|
||||
novel_text: str,
|
||||
):
|
||||
novel_chunks = self.splitter.split_text(novel_text)
|
||||
return novel_chunks
|
||||
|
||||
|
||||
async def compress(
|
||||
self,
|
||||
index_chunk_pairs: List[Tuple[int, str]],
|
||||
max_concurrent_tasks: int = 5,
|
||||
) -> str:
|
||||
sem = asyncio.Semaphore(max_concurrent_tasks)
|
||||
|
||||
tasks = [
|
||||
self.compress_single_novel_chunk(sem, index, novel_chunk)
|
||||
for index, novel_chunk in index_chunk_pairs
|
||||
]
|
||||
compressed_novel_chunks = await asyncio.gather(*tasks)
|
||||
return compressed_novel_chunks
|
||||
|
||||
|
||||
async def compress_single_novel_chunk(
|
||||
self,
|
||||
semaphore: asyncio.Semaphore,
|
||||
index,
|
||||
novel_chunk: str,
|
||||
) -> str:
|
||||
async with semaphore:
|
||||
logging.info(f"Compressing novel chunk {index}")
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_compress_novel_chunk
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_compress_novel_chunk.format(
|
||||
novel_chunk=novel_chunk
|
||||
)
|
||||
),
|
||||
]
|
||||
response = await self.chat_model.ainvoke(messages)
|
||||
compressed_novel_chunk = response.content
|
||||
logging.info(f"Compressed novel chunk {index}")
|
||||
return index, compressed_novel_chunk
|
||||
|
||||
|
||||
def aggregate(
|
||||
self,
|
||||
compressed_novel_chunks: List[str],
|
||||
):
|
||||
chunks_str = "\n".join([
|
||||
f"<CHUNK_{i}_START>\n{chunk}\n<CHUNK_{i}_END>"
|
||||
for i, chunk in enumerate(compressed_novel_chunks)
|
||||
])
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_aggregate
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_aggregate.format(
|
||||
chunks=chunks_str
|
||||
)
|
||||
),
|
||||
]
|
||||
response = self.chat_model.invoke(messages)
|
||||
aggregated_novel = response.content
|
||||
return aggregated_novel
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
from tenacity import retry, stop_after_attempt
|
||||
from pydantic import BaseModel, Field
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from utils.image import image_path_to_b64
|
||||
|
||||
from utils.retry import after_func
|
||||
|
||||
system_prompt_template_select_reference_images_only_text = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional visual creation assistant skilled in multimodal image analysis and reasoning.
|
||||
|
||||
[Task]
|
||||
Your core task is to intelligently select the most suitable reference images from a provided set of reference image descriptions (including multiple character reference images and existing scene images from prior frames) based on the user's text description (describing the target frame), ensuring that the subsequently generated image meets the following key consistencies:
|
||||
- Character Consistency: The appearance (e.g. gender, ethnicity, age, facial features, hairstyle, body shape), clothing, expression, posture, etc., of the generated character should highly match the reference image descriptions.
|
||||
- Environmental Consistency: The scene of the generated image (e.g., background, lighting, atmosphere, layout) should remain coherent with the existing image descriptions from prior frames.
|
||||
- Style Consistency: The visual style of the generated image (e.g., realistic, cartoon, film-like, color tone) should harmonize with the reference image descriptions.
|
||||
|
||||
[Input]
|
||||
You will receive a text description of the target frame, along with a sequence of reference image descriptions.
|
||||
- The text description of the target frame is enclosed within <FRAME_DESC> and </FRAME_DESC>.
|
||||
- The sequence of reference image descriptions is enclosed within <SEQ_DESC> and </SEQ_DESC>. Each description is prefixed with its index, starting from 0.
|
||||
|
||||
Below is an example of the input format:
|
||||
<FRAME_DESC>
|
||||
[Camera 1] Shot from Alice's over-the-shoulder perspective. Alice is on the side closer to the camera, with only her shoulder appearing in the lower left corner of the frame. Bob is on the side farther from the camera, positioned slightly right of center in the frame. Bob's expression shifts from surprise to delight as he recognizes Alice.
|
||||
</FRAME_DESC>
|
||||
|
||||
<SEQ_DESC>
|
||||
Image 0: A front-view portrait of Alice.
|
||||
Image 1: A front-view portrait of Bob.
|
||||
Image 2: [Camera 0] Medium shot of the supermarket aisle. Alice and Bob are shown in profile facing the right side of the frame. Bob is on the right side of the frame, and Alice is on the left side. Alice, looking down and pushing a shopping cart, follows closely behind Bob and accidentally bumps into his heel.
|
||||
Image 3: [Camera 1] Shot from Alice's over-the-shoulder perspective. Alice is on the side closer to the camera, with only her shoulder appearing in the lower left corner of the frame. Bob is on the side farther from the camera, positioned slightly right of center in the frame. Bob quickly turns around, and his expression shifts from neutral to surprised.
|
||||
Image 4: [Camera 2] Shot from Bob's over-the-shoulder perspective. Bob is on the side closer to the camera, with only his shoulder appearing in the lower right corner of the frame. Alice is on the side farther from the camera, positioned slightly left of center in the frame. Alice looks down, then up as she prepares to apologize. Upon realizing it's someone familiar, her expression shifts to one of surprise.
|
||||
</SEQ_DESC>
|
||||
|
||||
|
||||
[Output]
|
||||
You need to select up to 8 of the most relevant reference images based on the user's description and put the corresponding indices in the ref_image_indices field of the output. At the same time, you should generate a text prompt that describes the image to be created, specifying which elements in the generated image should reference which image description (and which elements within it).
|
||||
|
||||
{format_instructions}
|
||||
|
||||
|
||||
[Guidelines]
|
||||
- Ensure that the language of all output values (not include keys) matches that used in the frame description.
|
||||
- The reference image descriptions may depict the same character from different angles, in different outfits, or in different scenes. Identify the description closest to the version described by the user
|
||||
- Prioritize image descriptions with similar compositions, i.e., shots taken by the same camera.
|
||||
- The images from prior frames are arranged in chronological order. Give higher priority to more recent images (those closer to the end of the sequence).
|
||||
- Choose reference image descriptions that are as concise as possible and avoid including duplicate information. For example, if Image 3 depicts the facial features of Bob from the front, and Image 1 also depicts Bob's facial features from the front-view portrait, then Image 1 is redundant and should not be selected.
|
||||
- When a new character appears in the frame description, prioritize selecting their portrait image description (if available) to ensure accurate depiction of their appearance. Pay attention to whether the character is facing the camera from the front, side, or back. Choose the most suitable view as the reference image for the character.
|
||||
- For character portraits, you can only select at most one image from multiple views (front, side, back). Choose the most appropriate one based on the frame description. For example, when depicting a character from the side, choose the side view of the character.
|
||||
- Select at most **8** optimal reference image descriptions.
|
||||
"""
|
||||
|
||||
|
||||
system_prompt_template_select_reference_images_multimodal = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional visual creation assistant skilled in multimodal image analysis and reasoning.
|
||||
|
||||
[Task]
|
||||
Your core task is to intelligently select the most suitable reference images from a provided reference image library (including multiple character reference images and existing scene images from prior frames) based on the user's text description (describing the target frame), ensuring that the subsequently generated image meets the following key consistencies:
|
||||
- Character Consistency: The appearance (e.g. gender, ethnicity, age, facial features, hairstyle, body shape), clothing, expression, posture, etc., of the generated character should highly match the reference images.
|
||||
- Environmental Consistency: The scene of the generated image (e.g., background, lighting, atmosphere, layout) should remain coherent with the existing images from prior frames.
|
||||
- Style Consistency: The visual style of the generated image (e.g., realistic, cartoon, film-like, color tone) should harmonize with the reference images and existing images.
|
||||
|
||||
[Input]
|
||||
You will receive a text description of the target frame, along with a sequence of reference images.
|
||||
- The text description of the target frame is enclosed within <FRAME_DESC> and </FRAME_DESC>.
|
||||
- The sequence of reference images is enclosed within <SEQ_IMAGES> and </SEQ_IMAGES>. Each reference image is provided with a text description. The reference images are indexed starting from 0.
|
||||
|
||||
Below is an example of the input format:
|
||||
<FRAME_DESC>
|
||||
[Camera 1] Shot from Alice's over-the-shoulder perspective. <Alice> is on the side closer to the camera, with only her shoulder appearing in the lower left corner of the frame. <Bob> is on the side farther from the camera, positioned slightly right of center in the frame. <Bob>'s expression shifts from surprise to delight as he recognizes <Alice>.
|
||||
</FRAME_DESC>
|
||||
|
||||
<SEQ_IMAGES>
|
||||
Image 0: A front-view portrait of Alice.
|
||||
[Image 0 here]
|
||||
Image 1: A front-view portrait of Bob.
|
||||
[Image 1 here]
|
||||
Image 2: [Camera 0] Medium shot of the supermarket aisle. Alice and Bob are shown in profile facing the right side of the frame. Bob is on the right side of the frame, and Alice is on the left side. Alice, looking down and pushing a shopping cart, follows closely behind Bob and accidentally bumps into his heel.
|
||||
[Image 2 here]
|
||||
Image 3: [Camera 1] Shot from Alice's over-the-shoulder perspective. Alice is on the side closer to the camera, with only her shoulder appearing in the lower left corner of the frame. Bob is on the side farther from the camera, positioned slightly right of center in the frame. Bob is back to the camera.
|
||||
[Image 3 here]
|
||||
Image 4: [Camera 2] Shot from Bob's over-the-shoulder perspective. Bob is on the side closer to the camera, with only his shoulder appearing in the lower right corner of the frame. Alice is on the side farther from the camera, positioned slightly left of center in the frame. Alice looks down, then up as she prepares to apologize. Upon realizing it's someone familiar, her expression shifts to one of surprise.
|
||||
</SEQ_IMAGES>
|
||||
|
||||
[Output]
|
||||
You need to select the most relevant reference images based on the user's description and put the corresponding indices in the `ref_image_indices` field of the output. At the same time, you should generate a text prompt that describes the image to be created, specifying which elements in the generated image should reference which image (and which elements within it).
|
||||
|
||||
{format_instructions}
|
||||
|
||||
|
||||
[Guidelines]
|
||||
- Ensure that the language of all output values (not include keys) matches that used in the frame description.
|
||||
- The reference image descriptions may depict the same character from different angles, in different outfits, or in different scenes. Identify the description closest to the version described by the user
|
||||
- Prioritize image descriptions with similar compositions, i.e., shots taken by the same camera.
|
||||
- The images from prior frames are arranged in chronological order. Give higher priority to more recent images (those closer to the end of the sequence).
|
||||
- Choose reference image descriptions that are as concise as possible and avoid including duplicate information. For example, if Image 3 depicts the facial features of Bob from the front, and Image 1 also depicts Bob's facial features from the front-view portrait, then Image 1 is redundant and should not be selected.
|
||||
- For character portraits, you can only select at most one image from multiple views (front, side, back). Choose the most appropriate one based on the frame description. For example, when depicting a character from the side, choose the side view of the character.
|
||||
- Select at most **8** optimal reference image descriptions.
|
||||
- The text guiding image editing should be as concise as possible.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_select_reference_images = \
|
||||
"""
|
||||
<FRAME_DESC>
|
||||
{frame_description}
|
||||
</FRAME_DESC>
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
class RefImageIndicesAndTextPrompt(BaseModel):
|
||||
ref_image_indices: List[int] = Field(
|
||||
description="Indices of reference images selected from the provided images. For example, [0, 2, 5] means selecting the first, third, and sixth images. The indices should be 0-based.",
|
||||
examples=[
|
||||
[1, 3]
|
||||
]
|
||||
)
|
||||
text_prompt: str = Field(
|
||||
description="Text description to guide the image generation. You need to describe the image to be generated, specifying which elements in the generated image should reference which image (and which elements within it). For example, 'Create an image following the given description: \nThe man is standing in the landscape. The man should reference Image 0. The landscape should reference Image 1.' Here, the index of the reference image should refer to its position in the ref_image_indices list, not the sequence number in the provided image list. Refer to the reference image must be in the format of Image N. Do not use any other word except Image.",
|
||||
examples=[
|
||||
"Create an image based on the following guidance: \n Make modifications based on Image 1: Bob's body turns to face the camera, while all other elements remain unchanged. Bob's appearance should refer to Image 0.",
|
||||
"Create an image following the given description: \nThe man is standing in the landscape. The man should reference Image 0. The landscape should reference Image 1."
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
|
||||
class ReferenceImageSelector:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model,
|
||||
):
|
||||
|
||||
self.chat_model = chat_model
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=after_func,
|
||||
)
|
||||
async def select_reference_images_and_generate_prompt(
|
||||
self,
|
||||
available_image_path_and_text_pairs: List[Tuple[str, str]],
|
||||
frame_description: str,
|
||||
):
|
||||
filtered_image_path_and_text_pairs = available_image_path_and_text_pairs
|
||||
|
||||
# 1. filter images using text-only model
|
||||
if len(available_image_path_and_text_pairs) >= 8:
|
||||
human_content = []
|
||||
for idx, (_, text) in enumerate(available_image_path_and_text_pairs):
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": f"Image {idx}: {text}"
|
||||
})
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": human_prompt_template_select_reference_images.format(frame_description=frame_description)
|
||||
})
|
||||
parser = PydanticOutputParser(pydantic_object=RefImageIndicesAndTextPrompt)
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt_template_select_reference_images_only_text.format(format_instructions=parser.get_format_instructions())),
|
||||
HumanMessage(content=human_content)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
|
||||
try:
|
||||
ref = await chain.ainvoke(messages)
|
||||
filtered_image_path_and_text_pairs = select_pairs_by_indices(available_image_path_and_text_pairs, ref.ref_image_indices)
|
||||
logging.info(f"Filtered image idx:{ref.ref_image_indices}")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error get image prompt: \n{e}")
|
||||
raise e
|
||||
|
||||
# 2. filter images using multimodal model
|
||||
human_content = []
|
||||
for idx, (image_path, text) in enumerate(filtered_image_path_and_text_pairs):
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": f"Image {idx}: {text}"
|
||||
})
|
||||
human_content.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_path_to_b64(image_path)}
|
||||
})
|
||||
human_content.append({
|
||||
"type": "text",
|
||||
"text": human_prompt_template_select_reference_images.format(frame_description=frame_description)
|
||||
})
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=RefImageIndicesAndTextPrompt)
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt_template_select_reference_images_multimodal.format(format_instructions=parser.get_format_instructions())),
|
||||
HumanMessage(content=human_content)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
|
||||
try:
|
||||
response = await chain.ainvoke(messages)
|
||||
reference_image_path_and_text_pairs = select_pairs_by_indices(filtered_image_path_and_text_pairs, response.ref_image_indices)
|
||||
return {
|
||||
"reference_image_path_and_text_pairs": reference_image_path_and_text_pairs,
|
||||
"text_prompt": response.text_prompt,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error get image prompt: \n{e}")
|
||||
raise e
|
||||
|
||||
|
||||
|
||||
|
||||
def select_pairs_by_indices(pairs, indices):
|
||||
"""Index into pairs with LLM-emitted indices, rejecting out-of-range values.
|
||||
|
||||
Negative indices would silently select the wrong image via Python indexing.
|
||||
"""
|
||||
invalid = [i for i in indices if i < 0 or i >= len(pairs)]
|
||||
if invalid:
|
||||
raise ValueError(f"ref_image_indices out of range: {invalid} (have {len(pairs)} images)")
|
||||
return [pairs[i] for i in indices]
|
||||
@@ -0,0 +1,109 @@
|
||||
from langchain_community.vectorstores import FAISS
|
||||
from interfaces import Event, Scene
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Literal, Tuple, Dict
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from tenacity import retry, stop_after_attempt
|
||||
import logging
|
||||
|
||||
system_prompt_template_get_next_scene = \
|
||||
"""
|
||||
You are an expert scriptwriter specializing in adapting literary works into structured screenplay scenes. Your task is to analyze event descriptions from novels and transform them into compelling screenplay scenes, leveraging relevant context while ignoring extraneous information.
|
||||
|
||||
**TASK**
|
||||
Generate the next scene for a screenplay adaptation based on the provided input. Each scene must include:
|
||||
- Environment: slugline and detailed description
|
||||
- Characters: List of characters appearing in the scene, with their static features (e.g., facial features, body shape), dynamic features (e.g., clothing, accessories), and visibility status
|
||||
- Script: Character actions and dialogues in standard screenplay format
|
||||
|
||||
**INPUT**
|
||||
- Event Description: A clear, concise summary of the event to adapt. The event description is enclosed within <EVENT_DESCRIPTION_START> and <EVENT_DESCRIPTION_END> tags.
|
||||
- Context Fragments: Multiple excerpts retrieved from the novel via RAG. These may contain irrelevant passages. Ignore any content not directly related to the event. The sequence of context fragments is enclosed within <CONTEXT_FRAGMENTS_START> and <CONTEXT_FRAGMENTS_END> tags. Each fragment in the sequence is enclosed within its own <FRAGMENT_N_START> and <FRAGMENT_N_END> tags, with N being the fragment number.
|
||||
- Previous Scenes (if any): Already adapted scenes for context (may be empty). The sequence of previous scenes is enclosed within <PREVIOUS_SCENES_START> and <PREVIOUS_SCENES_END> tags. Each scene is enclosed within its own <SCENE_N_START> and <SCENE_N_END> tags, with N being the scene number.
|
||||
|
||||
**OUTPUT**
|
||||
{format_instructions}
|
||||
|
||||
**GUIDELINES**
|
||||
1. Extract scenes based on the provided context fragments. Strive to preserve the original meaning and dialogue without making arbitrary alterations. When adapting, ensure that every line of dialogue has a corresponding or derivative basis in the original text.
|
||||
2. Focus on Relevance: Use only context fragments that directly align with the event description. Disregard any unrelated paragraphs.
|
||||
3. Dialogues and Actions: Convert descriptive prose into actionable lines and dialogues. Invent minimal necessary dialogue if implied but not explicit in the context.
|
||||
4. Conciseness: Keep descriptions brief and visual. Avoid prose-like explanations.
|
||||
5. Format Consistency: Ensure industry-standard screenplay structure.
|
||||
6. Implicit Inference: If context fragments lack exact details, infer logically from the event description or broader narrative context.
|
||||
7. No Extraneous Content: Do not include scenes, characters, or dialogues unrelated to the core event.
|
||||
8. The character must be an individual, not a group of individuals (such as a crowd of onlookers or a rescue team).
|
||||
9. When the location or time changes, a new scene should be created. The total number of scenes should not more than 5!!!
|
||||
10. The language of outputs in values should be same as the input.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_get_next_scene = \
|
||||
"""
|
||||
<EVENT_DESCRIPTION_START>
|
||||
{event_description}
|
||||
<EVENT_DESCRIPTION_END>
|
||||
|
||||
<CONTEXT_FRAGMENTS_START>
|
||||
{context_fragments}
|
||||
<CONTEXT_FRAGMENTS_END>
|
||||
|
||||
<PREVIOUS_SCENES_START>
|
||||
{previous_scenes}
|
||||
<PREVIOUS_SCENES_END>
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
class SceneExtractor:
|
||||
def __init__(
|
||||
self,
|
||||
api_key,
|
||||
base_url,
|
||||
chat_model,
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
model_provider="openai",
|
||||
)
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(5),
|
||||
after=lambda retry_state: logging.warning(f"Retrying SceneExtractor.get_next_scene due to error: {retry_state.outcome.exception()}"),
|
||||
)
|
||||
async def get_next_scene(
|
||||
self,
|
||||
relevant_chunks: List[str],
|
||||
event: Event,
|
||||
previous_scenes: List[Scene]
|
||||
) -> Scene:
|
||||
|
||||
context_fragments_str = "\n".join([f"<FRAGMENT_{i}_START>\n{chunk}\n<FRAGMENT_{i}_END>" for i, chunk in enumerate(relevant_chunks)])
|
||||
|
||||
previous_scenes_str = "\n".join([f"<SCENE_{i}_START>\n{scene}\n<SCENE_{i}_END>" for i, scene in enumerate(previous_scenes)])
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=Scene)
|
||||
|
||||
messages = [
|
||||
SystemMessage(
|
||||
content=system_prompt_template_get_next_scene.format(
|
||||
format_instructions=parser.get_format_instructions(),
|
||||
),
|
||||
),
|
||||
HumanMessage(
|
||||
content=human_prompt_template_get_next_scene.format(
|
||||
event_description=str(event),
|
||||
context_fragments=context_fragments_str,
|
||||
previous_scenes=previous_scenes_str,
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
chain = self.chat_model | parser
|
||||
scene = await chain.ainvoke(messages)
|
||||
return scene
|
||||
@@ -0,0 +1,167 @@
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from utils.retry import after_func
|
||||
|
||||
|
||||
|
||||
system_prompt_template_develop_story = \
|
||||
"""
|
||||
[Role]
|
||||
You are a seasoned creative story generation expert. You possess the following core skills:
|
||||
- Idea Expansion and Conceptualization: The ability to expand a vague idea, a one-line inspiration, or a concept into a fleshed-out, logically coherent story world.
|
||||
- Story Structure Design: Mastery of classic narrative models like the three-act structure, the hero's journey, etc., enabling you to construct engaging story arcs with a beginning, middle, and end, tailored to the story's genre.
|
||||
- Character Development: Expertise in creating three-dimensional characters with motivations, flaws, and growth arcs, and designing complex relationships between them.
|
||||
- Scene Depiction and Pacing: The skill to vividly depict various settings and precisely control the narrative rhythm, allocating detail appropriately based on the required number of scenes.
|
||||
- Audience Adaptation: The ability to adjust the language style, thematic depth, and content suitability based on the target audience (e.g., children, teenagers, adults).
|
||||
- Screenplay-Oriented Thinking: When the story is intended for short film or movie adaptation, you can naturally incorporate visual elements (e.g., scene atmosphere, key actions, dialogue) into the narrative, making the story more cinematic and filmable.
|
||||
|
||||
[Task]
|
||||
Your core task is to generate a complete, engaging story that conforms to the specified requirements, based on the user's provided "Idea" and "Requirements."
|
||||
|
||||
[Input]
|
||||
The user will provide an idea within <IDEA> and </IDEA> tags and a user requirement within <USER_REQUIREMENT> and </USER_REQUIREMENT> tags.
|
||||
- Idea: This is the core seed of the story. It could be a sentence, a concept, a setting, or a scene. For example,
|
||||
- "A programmer discovers his shadow has a consciousness of its own.",
|
||||
- "What if memories could be deleted and backed up like files?",
|
||||
- "A locked-room murder mystery occurring on a space station."
|
||||
- User Requirement (Optional): Optional constraints or guidelines the user may specify. For example,
|
||||
- Target Audience: e.g., Children (7-12), Young Adults, Adults, All Ages.
|
||||
- Story Type/Genre: e.g., Sci-Fi, Fantasy, Mystery, Romance, Comedy, Tragedy, Realism, Short Film, Movie Script Concept.
|
||||
- Length: e.g., 5 key scenes, a tight story suitable for a 10-minute short film.
|
||||
- Other: e.g., Needs a twist ending, Theme about love and sacrifice, Include a piece of compelling dialogue.
|
||||
|
||||
[Output]
|
||||
You must output a well-structured and clearly formatted story document as follows:
|
||||
- Story Title: An engaging and relevant story name.
|
||||
- Target Audience & Genre: Start by explicitly restating: "This story is targeted at [User-Specified Audience], in the [User-Specified Genre] genre."
|
||||
- Story Outline/Summary: Provide a one-paragraph (100-200 words) summary of the entire story, covering the core plot, central conflict, and outcome.
|
||||
Main Characters Introduction: Briefly introduce the core characters, including their names, key traits, and motivations.
|
||||
- Full Story Narrative:
|
||||
- If the number of scenes is unspecified, narrate the story naturally in paragraphs following the "Introduction - Development - Climax - Conclusion" structure.
|
||||
- If a specific number of scenes (e.g., N scenes) is specified, clearly divide the story into N scenes, giving each a subheading (e.g., Scene One: Code at Midnight). The description for each scene should be relatively balanced, including atmosphere, character actions, and dialogue, all working together to advance the plot.
|
||||
- The narrative should be vivid and detailed, matching the specified genre and target audience.
|
||||
- The output should begin directly with the story, without any extra words.
|
||||
|
||||
[Guidelines]
|
||||
- The language of output should be same as the input.
|
||||
- Idea-Centric: Keep the user's core idea as the foundation; do not deviate from its essence. If the user's idea is vague, you can use creativity to make reasonable expansions.
|
||||
- Logical Consistency: Ensure that event progression and character actions within the story have logical motives and internal consistency, avoiding abrupt or contradictory plots.
|
||||
- Show, Don't Tell: Reveal characters' personalities and emotions through their actions, dialogues, and details, rather than stating them flatly. For example, use "He clenched - his fist, nails digging deep into his palm" instead of "He was very angry."
|
||||
- Originality & Compliance: Generate original content based on the user's idea, avoiding direct plagiarism of well-known existing works. The generated content must be positive, healthy, and comply with general content safety policies.
|
||||
"""
|
||||
|
||||
human_prompt_template_develop_story = \
|
||||
"""
|
||||
<IDEA>
|
||||
{idea}
|
||||
</IDEA>
|
||||
|
||||
<USER_REQUIREMENT>
|
||||
{user_requirement}
|
||||
</USER_REQUIREMENT>
|
||||
"""
|
||||
|
||||
|
||||
|
||||
system_prompt_template_write_script_based_on_story = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional AI script adaptation assistant skilled in adapting stories into scripts. You possess the following skills:
|
||||
- Story Analysis Skills: Ability to deeply understand the story content, identify key plot points, character arcs, and themes.
|
||||
- Scene Segmentation Skills: Ability to break down the story into logical scene units based on continuity of time and location.
|
||||
- Script Writing Skills: Familiarity with script formats (e.g., for short films or movies), capable of crafting vivid dialogue, action descriptions, and stage directions.
|
||||
- Adaptive Adjustment Skills: Ability to adjust the script's style, language, and content based on user requirements (e.g., target audience, story genre, number of scenes).
|
||||
- Creative Enhancement Skills: Ability to appropriately add dramatic elements to enhance the script's appeal while remaining faithful to the original story.
|
||||
|
||||
[Task]
|
||||
Your task is to adapt the user's input story, along with optional requirements, into a script divided by scenes. The output should be a list of scripts, each representing a complete script for one scene. Each scene must be a continuous dramatic action unit occurring at the same time and location.
|
||||
|
||||
[Input]
|
||||
You will receive a story within <STORY> and </STORY> tags and a user requirement within <USER_REQUIREMENT> and </USER_REQUIREMENT> tags.
|
||||
- Story: A complete or partial narrative text, which may contain one or more scenes. The story will provide plot, characters, dialogues, and background descriptions.
|
||||
- User Requirement (Optional): A user requirement, which may be empty. The user requirement may include:
|
||||
- Target audience (e.g., children, teenagers, adults).
|
||||
- Script genre (e.g., micro-film, moive, short drama).
|
||||
- Desired number of scenes (e.g., "divide into 3 scenes").
|
||||
- Other specific instructions (e.g., emphasize dialogue or action).
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
- The language of output in values should be same as the input story.
|
||||
- Scene Division Principles: Each scene must be based on the same time and location. Start a new scene when the time or location changes. If the user specifies the number of scenes, try to match the requirement. Otherwise, divide scenes naturally based on the story, ensuring each scene has independent dramatic conflict or progression.
|
||||
- Script Formatting Standards: Use standard script formatting: Scene headings in full caps or bold, character names centered or capitalized, dialogue indented, and action descriptions in parentheses.
|
||||
- Coherence and Fluidity: Ensure natural transitions between scenes and overall story flow. Avoid abrupt plot jumps.
|
||||
- Visual Enhancement Principles: All descriptions must be "filmable". Use concrete actions instead of abstract emotions (e.g., "He turns away to avoid eye contact" instead of "He feels ashamed"). Decribe rich environmental details include lighting, props, weather, etc., to enhance the atmosphere. Visualize character performances such as express internal states through facial expressions, gestures, and movements (e.g., "She bites her lip, her hands trembling" to imply nervousness).
|
||||
- Consistency: Ensure dialogue and actions align with the original story's intent, without deviating from the core plot.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_write_script_based_on_story = \
|
||||
"""
|
||||
<STORY>
|
||||
{story}
|
||||
</STORY>
|
||||
|
||||
<USER_REQUIREMENT>
|
||||
{user_requirement}
|
||||
</USER_REQUIREMENT>
|
||||
"""
|
||||
|
||||
|
||||
class Screenwriter:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model: str,
|
||||
):
|
||||
self.chat_model = chat_model
|
||||
|
||||
async def develop_story(
|
||||
self,
|
||||
idea: str,
|
||||
user_requirement: Optional[str] = None,
|
||||
) -> str:
|
||||
messages = [
|
||||
("system", system_prompt_template_develop_story),
|
||||
("human", human_prompt_template_develop_story.format(idea=idea, user_requirement=user_requirement)),
|
||||
]
|
||||
response = await self.chat_model.ainvoke(messages)
|
||||
story = response.content
|
||||
return story
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, max=30), after=after_func)
|
||||
async def write_script_based_on_story(
|
||||
self,
|
||||
story: str,
|
||||
user_requirement: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
|
||||
|
||||
class WriteScriptBasedOnStoryResponse(BaseModel):
|
||||
script: List[str] = Field(
|
||||
...,
|
||||
description="The script based on the story. Each element is a scene "
|
||||
)
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=WriteScriptBasedOnStoryResponse)
|
||||
format_instructions = parser.get_format_instructions()
|
||||
|
||||
messages = [
|
||||
("system", system_prompt_template_write_script_based_on_story.format(format_instructions=format_instructions)),
|
||||
("human", human_prompt_template_write_script_based_on_story.format(story=story, user_requirement=user_requirement)),
|
||||
]
|
||||
response = await self.chat_model.ainvoke(messages)
|
||||
response = parser.parse(response.content)
|
||||
script = response.script
|
||||
return script
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import logging
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt
|
||||
|
||||
|
||||
system_prompt_template_script_enhancer = \
|
||||
"""
|
||||
[Role]
|
||||
You are a senior screenplay polishing and continuity expert.
|
||||
|
||||
[Task]
|
||||
Enhance a planned narrative script by adding specific, concrete sensory details, tightening continuity, clarifying scene transitions, and keeping terminology consistent (character names, locations, objects). Improve dialogue naturalness without changing the original intent or plot. Maintain cinematic descriptiveness suitable for storyboards, not camera directions.
|
||||
|
||||
[Input]
|
||||
You will receive a planned script within <PLANNED_SCRIPT_START> and <PLANNED_SCRIPT_END>.
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
1. Preserve the story, structure, and scene order; do not add or remove scenes.
|
||||
2. Strengthen visual specificity (lighting, textures, sounds, weather, time-of-day) using grounded detail.
|
||||
3. Ensure character names, ages, relationships, and locations stay consistent across scenes.
|
||||
5. Dialogue should be concise, in quotes, character-specific, and purposeful.
|
||||
6. Avoid camera jargon (e.g., cut to, close-up) and voiceover formatting.
|
||||
7. No metaphors.
|
||||
8. Repetition for Precision
|
||||
Re‑state important objects/actors often (vehicle name, seat position, or character role) to remove ambiguity. Accuracy takes precedence over rhythm — redundancy is acceptable.
|
||||
9. Character Features for Dialogue
|
||||
For each character in the conversation, repeat the core voice description (e.g., male, early 50s, South African–North American accent) using the same prompt each time.
|
||||
10. Preserve the original narration symbols if exists (eg. Narration: "Everything is looking good").
|
||||
|
||||
Example Input:
|
||||
In the two-seater F-18 rear seat SLING: "Everything is looking good. All systems are green, Elon. We’re ready for takeoff."
|
||||
In the two-seater F-18 front seat Elon Musk: "Understood, Sling. Let’s get this show on the road."
|
||||
In the two‑seater F‑18 rear seat SLING: "Roger that. Strap in tight, boss. It’s gonna be a smooth ride."
|
||||
In the two‑seater F‑18 front seat ELON MUSK: "Smooth is good. Let’s keep it that way."
|
||||
|
||||
Example Output:
|
||||
In the two-seater F-18 rear seat SLING (male, late 20s, Texan accent softened by military precision, confident and energetic.): "Everything is looking good. All systems are green, Elon. We’re ready for takeoff."
|
||||
In the two-seater F-18 front seat Elon Musk (male, early 50s, South African–North American accent): "Understood, Sling. Let’s get this show on the road."
|
||||
In the two‑seater F‑18 rear seat SLING (male, late 20s, Texan accent softened by military precision, confident and energetic.): "Roger that. Strap in tight, boss. It’s gonna be a smooth ride."
|
||||
In the two‑seater F‑18 front seat ELON MUSK (male, early 50s, South African–North American accent): "Smooth is good. Let’s keep it that way."
|
||||
10. Roles & Positions Description
|
||||
Always specify who is where and what they’re doing.
|
||||
Example Input: “In the cockpit front seat of the two‑seat F‑18, the pilot checks his controls.”
|
||||
Example Output: “In the cockpit front seat of the two‑seat F‑18, Elon Musk checks his controls.”
|
||||
Avoid shorthand (“the pilot”) unless you’ve already identified them in that exact position.
|
||||
|
||||
Warnings
|
||||
No camera directions. No metaphors. Do not change the plot.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_script_enhancer = \
|
||||
"""
|
||||
<PLANNED_SCRIPT_START>
|
||||
{planned_script}
|
||||
<PLANNED_SCRIPT_END>
|
||||
"""
|
||||
|
||||
|
||||
class EnhancedScriptResponse(BaseModel):
|
||||
enhanced_script: str = Field(
|
||||
...,
|
||||
description="A refined script version with clearer continuity, stronger concrete detail, and improved dialogue while preserving the original story and scene order."
|
||||
)
|
||||
|
||||
|
||||
class ScriptEnhancer:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model: str,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model_provider: str = "openai",
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
model_provider=model_provider,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
after=lambda retry_state: logging.warning(f"Retrying enhance_script due to error: {retry_state.outcome.exception()}"),
|
||||
)
|
||||
async def enhance_script(
|
||||
self,
|
||||
planned_script: str,
|
||||
) -> EnhancedScriptResponse:
|
||||
"""
|
||||
Enhance a planned script with more concrete detail and continuity polish.
|
||||
"""
|
||||
parser = PydanticOutputParser(pydantic_object=EnhancedScriptResponse)
|
||||
prompt_template = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("system", system_prompt_template_script_enhancer),
|
||||
("human", human_prompt_template_script_enhancer),
|
||||
]
|
||||
)
|
||||
chain = prompt_template | self.chat_model | parser
|
||||
|
||||
try:
|
||||
logging.info("Enhancing planned script...")
|
||||
response: EnhancedScriptResponse = await chain.ainvoke(
|
||||
{
|
||||
"format_instructions": parser.get_format_instructions(),
|
||||
"planned_script": planned_script,
|
||||
}
|
||||
)
|
||||
logging.info("Script enhancement completed.")
|
||||
return response.enhanced_script
|
||||
except Exception as e:
|
||||
logging.error(f"Error enhancing script: \n{e}")
|
||||
raise e
|
||||
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
import logging
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from langchain.chat_models import init_chat_model
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Literal
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from utils.retry import after_func
|
||||
|
||||
|
||||
narrative_script_prompt_template = \
|
||||
"""
|
||||
You are a world-class creative writing and screenplay development expert with extensive experience in story structure, character development, and narrative pacing.
|
||||
|
||||
**Task**
|
||||
Your task is to transform a basic story idea into a comprehensive, engaging script with rich narrative detail, compelling character arcs, and cinematic storytelling elements.
|
||||
|
||||
**Input**
|
||||
You will receive a basic story idea or concept enclosed within <BASIC_IDEA_START> and <BASIC_IDEA_END>.
|
||||
|
||||
Below is a simple example of the input:
|
||||
|
||||
<BASIC_IDEA_START>
|
||||
A person discovers they can time travel but every time they change something, they lose a memory.
|
||||
<BASIC_IDEA_END>
|
||||
|
||||
**Output**
|
||||
{format_instructions}
|
||||
|
||||
**Guidelines**
|
||||
No metaphors allowed!!! (eg. A gust of wind rustled through it, a ghostly touch. ; an F1 car that looks less like a vehicle and more like a fighter jet stripped of its wings)
|
||||
|
||||
1. **Story Structure**: Develop a clear three-act structure with proper setup, confrontation, and resolution. Include compelling plot points, rising action, climax, develop the content according to the plot timeline, maintain a clear main plotline, and maintain coherent narrative connections. Keep the plot moving forward. Avoid summarizing events and characters, and use dialogue between key characters appropriately.
|
||||
|
||||
2. **Character Development**: Create well-rounded characters with clear motivations, flaws, and character arcs. Ensure protagonists have relatable goals and face meaningful obstacles.
|
||||
|
||||
3. **Visual Storytelling**: Write with cinematic language that emphasizes visual elements, actions, and atmospheric details rather than exposition-heavy dialogue.
|
||||
|
||||
4. **Emotional Depth**: Incorporate emotional beats, internal conflicts, and character relationships that resonate with audiences.
|
||||
|
||||
5. **Pacing and Tension**: Build suspense and maintain engagement through proper scene transitions, conflict escalation, and strategic revelation of information.
|
||||
|
||||
6. **Genre Consistency**: Maintain appropriate tone, style, and conventions for the story's genre while adding unique creative elements.
|
||||
|
||||
7. **Dialogue Quality**: When you writing some dialogue, you should use the:" " symbols (eg. Peter says: "Everything is looking good. All systems are green, Elon. We’re ready for takeoff."). Do not use voiceover format. Create natural, character-specific dialogue that advances plot and reveals personality without being overly expository.
|
||||
|
||||
8. **Thematic Elements**: Weave in meaningful themes and subtext that give the story depth and universal appeal.
|
||||
|
||||
9. **Conflict and Stakes**: Establish clear external and internal conflicts with high stakes that matter to both characters and audience.
|
||||
|
||||
10. **Satisfying Resolution**: Ensure all major plot threads are resolved and character arcs reach meaningful conclusions.
|
||||
|
||||
11. **Each dialogue should not too short or too long, (eg."Everything is looking good. All systems are green, Elon. We’re ready for takeoff." )
|
||||
|
||||
|
||||
**Warnings**
|
||||
|
||||
Don't write any camera movement in the script (eg. cut to), you should write the script by using storyboard description, not camera view.
|
||||
No metaphors allowed!!! (eg. A gust of wind rustled through it, a ghostly touch. ; an F1 car that looks less like a vehicle and more like a fighter jet stripped of its wings)
|
||||
|
||||
|
||||
**Examples of narrative scripts**
|
||||
|
||||
The starry sky is vast, the Milky Way glittering.
|
||||
On the beach, there's a fire, a portable dining table and chairs (three balloons tied to one corner, swaying in the wind), an SUV, and a camping tent. Next to the tent is an astronomical telescope. A man (Liu Peiqiang, 35, reserved) operates the telescope, while a little boy (Liu Qi, 4, Liu Peiqiang's son) observes under his father's guidance.
|
||||
Liu Peiqiang (somewhat excitedly) Quick, quick, quick... Look, it's Jupiter... the largest planet in the solar system.
|
||||
Adjusting the telescope's eyepiece's focus and position, Jupiter gradually comes into focus. Liu Qi: Dad, there's an eye on Jupiter.
|
||||
Liu Peiqiang: That's not an eye, it's a massive storm on Jupiter's surface. Liu Qi: Why...?
|
||||
Liu Peiqiang: (touching the boy's head, pointing to the balloons on the table) Jupiter is just a giant balloon, 90% hydrogen. Liu Qi: What is hydrogen?
|
||||
An old man (Han Ziang, 59, Liu Peiqiang's father-in-law and Liu Qi's grandfather) walked out of the tent and stood silently beside Liu Peiqiang and his son.
|
||||
Liu Peiqiang: Hydrogen... Hydrogen is the fuel for Dad's big rocket. The campfire flickered, and Han Ziang turned to look at Liu Peiqiang. Liu Qi: Why? Liu Peiqiang smiled and patted his son's head.
|
||||
Liu Peiqiang (O.S.): When the day comes when you can see Jupiter without a telescope, Dad will be back.
|
||||
|
||||
|
||||
|
||||
**Scriptwriting Guidelines End**
|
||||
|
||||
|
||||
"""
|
||||
|
||||
motion_script_prompt_template = \
|
||||
"""
|
||||
You are a top-tier action and motion-sequence script designer with deep visual expertise in conveying speed, force, choreography, and technical precision. Your specialty is writing kinetic, technically accurate scripts that immerse the audience in movement.
|
||||
|
||||
**Task**
|
||||
Transform a basic idea into a motion-driven script that emphasizes precise action description, clear spatial orientation, and unambiguous, technically accurate details.
|
||||
|
||||
**Input**
|
||||
You will receive a basic idea enclosed within <BASIC_IDEA_START> and <BASIC_IDEA_END>.
|
||||
|
||||
**Output**
|
||||
{format_instructions}
|
||||
|
||||
**Global Rules**
|
||||
No metaphors allowed. Less conversation
|
||||
|
||||
**Motion Style Guidelines**
|
||||
1. Technical Explicitness: Prefer precise nouns and qualifiers over poetic language. Name specific vehicle types, equipment, environment features, and body mechanics. If vehicles are implied, specify make/class if reasonable. If combat, specify stance, guard, strike type, target, and contact result.
|
||||
2. Kinetic Clarity: Make trajectories, vectors, speed/acceleration sensations, and force outcomes explicit. Describe distances and orientations when helpful (e.g., left/right, fore/aft).
|
||||
3. Spatial Cohesion: Maintain a consistent mental map of positions. Keep continuity of who/what is where. When positions change, describe how and by what path.
|
||||
4. Sequenced Action Beats: Write step-by-step beats that can be storyboarded. Each beat should be actionable and unambiguous.
|
||||
5. Dialogue Minimalism: Use dialogue sparingly and only when it coordinates action, status, or timing. Use :"dialogue" quotes for spoken lines.
|
||||
6. Keep the script length similar to the following examples.
|
||||
7. If the user does not specify, only one character can appear at most.
|
||||
8. Less character's actions close-ups, more exterior shots
|
||||
9. Don't describe the character's physical state (e.g. jowls and the loose skin around its neck to press back).
|
||||
|
||||
**Examples of motion & speed immersion fighter scripts** (should be accurate, technical, and explicit, Technical Explicitness: Consistently repeats “two seats F‑18” in each stage direction. Prioritizes precision in identifying the aircraft type and location (front seat / rear seat). Reads almost like a technical report or aviation manual, ensuring no ambiguity.)
|
||||
The immense gray flight deck of a nuclear aircraft carrier cuts through a deep blue ocean. The horizon is a clean, sharp line. Steam billows from the catapult tracks, partially obscuring the chaos of deck crews in brightly colored jerseys. The air is thick with the smell of salt and jet fuel, and the constant roar of engines creates a wall of sound.
|
||||
|
||||
An F-18, is positioned on the steam-powered catapult. Its twin engines blast waves of heat that distort the air behind it. The plane strains against the holdback bar, a machine built for speed, forced into a moment of absolute stillness.
|
||||
|
||||
Epic cinematic style with dramatic wide shots, dynamic camera movements, rich color grading, and theatrical lighting reminiscent of major Hollywood productions. Camera gradually moves forward to pilot Elon Musk (50s, sharp eyes and unwavering focus) sits in the cockpit of a F-18. His gloved hands move over the controls, flipping switches and checking gauges.
|
||||
|
||||
In the F-18 cockpit Elon Musk: "Understood, Sling. Let’s get this show on the road."
|
||||
|
||||
In the F-18 cockpit Elon Musk's left hand push on the F-18 throttle, his right grips the control stick.
|
||||
|
||||
A side view. The Shooter drops to one knee, pointing down the deck. The world seems to hold its breath. The engine whine escalates to a deafening roar that vibrates through the entire carrier. The F-18's twin vertical stabilizers shudder with contained power.
|
||||
|
||||
First-person POV from inside the cockpit of F18. With a violent jolt, the catapult fires. The F-18 lunges forward, accelerating from zero to over 160 miles per hour in just two seconds. The deck becomes a blur of motion. Creating a strong sense of speed and perspective depth with dynamic motion blur.
|
||||
|
||||
A side camera view. Then, with a surge of raw power from the afterburners igniting. The F-18 climbs, asserting its dominance over gravity. The landing gear retracts into the fuselage with a solid thud. Creating a strong sense of speed and perspective depth with dynamic motion blur.
|
||||
|
||||
Elon Musk levels the F-18 wings, the sun glinting off his visor as he scans the empty sky ahead.
|
||||
|
||||
The F-18, a sleek instrument of combat, roars to life as it pushes, slicing through the air with an elegant grace. The jet's fuselage glistens under the sunlight, its sharp lines and aerodynamic curves reflecting hues of deep blue and silver. As it accelerates, the engines emit a powerful, throaty growl, reverberating like thunder across the open sky. Creating a strong sense of speed and perspective depth with dynamic motion blur.
|
||||
|
||||
**Examples of motion & speed immersion F1 racing scripts**
|
||||
Epic cinematic style with dramatic wide shots, dynamic camera movements, rich color grading, and theatrical lighting reminiscent of major Hollywood productions. In the black and gold Formula One cockpit, Camera gradually moves forward to F1 driver Elon Musk (playing the driver, a man in his 40s, with a steely gaze and utter concentration) buckling his harness, his helmet visor which reflects the fluttering checkered flags and a blur of cheering spectators in the stands. He drives a sleek black and gold F1 car.
|
||||
|
||||
The starting lights on the track go out, and First-person POV from inside the cockpit of a black and gold F1 car which starts and speeding through the Arena. You grip the wheel — full throttle. The engine roars, gear shifts snap. The blur of the cheering spectators in the stands flashes on your left. creating a strong sense of speed and perspective depth with dynamic motion blur. are engaged in a frenetic, no-holds-barred race. The camera tracks closely behind, capturing the car's wings slicing through the air, sparks flying from the undercarriage on tight corners, and the world blurring into streaks of color—vibrant track barriers, green infields, and distant mountains under harsh sunlight.
|
||||
|
||||
The camera closely tracks the side with dynamic chasing shots., hugging the ground to capture Elon Musk's sleek black and gold F1 car slicing through the air, its APX tail wing flexing under the wind, sparks erupting from the chassis like fireworks as it powers through tight turns and begins overtaking rivals—dodging a pursuing Formula One car , nearly clipping in a heart-pounding near-miss. Cutting to another close-up on Elon Musk, his gloved hands gripping the F1 steering wheel tightly, while the background track barriers streak by in accelerated motion. Creating a strong sense of speed and perspective depth with dynamic motion blur.
|
||||
|
||||
An aerial view for a wide chase perspective, showing Elon Musk's APX Formula One car boldly overtaking another rival in a daring maneuver, debris scattering across the asphalt as it pulls ahead, the pulsating to a crescendo amidst the intensified roar of engines, whistling wind, and the stronger surge of acceleration that makes the entire frame vibrate with raw power. Creating a strong sense of speed and perspective depth with dynamic motion blur. are engaged in a frenetic, no-holds-barred race.
|
||||
|
||||
A front-mounted chase shot follows, emphasizing the APX tail wing's metallic sheen as the black and gold F1 car banks into a hairpin turn, other Formula One rivals closing in from both sides in a tense three-way battle, the movement acceleration pushing the limits as Elon Musk's black and gold F1 car breaks free, leaving F1 competitors in a cloud of dust.
|
||||
|
||||
The camera jolts into a raw handheld shot as Elon Musk’s APX black and gold F1 car rockets down a blistering straightaway, creating a strong sense of speed and perspective depth with dynamic motion blur, are engaged in a frenetic, no-holds-barred race. Rivals' red-white Formula one car closing in tight on both flanks. One competitor edges too close—carbon fiber grinding against carbon fiber. Sparks erupt in a spray of gold as Elon Musk wrenches the wheel, but the rival's red-white Formula one car fishtails, spinning out of control before slamming violently into the barrier. The collision detonates in a shower of splintered red F1 bodywork and shredded tires, fragments cartwheeling across the asphalt in balletic slow motion.
|
||||
|
||||
Wide aerial shots capture the chaos as smoke and dust mushroom upward, the track swallowed in a haze of flame-orange light. Then—an explosive cut back to full speed—Elon Musk’s sleek black and gold F1 APX car bursts through the choking smoke cloud, unbroken, streaking down the straight. Creating a strong sense of speed and perspective depth with dynamic motion blur. are engaged in a frenetic, no-holds-barred race.
|
||||
|
||||
Another extreme close-up zooms in on F1 driver Elon Musk's visor, the lens focus pronouncing the reflection of the track rushing by, capturing the intensity of his focus amid the chaos. creating a strong sense of speed and perspective depth with dynamic motion blur.
|
||||
|
||||
The sequence escalates with a low-angle chase shot from behind, creating a strong sense of speed and perspective depth with dynamic motion blur. Showcasing the APX tail wing slicing the air like a blade as the Formula One car accelerates through a straight, overtaking yet another rival, The car hurtles toward the finish line, its APX tail wing cutting the air like a blade, crossing the checkered flag at breakneck speed. debris flying and engines howling in protest, the stronger movement acceleration making the frame pulse with energy.
|
||||
|
||||
**Warnings**
|
||||
- Do not use metaphors.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
montage_script_prompt_template = \
|
||||
"""
|
||||
You are a top-tier montage script designer with deep expertise in compressing time, juxtaposing images, and shaping emotional arcs through shot selection and rhythm. Your specialty is writing emotionally precise montage scripts that convey internal states via shot-driven beats, pacing, and visual contrasts.
|
||||
|
||||
Task
|
||||
Transform a basic idea into an emotion-driven montage script that emphasizes internal experience through visual sequencing, clear emotional expression per shot/beat, and unambiguous psychological details.
|
||||
|
||||
Input
|
||||
You will receive a basic idea enclosed within <BASIC_IDEA_START> and <BASIC_IDEA_END>.
|
||||
|
||||
Output
|
||||
{format_instructions}
|
||||
|
||||
**Global Rules**
|
||||
No metaphors allowed.
|
||||
Keep dialogue minimal.
|
||||
Use pure paragraph.
|
||||
Convey meaning primarily through shot progression, rhythm, and visual juxtaposition.
|
||||
Montage Style Guidelines
|
||||
Use plain sentence/paragraph
|
||||
For each secene, you should write multiple shots to enhance montage effect.
|
||||
Total no less than 500 words, each paragraph no more than 50 words.
|
||||
Escalation or Resolution: Build an emotional arc across beats. Show explicit changes in emotional state and the cause for each change.
|
||||
Sound Design Minimalism: Use sparse, precise notes for sound/music that influence emotion (tempo rise, percussive cuts, breath presence). Avoid lyrical description.
|
||||
Dialogue Minimalism: Include dialogue only if it marks a clear emotional shift. Use :"dialogue" quotes.
|
||||
Visual Clarity Over Action: Limit complex external action. Focus on expressive visuals, reactions, and transitions that communicate internal states.
|
||||
No extraneous physical traits. Only describe details that influence or reveal emotion.
|
||||
**Warnings**
|
||||
Do not use metaphors.
|
||||
Avoid poetic language; prefer precise, observable details.
|
||||
|
||||
**Examples of scripts**
|
||||
Morning light across a small practice room. A girl (Lisa) around seven lifts a violin from its case. Bow slips on the first note.
|
||||
|
||||
|
||||
She (Lisa) winces, then tries again. Shoulders ease. Relief. Quiet room, a single chair creak.
|
||||
|
||||
|
||||
She (Lisa) rests her cheek on the chinrest. The string hum stabilizes.
|
||||
|
||||
|
||||
A small smile shows on Lisa.
|
||||
|
||||
|
||||
Front hall. School shoes near a folded music stand.
|
||||
|
||||
|
||||
She (Lisa) struggles with the latch. The stand clicks open. Light metal tap on tile.
|
||||
|
||||
|
||||
Afternoon window. She (Lisa) traces notes with a finger. Her mother taps a rhythm on the table.
|
||||
|
||||
|
||||
She (Lisa) frowns, then raises her elbow. Concentration holds. The bow settles. Shared stillness. Page flip, steady breath.
|
||||
|
||||
|
||||
Bathroom. She (Lisa) wipes rosin dust off the instrument, coughing once.
|
||||
|
||||
|
||||
Bedroom floor. Sheet music spread. She (Lisa) circles three notes with a red pencil.
|
||||
|
||||
|
||||
She (Lisa) plays them alone, slow, then again faster. Frustration dips, control returns. Pencil tap stops.
|
||||
|
||||
|
||||
Kitchen doorway. A metronome ticks beside a bowl of fruit. She (Lisa) dials it down two clicks. Shoulders drop. She follows the pulse, bow hand steadier with each measure.
|
||||
|
||||
|
||||
Living room. A TV murmurs. She (Lisa) crosses, lowers the volume, returns to her stand. Boundary set without words. The room holds for practice.
|
||||
|
||||
|
||||
Front steps. Case open to the sun. A neighbor waves. She (Lisa) shields the strings with her palm, smiles, and closes the lid. Protection learned.
|
||||
|
||||
|
||||
Music store aisle. Shoulder rests in a row. She (Lisa) tries one that squeaks, then another that fits. Jaw unclenches. She nods, decision made.
|
||||
|
||||
|
||||
Rain on the window. She (Lisa) misses a shift three times. Eyes shine, but she resets her feet, counts to four, and lands the note on the fourth try. Relief, not triumph. Bow lifts, still.
|
||||
|
||||
|
||||
Mirror practice. Thin tape marks on the fingerboard. She (Lisa) glances once, places a finger true, then plays without looking. Confidence grows around the guide.
|
||||
|
||||
|
||||
School hallway before recital. Cold hands under a dryer. She (Lisa) shakes out wrists. Fear thins to focus. She walks toward the stage door, steps even.
|
||||
|
||||
|
||||
Curtain edge. Small tremor at the frog. She (Lisa) loosens grip, breathes, and steps into light.
|
||||
|
||||
|
||||
Two clean phrases. One fuzzy entrance. She (Lisa) holds tempo, corrects on the next measure. Recovery without apology.
|
||||
|
||||
|
||||
Exit corridor. Water bottle cap clicks. She (Lisa) writes in a pocket notebook: “Entrance softer, elbow high.” Emotion measured by action.
|
||||
|
||||
|
||||
Saturday morning. An online tutorial freezes mid-vibrato. She (Lisa) mimics the motion without sound. Adds bow. Wobble uneven. She smiles anyway. Incremental progress accepted.
|
||||
|
||||
|
||||
Park bench. Practice mute on the bridge. Joggers pass without looking. She (Lisa) finishes a scale, closes her eyes a moment, then starts the etude. Privacy inside noise.
|
||||
|
||||
|
||||
Bedroom desk. A planner open. She (Lisa) blocks out “scales + shifts” for fifteen minutes daily. A small star beside Sunday. Plan replaces hope.
|
||||
|
||||
|
||||
Evening soreness. A red mark under her jaw. She (Lisa) folds a soft cloth over the rest, tries again. Mark fades. Comfort adjusted, practice continues.
|
||||
|
||||
|
||||
String snap. Sharp, quick. She (Lisa) flinches, then opens a spare packet, threads, winds, tunes slow. Disruption handled. Bow returns to the string.
|
||||
|
||||
|
||||
Phone buzz. A friend’s invitation lights the screen. She (Lisa) looks once, turns it face down, and plays the piece end to end. Reward after task.
|
||||
|
||||
|
||||
Audition day. Waiting chairs in a line. She (Lisa) air-bows the first phrase, eyes closed. Shoulders stay low. Name called. She stands smoothly.
|
||||
|
||||
|
||||
Small studio. Two judges, still faces. She (Lisa) tunes, begins. First note centered, breath even. A slip in the middle; tempo holds. The last note rings.
|
||||
|
||||
|
||||
Street outside. She (Lisa) exhales into cool air, checks her watch, and walks home. No jump, no slump. Next step implied.
|
||||
|
||||
|
||||
Kitchen table. Acceptance email on a tablet. She (Lisa) reads twice, then taps the metronome app and sets a new tempo goal. Celebration nested inside routine.
|
||||
|
||||
|
||||
Summer afternoon. Open window, distant mower. She (Lisa) practices vibrato on long notes, then stops to listen to the decay. Ear sharpens.
|
||||
|
||||
|
||||
Library corner. She (Lisa) copies fingerings in neat pencil on a fresh sheet. The messy draft slides into recycling. Order replaces clutter.
|
||||
|
||||
|
||||
Community center stage. A quartet rehearsal. She (Lisa) watches the leader’s breath, lifts with it, and enters together. Listening added to playing.
|
||||
|
||||
|
||||
Night lamp. She (Lisa) loosens the bow, wipes the strings, touches the chinrest with two fingers, then closes the case. Habit completes the day. Quiet returns.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
human_prompt_template_script_planner = \
|
||||
"""
|
||||
<BASIC_IDEA_START>
|
||||
{basic_idea}
|
||||
<BASIC_IDEA_END>
|
||||
"""
|
||||
|
||||
|
||||
class IntentRouterResponse(BaseModel):
|
||||
intent: Literal["narrative", "motion", "montage"] = Field(
|
||||
..., description="Routing decision: 'narrative' for characters multi-conversation focus, 'motion' for action/kinetic focus, 'montage' for emotional montage focus"
|
||||
)
|
||||
rationale: Optional[str] = Field(
|
||||
default=None, description="Brief reason for the classification"
|
||||
)
|
||||
|
||||
|
||||
class PlannedScriptResponse(BaseModel):
|
||||
planned_script: str = Field(
|
||||
...,
|
||||
description="The full planned script with rich narrative detail, character development, dialogue, and cinematic descriptions. Should be significantly more detailed and engaging than the original basic idea."
|
||||
)
|
||||
|
||||
|
||||
|
||||
class ScriptPlanner:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model: str,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model_provider: str = "openai",
|
||||
):
|
||||
self.chat_model = init_chat_model(
|
||||
model=chat_model,
|
||||
model_provider=model_provider,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, max=30), after=after_func)
|
||||
def plan_script(
|
||||
self,
|
||||
basic_idea: str,
|
||||
) -> PlannedScriptResponse:
|
||||
"""
|
||||
Plan a comprehensive script from a basic idea.
|
||||
|
||||
Args:
|
||||
basic_idea: A simple story concept or idea to be expanded
|
||||
|
||||
Returns:
|
||||
PlannedScriptResponse: A comprehensive script with structure, characters, and narrative detail
|
||||
"""
|
||||
# 1) Route intent to select the appropriate template
|
||||
router_parser = PydanticOutputParser(pydantic_object=IntentRouterResponse)
|
||||
router_prompt_template = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
(
|
||||
'system',
|
||||
"""
|
||||
You are an intent router for script planning. Classify the user's basic idea into one of following intents:
|
||||
|
||||
- narrative: The idea centers on character, plot, themes, dialogue, or broad storytelling beats.
|
||||
- motion: The idea centers on action, speed, vehicles, combat, choreography, sports, or any kinetic sequence where precise, technical motion description is primary.
|
||||
- montage: The idea centers on a series of shots that convey an emotional arc through imagery, pacing, and juxtaposition.
|
||||
|
||||
Respond using the required JSON format only
|
||||
{format_instructions}
|
||||
"""
|
||||
),
|
||||
('human', human_prompt_template_script_planner),
|
||||
]
|
||||
)
|
||||
router_chain = router_prompt_template | self.chat_model | router_parser
|
||||
|
||||
routing = router_chain.invoke(
|
||||
{
|
||||
"format_instructions": router_parser.get_format_instructions(),
|
||||
"basic_idea": basic_idea,
|
||||
}
|
||||
)
|
||||
chosen_intent = routing.intent if isinstance(routing, IntentRouterResponse) else "narrative"
|
||||
logging.info(f"[ScriptPlanner] Intent routed to: {chosen_intent}")
|
||||
|
||||
# 2) Build the planning chain with the selected template
|
||||
planning_parser = PydanticOutputParser(pydantic_object=PlannedScriptResponse)
|
||||
|
||||
# Template selection with graceful fallbacks
|
||||
def get_system_template(intent: str) -> str:
|
||||
try:
|
||||
if intent == "narrative":
|
||||
return narrative_script_prompt_template
|
||||
if intent == "motion":
|
||||
return motion_script_prompt_template
|
||||
if intent == "montage":
|
||||
return montage_script_prompt_template
|
||||
except NameError:
|
||||
# Fallbacks if specific templates not defined in scope
|
||||
pass
|
||||
# Default fallback
|
||||
return narrative_script_prompt_template
|
||||
|
||||
system_template = get_system_template(chosen_intent)
|
||||
|
||||
planning_prompt_template = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
('system', system_template),
|
||||
('human', human_prompt_template_script_planner),
|
||||
]
|
||||
)
|
||||
planning_chain = planning_prompt_template | self.chat_model | planning_parser
|
||||
|
||||
try:
|
||||
logging.info(f"Planning script from basic idea: {basic_idea[:100]}...")
|
||||
response = planning_chain.invoke(
|
||||
{
|
||||
"format_instructions": planning_parser.get_format_instructions(),
|
||||
"basic_idea": basic_idea,
|
||||
}
|
||||
)
|
||||
logging.info("Script planning completed.")
|
||||
return response
|
||||
except Exception as e:
|
||||
logging.error(f"Error planning script: \n{e}")
|
||||
raise e
|
||||
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
from typing import List, Optional, Literal
|
||||
import asyncio
|
||||
from pydantic import BaseModel, Field
|
||||
from tenacity import retry, stop_after_attempt
|
||||
|
||||
from langchain.chat_models.base import BaseChatModel
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import PydanticOutputParser
|
||||
from interfaces import CharacterInScene, ShotDescription, ShotBriefDescription
|
||||
|
||||
from utils.retry import after_func
|
||||
|
||||
|
||||
|
||||
system_prompt_template_design_storyboard = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional storyboard artist with the following core skills:
|
||||
- Script Analysis: Ability to quickly interpret a script's text, identifying the setting, character actions, dialogue, emotions, and narrative pacing.
|
||||
- Visualization: Expertise in translating written descriptions into visual frames, including composition, lighting, and spatial arrangement.
|
||||
- Storyboarding: Proficiency in cinematic language, such as shot types (e.g., close-up, medium shot, wide shot), camera angles (e.g., high angle, eye-level), camera movements (e.g., zoom, pan), and transitions.
|
||||
- Narrative Continuity: Ability to ensure the storyboard sequence is logically smooth, highlights key plot points, and maintains emotional consistency.
|
||||
- Technical Knowledge: Understanding of basic storyboard formats and industry standards, such as using numbered shots and concise descriptions.
|
||||
|
||||
[Task]
|
||||
Your task is to design a complete storyboard based on a user-provided script (which contains only one scene). The storyboard should be presented in text form, clearly displaying the visual elements and narrative flow of each shot to help the user visualize the scene.
|
||||
|
||||
[Input]
|
||||
The user will provide the following input.
|
||||
- Script:A complete scene script containing dialogue, action descriptions, and scene settings. The script focuses on only one scene; there is no need to handle multiple scene transitions. The script input is enclosed within <SCRIPT> and </SCRIPT>.
|
||||
- Characters List: A list describing basic information for each character, such as name, personality traits, appearance (if relevant). The character list is enclosed within <CHARACTERS> and </CHARACTERS>.
|
||||
- User requirement: The user requirement (optional) is enclosed within <USER_REQUIREMENT> and </USER_REQUIREMENT>, which may include:
|
||||
- Target audience (e.g., children, teenagers, adults).
|
||||
- Storyboard style (e.g., realistic, cartoon, abstract).
|
||||
- Desired number of shots (e.g., "not more than 10 shots").
|
||||
- Other specific instructions (e.g., emphasize the characters' actions).
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
- Ensure all output values (except keys) match the language used in the script.
|
||||
- Each shot must have a clear narrative purpose—such as establishing the setting, showing character relationships, or highlighting reactions.
|
||||
- Use cinematic language deliberately: close-ups for emotion, wide shots for context, and varied angles to direct audience attention.
|
||||
- When designing a new shot, first consider whether it can be filmed using an existing camera position. Introduce a new one only if the shot size, angle, and focus differ significantly. If the camera undergoes significant movement, it cannot be used thereafter.
|
||||
- Keep character names in visual descriptions and speaker fields consistent with the character list. In visual descriptions, enclose names in angle brackets (e.g., <Alice>), but not in dialogue or speaker fields.
|
||||
- When describing visual elements, it is necessary to indicate the position of the element within the frame. For example, Character A is on the left side of the frame, facing toward the right, with a table in front of him. The table is positioned slightly to the left of the center of the frame. Ensure that invisible elements are not included. For instance, do not describe someone behind a closed door if they cannot be seen.
|
||||
- Avoid unsafe content (violence, discrimination, etc.) in visual descriptions. Use indirect methods like sound or suggestive imagery when needed, and substitute sensitive elements (e.g., ketchup for blood).
|
||||
- Assign at most one dialogue line per character per shot. Each line of dialogue should correspond to a shot.
|
||||
- Each shot requires an independent description without reference to each other.
|
||||
- When the shot focuses on a character, describe which specific body part the focus is on.
|
||||
- When describing a character, it is necessary to indicate the direction they are facing.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_design_storyboard = \
|
||||
"""
|
||||
<SCRIPT>
|
||||
{script_str}
|
||||
</SCRIPT>
|
||||
|
||||
<CHARACTERS>
|
||||
{characters_str}
|
||||
</CHARACTERS>
|
||||
|
||||
<USER_REQUIREMENT>
|
||||
{user_requirement_str}
|
||||
</USER_REQUIREMENT>
|
||||
"""
|
||||
|
||||
|
||||
|
||||
system_prompt_template_decompose_visual_description = \
|
||||
"""
|
||||
[Role]
|
||||
You are a professional visual text analyst, proficient in cinematic language and shot narration. Your expertise lies in deconstructing a comprehensive shot description accurately into three core components: the static first frame, the static last frame, and the dynamic motion that connects them.
|
||||
|
||||
[Task]
|
||||
Your task is to dissect and rewrite a user-provided visual text description of a shot strictly and insightfully into three distinct parts:
|
||||
- First Frame Description: Describe the static image at the very beginning of the shot. Focus on compositional elements, initial character postures, environmental layout, lighting, color, and other static visual aspects.
|
||||
- Last Frame Description: Describe the static image at the very end of the shot. Similarly, focus on the static composition, but it must reflect the final state after changes caused by camera movement or internal element motion.
|
||||
- Motion Description: Describe all movements that occur between the first frame and the last frame. This includes camera movement (e.g., static, push-in, pull-out, pan, track, follow, tilt, etc.) and movement of elements within the shot (e.g., character movement, object displacement, changes in lighting, etc.). This is the most dynamic part of the entire description. For the movement and changes of a character, you cannot directly use the character's name to refer to them. Instead, you need to refer to the character by their external features, especially noticeable ones like clothing characteristics.
|
||||
|
||||
[Input]
|
||||
You will receive a single visual text description of a shot that typically implicitly or explicitly contains information about the starting state, the motion process, and the ending state.
|
||||
Additionally, you will receive a sequence of potential characters, each containing an identifier and a feature.
|
||||
- The description is enclosed within <VISUAL_DESC> and </VISUAL_DESC>.
|
||||
- The character list is enclosed within <CHARACTERS> and </CHARACTERS>.
|
||||
|
||||
|
||||
[Output]
|
||||
{format_instructions}
|
||||
|
||||
[Guidelines]
|
||||
- Ensure all output values (except keys) match the language used in the script.
|
||||
- Ensure the first and last frame descriptions are pure "snapshots," containing no ongoing actions (e.g., "He is about to stand up" is unacceptable; it should be "He is sitting on the chair, leaning slightly forward").
|
||||
- In the motion description, you must clearly distinguish between camera movement and on-screen movement. Use professional cinematic terminology (e.g., dolly shot, pan, zoom, etc.) as precisely as possible to describe camera movement.
|
||||
- In the motion description, you cannot directly use character names to refer to characters; instead, you should use the characters' visible characteristics to refer to them. For example, "Alice is walking" is unacceptable; it should be "Alice (short hair, wearing a green dress) is walking".
|
||||
- The last frame description must be logically consistent with the first frame description and the motion description. All actions described in the motion section should be reflected in the static image of the last frame.
|
||||
- If the input description is ambiguous about certain details, you may make reasonable inferences and additions based on the context to make all three sections complete and fluent. However, core elements must strictly adhere to the input text.
|
||||
- Use accurate, concise, and professional descriptive language. Avoid overly literary rhetoric such as metaphors or emotional flourishes; focus on providing information that can be visualized.
|
||||
- Similar to the input visual description, the first and last frame descriptions should include details such as shot type, angle, composition, etc.
|
||||
- Below are the three types of variation within a shot (not between two shots):
|
||||
(1) 'large' cases typically involve the exaggerated transition shots which means a significant change in the composition and focus, such as smoothly changing from a wide shot to a close-up. It is usually accompanied by significant camera movement (e.g., drone perspective shots across the city).
|
||||
(2) 'medium' cases often involve the introduction of new characters and a character turns from the back to face the front (facing the camera).
|
||||
(3) 'small' cases usually involve minor changes, such as expression changes, movement and pose changes of existing characters(e.g., walking, sitting down, standing up), moderate camera movements(e.g., pan, tilt, track).
|
||||
- When describing a character, it is necessary to indicate the direction they are facing.
|
||||
- The first shot must establish the overall scene environment, using the widest possible shot.
|
||||
- Use as few camera positions as possible.
|
||||
"""
|
||||
|
||||
|
||||
human_prompt_template_decompose_visual_description = \
|
||||
"""
|
||||
<VISUAL_DESC>
|
||||
{visual_desc}
|
||||
</VISUAL_DESC>
|
||||
|
||||
<CHARACTERS>
|
||||
{characters_str}
|
||||
</CHARACTERS>
|
||||
"""
|
||||
|
||||
|
||||
class VisDescDecompositionResponse(BaseModel):
|
||||
ff_desc: str = Field(
|
||||
description="A detailed description of the first frame of the shot, capturing the initial visual elements and composition.",
|
||||
# examples=[
|
||||
# "Medium shot of a supermarket aisle at eye level. Bob(a tall man wearing a blue shirt and jeans) is positioned on the right side of the frame, captured in profile and facing right, while Alice(a young woman with short hair, wearing a green dress) is on the left, shown pushing a shopping cart with her gaze lowered toward the ground. They are arranged in a front-to-back spatial relationship. Shelves line both sides of the frame, and cool-toned fluorescent lighting from above washes over the scene. The vibrant colors of product packaging contrast with the metallic gray of the shopping cart, all contained within a stable, horizontally balanced composition.",
|
||||
# "Extreme long shot. Aerial view from hundreds of meters above the ground. The boundless golden desert resembles undulating frozen waves, occupying the vast majority of the frame. At the very center of the image, a tiny, solitary explorer appears only as a faint dark speck, dragging a long, lonely trail of footprints behind him, stretching all the way to the edge of the frame.",
|
||||
# "Medium shot at eye level angle. Designer A(with a beard, wearing a white suit) leans forward passionately, speaking emphatically. Product Manager B(with a beard, wearing a white T-shirt) sits with crossed arms, looking skeptical. Between them, Development Engineer C(brown hair, wearing a blue T-shirt) appears anxious, glancing between the two. Project Manager D(curly hair, wearing a red T-shirt) prepares to mediate, focusing on a whiteboard. Bright overhead lighting highlights their expressions, with a blurred whiteboard and glass wall in the background.",
|
||||
# "A low-angle close-up shot captures the figure from below, framing him from the chest up. His face appears resolute and commanding, his eyes piercing as he speaks passionately. Flecks of saliva are visible, emphasizing his intensity. The overcast sky breaks with occasional light, casting him as a heroic, almost monumental figure against the gloom.",
|
||||
# "An extremely close-up of an old, motionless pocket watch. Soft light highlights scratches on its brass case and the enamel dial with Roman numerals. The second hand remains fixed at 'VIII', casting a sharp shadow. A wrinkled finger gently touches the glass surface, evoking a tangible sense of stillness and time.",
|
||||
# "An over-the-shoulder shot at eye level, positioned behind Character A(red hair, wearing a white T-shirt). The foreground, including A's shoulder and head, is softly blurred, directing focus onto Character B(with a beard, wearing a white T-shirt)'s face. B's subtle reactions—shifting from surprise to confusion, then to a glimmer of understanding—are clearly visible. The café background is gently blurred with warm lighting.",
|
||||
# ]
|
||||
)
|
||||
ff_vis_char_idxs: List[int] = Field(
|
||||
description="A list of indices of characters that are visible in the first frame of the shot, corresponding to the character list provided in the input.",
|
||||
examples=[[0], [1], [0, 1], []]
|
||||
)
|
||||
lf_desc: str = Field(
|
||||
description="A detailed description of the last frame of the shot, capturing the concluding visual elements and composition.",
|
||||
)
|
||||
lf_vis_char_idxs: List[int] = Field(
|
||||
description="A list of indices of characters that are visible in the last frame of the shot, corresponding to the character list provided in the input.",
|
||||
examples=[[0], [1], [0, 1], []]
|
||||
)
|
||||
motion_desc: str = Field(
|
||||
description="The motion description of the shot. Describe the dynamic visual changes within the shot (camera movement and the movement of elements within the frame)",
|
||||
examples=[
|
||||
"Static camera. Alice (short hair, wearing a green dress) is walking towards the camera.",
|
||||
"Dolly in from meidum shot to close-up. Bob (with a beard, wearing a white T-shirt) smiles to the camera.",
|
||||
]
|
||||
)
|
||||
variation_type: Literal["large", "medium", "small"] = Field(
|
||||
description="Indicates the degree of change between the first frame and the last frame.",
|
||||
)
|
||||
variation_reason: str = Field(
|
||||
description="The reason for the variation type of the shot.",
|
||||
examples=[
|
||||
"This is a smooth transition shot from the sky to the ground. The content of the shot has changed significantly, so the variation type is large.",
|
||||
"Compared to the first frame, a new character appears in the last frame, and there are no significant changes in the composition. So the variation type is medium.",
|
||||
"Compared to the first frame, there are only minor changes in the composition. So the variation type is small.",
|
||||
"This shot only shows Alice speaking and the changes in her facial expressions, thus the variation type is small.",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
|
||||
class StoryboardArtist:
|
||||
def __init__(
|
||||
self,
|
||||
chat_model: BaseChatModel,
|
||||
):
|
||||
self.chat_model = chat_model
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), after=after_func)
|
||||
async def design_storyboard(
|
||||
self,
|
||||
script: str,
|
||||
characters: List[CharacterInScene],
|
||||
user_requirement: Optional[str] = None,
|
||||
retry_timeout: int = 150,
|
||||
) -> List[ShotBriefDescription]:
|
||||
|
||||
class StoryboardResponse(BaseModel):
|
||||
storyboard: List[ShotBriefDescription] = Field(
|
||||
description="A complete storyboard of the scene, including the visual and audio description of each shot.",
|
||||
)
|
||||
|
||||
script_str = script.strip()
|
||||
characters_str = "\n".join([f"Character {index}: {char}" for index, char in enumerate(characters)])
|
||||
user_requirement_str = user_requirement.strip() if user_requirement else ""
|
||||
|
||||
parser = PydanticOutputParser(pydantic_object=StoryboardResponse)
|
||||
messages = [
|
||||
('system', system_prompt_template_design_storyboard.format(format_instructions=parser.get_format_instructions())),
|
||||
('human', human_prompt_template_design_storyboard.format(script_str=script_str, characters_str=characters_str, user_requirement_str=user_requirement_str)),
|
||||
]
|
||||
chain = self.chat_model | parser
|
||||
response: StoryboardResponse = await asyncio.wait_for(
|
||||
chain.ainvoke(messages),
|
||||
timeout=retry_timeout,
|
||||
)
|
||||
storyboard = response.storyboard
|
||||
|
||||
return storyboard
|
||||
|
||||
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), after=after_func)
|
||||
async def decompose_visual_description(
|
||||
self,
|
||||
shot_brief_desc: ShotBriefDescription,
|
||||
characters: List[CharacterInScene],
|
||||
retry_timeout: int = 150,
|
||||
) -> ShotDescription:
|
||||
parser = PydanticOutputParser(pydantic_object=VisDescDecompositionResponse)
|
||||
prompt_template = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
('system', system_prompt_template_decompose_visual_description),
|
||||
('human', human_prompt_template_decompose_visual_description),
|
||||
]
|
||||
)
|
||||
chain = prompt_template | self.chat_model | parser
|
||||
|
||||
visual_desc = shot_brief_desc.visual_desc.strip()
|
||||
|
||||
characters_str = "\n".join([f"{char.identifier_in_scene}: (static) {char.static_features}; (dynamic) {char.dynamic_features}" for char in characters])
|
||||
|
||||
decomposition: VisDescDecompositionResponse = await asyncio.wait_for(
|
||||
chain.ainvoke(
|
||||
input={
|
||||
"format_instructions": parser.get_format_instructions(),
|
||||
"visual_desc": visual_desc,
|
||||
"characters_str": characters_str,
|
||||
},
|
||||
),
|
||||
timeout=retry_timeout,
|
||||
)
|
||||
|
||||
validate_char_idxs(decomposition.ff_vis_char_idxs, len(characters), "ff_vis_char_idxs")
|
||||
validate_char_idxs(decomposition.lf_vis_char_idxs, len(characters), "lf_vis_char_idxs")
|
||||
|
||||
return ShotDescription(
|
||||
idx=shot_brief_desc.idx,
|
||||
is_last=shot_brief_desc.is_last,
|
||||
cam_idx=shot_brief_desc.cam_idx,
|
||||
visual_desc=shot_brief_desc.visual_desc,
|
||||
variation_type=decomposition.variation_type,
|
||||
variation_reason=decomposition.variation_reason,
|
||||
ff_desc=decomposition.ff_desc,
|
||||
ff_vis_char_idxs=decomposition.ff_vis_char_idxs,
|
||||
lf_desc=decomposition.lf_desc,
|
||||
lf_vis_char_idxs=decomposition.lf_vis_char_idxs,
|
||||
motion_desc=decomposition.motion_desc,
|
||||
audio_desc=shot_brief_desc.audio_desc,
|
||||
)
|
||||
|
||||
|
||||
def validate_char_idxs(idxs, num_characters, field_name):
|
||||
"""Reject LLM-emitted character indices outside [0, num_characters).
|
||||
|
||||
Negative values would silently select the wrong character via Python
|
||||
indexing; out-of-range values would crash deep inside the render gather.
|
||||
Raising here lets the @retry on decompose_visual_description re-ask.
|
||||
"""
|
||||
invalid = [idx for idx in idxs if idx < 0 or idx >= num_characters]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{field_name} contains invalid character indices {invalid}; "
|
||||
f"valid range is 0..{num_characters - 1}"
|
||||
)
|
||||
Reference in New Issue
Block a user