chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,60 @@
|
||||
import logging
|
||||
import requests
|
||||
import base64
|
||||
import mimetypes
|
||||
from io import BytesIO
|
||||
import cv2
|
||||
|
||||
from utils.retry import download_retry
|
||||
|
||||
|
||||
@download_retry
|
||||
def download_image(url, save_path):
|
||||
try:
|
||||
logging.info(f"Downloading image from {url} to {save_path}")
|
||||
|
||||
response = requests.get(url, stream=True, timeout=(10, 300))
|
||||
response.raise_for_status() # Check for HTTP errors
|
||||
|
||||
with open(save_path, 'wb') as file:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
file.write(chunk)
|
||||
logging.info(f"Image downloaded successfully to {save_path}")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error downloading image: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def image_path_to_b64(image_path, mime: bool = True) -> str:
|
||||
with open(image_path, 'rb') as image_file:
|
||||
b64 = base64.b64encode(image_file.read()).decode('utf-8')
|
||||
|
||||
if mime:
|
||||
mime_type, _ = mimetypes.guess_type(image_path)
|
||||
if mime_type is None:
|
||||
mime_type = 'application/octet-stream'
|
||||
return f"data:{mime_type};base64,{b64}"
|
||||
|
||||
return b64
|
||||
|
||||
|
||||
def pil_to_b64(image, mime: bool = True) -> str:
|
||||
buffered = BytesIO()
|
||||
image.save(buffered, format="PNG")
|
||||
b64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
|
||||
|
||||
if mime:
|
||||
return f"data:image/png;base64,{b64}"
|
||||
|
||||
return b64
|
||||
|
||||
|
||||
def save_base64_image(b64_string, save_path):
|
||||
# If the base64 string has a data URL prefix, remove it
|
||||
if ',' in b64_string:
|
||||
b64_string = b64_string.split(',')[1]
|
||||
|
||||
with open(save_path, 'wb') as image_file:
|
||||
image_file.write(base64.b64decode(b64_string))
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
Provider preset system for ViMax chat model configuration.
|
||||
|
||||
Supports auto-detection and resolution of LLM provider settings,
|
||||
allowing users to specify a provider name (e.g., ``minimax``) instead
|
||||
of manually configuring base_url and model details.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider presets
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PROVIDER_PRESETS: Dict[str, Dict[str, Any]] = {
|
||||
"minimax": {
|
||||
"base_url": "https://api.minimax.io/v1",
|
||||
"env_key": "MINIMAX_API_KEY",
|
||||
"default_model": "MiniMax-M3",
|
||||
"models": [
|
||||
"MiniMax-M3",
|
||||
"MiniMax-M2.7",
|
||||
"MiniMax-M2.7-highspeed",
|
||||
],
|
||||
"temperature_range": (0.0, 1.0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def resolve_chat_model_config(init_args: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Resolve provider presets and return final ``init_chat_model`` kwargs.
|
||||
|
||||
If ``model_provider`` matches a known preset (e.g. ``minimax``), the
|
||||
returned dict will have:
|
||||
|
||||
* ``model_provider`` rewritten to ``"openai"`` (OpenAI-compatible API)
|
||||
* ``base_url`` filled in from the preset when not already set
|
||||
* ``api_key`` sourced from the environment when not already set
|
||||
* ``model`` defaulted to the preset's default model when not already set
|
||||
* ``temperature`` clamped to the provider's supported range
|
||||
|
||||
For unknown providers the dict is returned unchanged.
|
||||
"""
|
||||
args = dict(init_args) # shallow copy
|
||||
provider = args.get("model_provider", "openai")
|
||||
|
||||
preset = PROVIDER_PRESETS.get(provider)
|
||||
if preset is None:
|
||||
return args
|
||||
|
||||
# base_url
|
||||
if not args.get("base_url"):
|
||||
args["base_url"] = preset["base_url"]
|
||||
|
||||
# api_key – fall back to env var
|
||||
if not args.get("api_key"):
|
||||
env_key = preset.get("env_key", "")
|
||||
env_val = os.environ.get(env_key, "")
|
||||
if env_val:
|
||||
args["api_key"] = env_val
|
||||
logger.info("Using %s API key from environment variable %s", provider, env_key)
|
||||
|
||||
# default model
|
||||
if not args.get("model"):
|
||||
args["model"] = preset["default_model"]
|
||||
logger.info("Defaulting to model %s for provider %s", args["model"], provider)
|
||||
|
||||
# temperature clamping
|
||||
temp_range = preset.get("temperature_range")
|
||||
if temp_range and "temperature" in args and args["temperature"] is not None:
|
||||
lo, hi = temp_range
|
||||
original = args["temperature"]
|
||||
args["temperature"] = max(lo, min(hi, original))
|
||||
if args["temperature"] != original:
|
||||
logger.warning(
|
||||
"Clamped temperature %.2f -> %.2f for provider %s",
|
||||
original, args["temperature"], provider,
|
||||
)
|
||||
|
||||
# rewrite to openai-compatible provider for LangChain
|
||||
args["model_provider"] = "openai"
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def detect_provider_from_env() -> Optional[str]:
|
||||
"""Return the name of a provider whose API key is found in the environment.
|
||||
|
||||
Checks ``PROVIDER_PRESETS`` in definition order and returns the first
|
||||
match, or ``None`` if no key is set.
|
||||
"""
|
||||
for name, preset in PROVIDER_PRESETS.items():
|
||||
env_key = preset.get("env_key", "")
|
||||
if env_key and os.environ.get(env_key):
|
||||
return name
|
||||
return None
|
||||
@@ -0,0 +1,94 @@
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""
|
||||
Rate limiter to control API request frequency.
|
||||
|
||||
Ensures that no more than max_requests_per_minute requests are made per minute
|
||||
and no more than max_requests_per_day requests are made per day.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_requests_per_minute: Optional[int] = None,
|
||||
max_requests_per_day: Optional[int] = None
|
||||
):
|
||||
"""
|
||||
Initialize the rate limiter.
|
||||
|
||||
Args:
|
||||
max_requests_per_minute: Maximum number of requests allowed per minute.
|
||||
If None, no per-minute limit is enforced.
|
||||
max_requests_per_day: Maximum number of requests allowed per day.
|
||||
If None, no per-day limit is enforced.
|
||||
"""
|
||||
self.max_requests_per_minute = max_requests_per_minute
|
||||
self.max_requests_per_day = max_requests_per_day
|
||||
self.request_times = []
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
# If per-minute rate limiting is enabled, calculate the minimum delay between requests
|
||||
if max_requests_per_minute and max_requests_per_minute > 0:
|
||||
self.min_delay = 60.0 / max_requests_per_minute
|
||||
else:
|
||||
self.min_delay = 0
|
||||
|
||||
async def acquire(self):
|
||||
"""
|
||||
Acquire permission to make a request.
|
||||
|
||||
This method will block until it's safe to make a request according to the rate limits.
|
||||
|
||||
The lock is only held while checking and recording, never while sleeping:
|
||||
a caller waiting out a window (up to 24h for the daily limit) must not
|
||||
block every other caller's check. After each sleep the limits are
|
||||
re-checked, since another caller may have taken the freed slot.
|
||||
"""
|
||||
if not self.max_requests_per_minute and not self.max_requests_per_day:
|
||||
# Rate limiting is disabled
|
||||
return
|
||||
|
||||
while True:
|
||||
message = None
|
||||
async with self.lock:
|
||||
current_time = time.time()
|
||||
|
||||
# Clean up old request times (keep requests from last 24 hours for daily limit)
|
||||
if self.max_requests_per_day:
|
||||
self.request_times = [t for t in self.request_times if current_time - t < 86400]
|
||||
elif self.max_requests_per_minute:
|
||||
self.request_times = [t for t in self.request_times if current_time - t < 60]
|
||||
|
||||
wait_time = 0.0
|
||||
|
||||
# Check daily limit first
|
||||
if self.max_requests_per_day and self.max_requests_per_day > 0:
|
||||
daily_requests = [t for t in self.request_times if current_time - t < 86400]
|
||||
if len(daily_requests) >= self.max_requests_per_day:
|
||||
wait_time = 86400 - (current_time - daily_requests[0])
|
||||
hours = wait_time / 3600
|
||||
message = f"Daily rate limit reached ({self.max_requests_per_day} requests/day). Waiting {hours:.1f} hours..."
|
||||
|
||||
# Check per-minute limit
|
||||
if wait_time <= 0 and self.max_requests_per_minute and self.max_requests_per_minute > 0:
|
||||
minute_requests = [t for t in self.request_times if current_time - t < 60]
|
||||
if len(minute_requests) >= self.max_requests_per_minute:
|
||||
wait_time = 60 - (current_time - minute_requests[0])
|
||||
message = f"Rate limit reached ({self.max_requests_per_minute} requests/min). Waiting {wait_time:.1f}s..."
|
||||
elif self.request_times and self.min_delay > 0:
|
||||
# Also ensure minimum delay between consecutive requests
|
||||
time_since_last = current_time - self.request_times[-1]
|
||||
if time_since_last < self.min_delay:
|
||||
wait_time = self.min_delay - time_since_last
|
||||
|
||||
if wait_time <= 0:
|
||||
# Record this request
|
||||
self.request_times.append(current_time)
|
||||
return
|
||||
|
||||
if message:
|
||||
print(message)
|
||||
await asyncio.sleep(wait_time)
|
||||
@@ -0,0 +1,29 @@
|
||||
import tenacity
|
||||
import traceback
|
||||
import logging
|
||||
|
||||
import requests
|
||||
|
||||
def after_func(retry_state: tenacity.RetryCallState) -> None:
|
||||
if retry_state.outcome.failed:
|
||||
exc = retry_state.outcome.exception()
|
||||
logging.warning(f"Retrying {retry_state.fn.__name__} due to {repr(exc)} (Attempt {retry_state.attempt_number})")
|
||||
logging.debug(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
||||
|
||||
|
||||
def is_retryable_download_error(exc: BaseException) -> bool:
|
||||
"""Network errors and 5xx responses are retryable; other HTTP errors (expired
|
||||
or invalid URLs, auth failures) will never succeed and must fail fast."""
|
||||
if isinstance(exc, requests.HTTPError):
|
||||
response = exc.response
|
||||
return response is None or response.status_code >= 500
|
||||
return isinstance(exc, requests.RequestException)
|
||||
|
||||
|
||||
download_retry = tenacity.retry(
|
||||
stop=tenacity.stop_after_attempt(3),
|
||||
wait=tenacity.wait_exponential(multiplier=1, max=10),
|
||||
retry=tenacity.retry_if_exception(is_retryable_download_error),
|
||||
after=after_func,
|
||||
reraise=True,
|
||||
)
|
||||
@@ -0,0 +1,14 @@
|
||||
import re
|
||||
|
||||
|
||||
def safe_path_component(name) -> str:
|
||||
"""Sanitize an LLM-derived identifier for use as a filesystem path component.
|
||||
|
||||
Identifiers come from model output over user-supplied story text, so they may
|
||||
contain separators or traversal sequences; keep word characters (including
|
||||
CJK), dashes, dots and spaces, replace everything else, and strip leading
|
||||
dots so the result can never escape or hide within the working directory.
|
||||
"""
|
||||
cleaned = re.sub(r"[^\w\-. ]", "_", str(name))
|
||||
cleaned = cleaned.strip().lstrip(".")
|
||||
return cleaned or "unnamed"
|
||||
@@ -0,0 +1,70 @@
|
||||
import time
|
||||
from functools import wraps
|
||||
|
||||
|
||||
class Timer:
|
||||
def __init__(
|
||||
self,
|
||||
prefix: str = "Start at {start_time}",
|
||||
postfix: str = "End at {end_time}, took {duration} seconds.",
|
||||
):
|
||||
self.prefix = prefix
|
||||
self.format = format
|
||||
self.postfix = postfix
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
func,
|
||||
):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
start_time = time.time()
|
||||
prefix = self.prefix.replace("{start_time}", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start_time)))
|
||||
print(prefix)
|
||||
|
||||
result = await func(*args, **kwargs)
|
||||
|
||||
end_time = time.time()
|
||||
duration = end_time - start_time
|
||||
postfix = self.postfix.replace("{end_time}", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(end_time))).replace("{duration}", f"{duration:.2f}")
|
||||
print(postfix)
|
||||
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def __enter__(self):
|
||||
self.start_time = time.time()
|
||||
prefix = self.prefix.replace("{start_time}", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(self.start_time)))
|
||||
print(prefix)
|
||||
return self
|
||||
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if exc_type is not None:
|
||||
return False
|
||||
|
||||
end_time = time.time()
|
||||
duration = end_time - self.start_time
|
||||
postfix = self.postfix.replace("{end_time}", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(end_time))).replace("{duration}", f"{duration:.2f}")
|
||||
print(postfix)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with Timer(
|
||||
prefix="Begin timing at {start_time}",
|
||||
postfix="Finished at {end_time}",
|
||||
):
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
@Timer()
|
||||
def test_sleep():
|
||||
time.sleep(1)
|
||||
|
||||
test_sleep()
|
||||
@@ -0,0 +1,44 @@
|
||||
import logging
|
||||
import requests
|
||||
from moviepy import VideoFileClip, concatenate_videoclips
|
||||
from utils.retry import download_retry
|
||||
|
||||
|
||||
@download_retry
|
||||
def download_video(url, save_path):
|
||||
try:
|
||||
logging.info(f"Downloading video from {url} to {save_path}")
|
||||
|
||||
response = requests.get(url, stream=True, timeout=(10, 300))
|
||||
response.raise_for_status() # 检查请求是否成功
|
||||
|
||||
with open(save_path, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
logging.info(f"Video downloaded successfully to {save_path}")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error downloading video: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def concatenate_video_files(video_paths, output_path, codec="libx264", preset="medium"):
|
||||
"""Concatenate video files, releasing every ffmpeg reader even on failure.
|
||||
|
||||
Each VideoFileClip keeps an ffmpeg subprocess and file handle open until
|
||||
closed; leaking them exhausts file descriptors on long multi-scene runs.
|
||||
"""
|
||||
clips = []
|
||||
final = None
|
||||
try:
|
||||
for path in video_paths:
|
||||
clips.append(VideoFileClip(path))
|
||||
final = concatenate_videoclips(clips)
|
||||
final.write_videofile(output_path, codec=codec, preset=preset)
|
||||
finally:
|
||||
if final is not None:
|
||||
final.close()
|
||||
for clip in clips:
|
||||
clip.close()
|
||||
return output_path
|
||||
Reference in New Issue
Block a user