chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,426 @@
|
||||
import hashlib
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
import json
|
||||
import stat
|
||||
from wcmatch import glob
|
||||
import re
|
||||
import inspect
|
||||
from types import TracebackType, FrameType
|
||||
from typing import IO, List, Optional, Union
|
||||
|
||||
from e2b.exceptions import TemplateException
|
||||
from e2b.template.consts import BASE_STEP_NAME, FINALIZE_STEP_NAME
|
||||
|
||||
|
||||
def make_traceback(caller_frame: Optional[FrameType]) -> Optional[TracebackType]:
|
||||
"""
|
||||
Create a TracebackType from a caller frame for error reporting.
|
||||
|
||||
:param caller_frame: The caller's frame object, or None
|
||||
:return: A TracebackType object for use with exception.with_traceback(), or None
|
||||
"""
|
||||
if caller_frame is None:
|
||||
return None
|
||||
return TracebackType(
|
||||
tb_next=None,
|
||||
tb_frame=caller_frame,
|
||||
tb_lasti=caller_frame.f_lasti,
|
||||
tb_lineno=caller_frame.f_lineno,
|
||||
)
|
||||
|
||||
|
||||
def validate_relative_path(
|
||||
src: str,
|
||||
stack_trace: Optional[TracebackType],
|
||||
) -> None:
|
||||
"""
|
||||
Validate that a source path for copy operations is a relative path that stays
|
||||
within the context directory. This prevents path traversal attacks and ensures
|
||||
files are copied from within the expected directory.
|
||||
|
||||
:param src: The source path to validate
|
||||
:param stack_trace: Optional stack trace for error reporting
|
||||
|
||||
:raises TemplateException: If the path is absolute or escapes the context directory
|
||||
|
||||
Invalid paths:
|
||||
- Absolute paths: /absolute/path, C:\\Windows\\path
|
||||
- Parent directory escapes: ../foo, foo/../../bar, ./foo/../../../bar
|
||||
|
||||
Valid paths:
|
||||
- Simple relative: foo, foo/bar
|
||||
- Current directory prefix: ./foo, ./foo/bar
|
||||
- Internal parent refs that don't escape: foo/../bar (stays within context)
|
||||
"""
|
||||
# Check for absolute paths using Python's cross-platform implementation
|
||||
if os.path.isabs(src):
|
||||
raise TemplateException(
|
||||
f'Invalid source path "{src}": absolute paths are not allowed. '
|
||||
"Use a relative path within the context directory."
|
||||
).with_traceback(stack_trace)
|
||||
|
||||
# Normalize the path and check if it escapes the context directory
|
||||
normalized = os.path.normpath(src)
|
||||
|
||||
# After normalization, a path that escapes would be '..' or start with '../'
|
||||
# We check for '..' followed by path separator to avoid false positives on filenames like '..myconfig'
|
||||
# Examples:
|
||||
# - '../foo' -> '../foo' (escapes)
|
||||
# - 'foo/../../bar' -> '../bar' (escapes)
|
||||
# - './foo/../../../bar' -> '../../bar' (escapes)
|
||||
# - 'foo/../bar' -> 'bar' (doesn't escape)
|
||||
# - './foo/bar' -> 'foo/bar' (doesn't escape)
|
||||
# - '..myconfig' -> '..myconfig' (valid filename, doesn't escape)
|
||||
escapes = normalized == ".." or normalized.startswith(".." + os.sep)
|
||||
|
||||
if escapes:
|
||||
raise TemplateException(
|
||||
f'Invalid source path "{src}": path escapes the context directory. '
|
||||
"The path must stay within the context directory."
|
||||
).with_traceback(stack_trace)
|
||||
|
||||
|
||||
def normalize_build_arguments(
|
||||
name: Optional[str] = None,
|
||||
alias: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Normalize build arguments from different parameter signatures.
|
||||
Handles string name or legacy alias parameter.
|
||||
|
||||
:param name: Template name in 'name' or 'name:tag' format
|
||||
:param alias: (Deprecated) Alias name for the template. Use name instead.
|
||||
:return: Normalized template name
|
||||
:raises TemplateException: If no template name is provided
|
||||
"""
|
||||
if name and len(name) > 0:
|
||||
return name
|
||||
if alias and len(alias) > 0:
|
||||
return alias
|
||||
raise TemplateException("Name must be provided")
|
||||
|
||||
|
||||
def read_dockerignore(context_path: str) -> List[str]:
|
||||
"""
|
||||
Read and parse a .dockerignore file.
|
||||
|
||||
:param context_path: Directory path containing the .dockerignore file
|
||||
|
||||
:return: Array of ignore patterns (empty lines and comments are filtered out)
|
||||
"""
|
||||
dockerignore_path = os.path.join(context_path, ".dockerignore")
|
||||
if not os.path.exists(dockerignore_path):
|
||||
return []
|
||||
|
||||
with open(dockerignore_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
return [
|
||||
line.strip()
|
||||
for line in content.split("\n")
|
||||
if line.strip() and not line.strip().startswith("#")
|
||||
]
|
||||
|
||||
|
||||
def normalize_path(path: str) -> str:
|
||||
"""
|
||||
Normalize path separators to forward slashes for glob patterns (glob expects / even on Windows).
|
||||
|
||||
:param path: The path to normalize
|
||||
:return: The normalized path
|
||||
"""
|
||||
return path.replace(os.sep, "/")
|
||||
|
||||
|
||||
def get_all_files_in_path(
|
||||
src: str,
|
||||
context_path: str,
|
||||
ignore_patterns: List[str],
|
||||
include_directories: bool = True,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Get all files for a given path and ignore patterns.
|
||||
|
||||
:param src: Path to the source directory
|
||||
:param context_path: Base directory for resolving relative paths
|
||||
:param ignore_patterns: Ignore patterns
|
||||
:param include_directories: Whether to include directories
|
||||
:return: Array of files
|
||||
"""
|
||||
files = set()
|
||||
|
||||
# Use glob to find all files/directories matching the pattern under context_path
|
||||
abs_context_path = os.path.abspath(context_path)
|
||||
files_glob = glob.glob(
|
||||
src,
|
||||
flags=glob.GLOBSTAR | glob.DOTMATCH,
|
||||
root_dir=abs_context_path,
|
||||
exclude=ignore_patterns,
|
||||
)
|
||||
|
||||
for file in files_glob:
|
||||
# Join it with abs_context_path to get the absolute path
|
||||
file_path = os.path.join(abs_context_path, file)
|
||||
|
||||
if os.path.isdir(file_path):
|
||||
# If it's a directory, add the directory and all entries recursively
|
||||
if include_directories:
|
||||
files.add(file_path)
|
||||
dir_files = glob.glob(
|
||||
normalize_path(file) + "/**/*",
|
||||
flags=glob.GLOBSTAR | glob.DOTMATCH,
|
||||
root_dir=abs_context_path,
|
||||
exclude=ignore_patterns,
|
||||
)
|
||||
for dir_file in dir_files:
|
||||
dir_file_path = os.path.join(abs_context_path, dir_file)
|
||||
files.add(dir_file_path)
|
||||
else:
|
||||
files.add(file_path)
|
||||
|
||||
return sorted(list(files))
|
||||
|
||||
|
||||
def calculate_files_hash(
|
||||
src: str,
|
||||
dest: str,
|
||||
context_path: str,
|
||||
ignore_patterns: List[str],
|
||||
resolve_symlinks: bool,
|
||||
stack_trace: Optional[TracebackType],
|
||||
) -> str:
|
||||
"""
|
||||
Calculate a hash of files being copied to detect changes for cache invalidation.
|
||||
|
||||
The hash includes file content, metadata (mode, size), and relative paths.
|
||||
Note: uid, gid, and mtime are excluded to ensure stable hashes across environments.
|
||||
|
||||
:param src: Source path pattern for files to copy
|
||||
:param dest: Destination path where files will be copied
|
||||
:param context_path: Base directory for resolving relative paths
|
||||
:param ignore_patterns: Glob patterns to ignore
|
||||
:param resolve_symlinks: Whether to resolve symbolic links when hashing
|
||||
:param stack_trace: Optional stack trace for error reporting
|
||||
|
||||
:return: Hex string hash of all files
|
||||
|
||||
:raises ValueError: If no files match the source pattern
|
||||
"""
|
||||
src_path = os.path.join(context_path, src)
|
||||
hash_obj = hashlib.sha256()
|
||||
content = f"COPY {src} {dest}"
|
||||
|
||||
hash_obj.update(content.encode())
|
||||
|
||||
files = get_all_files_in_path(src, context_path, ignore_patterns, True)
|
||||
|
||||
if len(files) == 0:
|
||||
raise ValueError(f"No files found in {src_path}").with_traceback(stack_trace)
|
||||
|
||||
def hash_stats(stat_info: os.stat_result) -> None:
|
||||
# Only include stable metadata (mode, size)
|
||||
# Exclude uid, gid, and mtime to ensure consistent hashes across environments
|
||||
hash_obj.update(str(stat_info.st_mode).encode())
|
||||
hash_obj.update(str(stat_info.st_size).encode())
|
||||
|
||||
for file in files:
|
||||
# Hash the relative path
|
||||
relative_path = os.path.relpath(file, context_path)
|
||||
hash_obj.update(relative_path.encode())
|
||||
|
||||
# Add stat information to hash calculation
|
||||
if os.path.islink(file):
|
||||
stats = os.lstat(file)
|
||||
should_follow = resolve_symlinks and (
|
||||
os.path.isfile(file) or os.path.isdir(file)
|
||||
)
|
||||
|
||||
if not should_follow:
|
||||
hash_stats(stats)
|
||||
|
||||
content = os.readlink(file)
|
||||
hash_obj.update(content.encode())
|
||||
continue
|
||||
|
||||
stats = os.stat(file)
|
||||
hash_stats(stats)
|
||||
|
||||
if stat.S_ISREG(stats.st_mode):
|
||||
with open(file, "rb") as f:
|
||||
hash_obj.update(f.read())
|
||||
|
||||
return hash_obj.hexdigest()
|
||||
|
||||
|
||||
def tar_file_stream(
|
||||
file_name: str,
|
||||
file_context_path: str,
|
||||
ignore_patterns: List[str],
|
||||
resolve_symlinks: bool,
|
||||
gzip: bool,
|
||||
) -> IO[bytes]:
|
||||
"""
|
||||
Create a tar archive of files matching a pattern in a temporary file.
|
||||
|
||||
The archive is spooled to disk so it can be uploaded as a stream instead
|
||||
of being buffered in memory. The temporary file is deleted when closed.
|
||||
|
||||
:param file_name: Glob pattern for files to include
|
||||
:param file_context_path: Base directory for resolving file paths
|
||||
:param ignore_patterns: Ignore patterns
|
||||
:param resolve_symlinks: Whether to resolve symbolic links
|
||||
:param gzip: Whether to gzip the archive
|
||||
|
||||
:return: Binary file object positioned at the start of the archive
|
||||
"""
|
||||
tar_file = tempfile.TemporaryFile()
|
||||
try:
|
||||
with tarfile.open(
|
||||
fileobj=tar_file,
|
||||
mode="w:gz" if gzip else "w",
|
||||
dereference=resolve_symlinks,
|
||||
) as tar:
|
||||
files = get_all_files_in_path(
|
||||
file_name, file_context_path, ignore_patterns, True
|
||||
)
|
||||
for file in files:
|
||||
tar.add(
|
||||
file,
|
||||
arcname=os.path.relpath(file, file_context_path),
|
||||
recursive=False,
|
||||
)
|
||||
|
||||
tar_file.seek(0)
|
||||
return tar_file
|
||||
except Exception:
|
||||
# Best-effort cleanup: a close failure must not replace the real
|
||||
# archive-creation error.
|
||||
try:
|
||||
tar_file.close()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def strip_ansi_escape_codes(text: str) -> str:
|
||||
"""
|
||||
Strip ANSI escape codes from a string.
|
||||
|
||||
Source: https://github.com/chalk/ansi-regex/blob/main/index.js
|
||||
|
||||
:param text: String with ANSI escape codes
|
||||
|
||||
:return: String without ANSI escape codes
|
||||
"""
|
||||
# Valid string terminator sequences are BEL, ESC\, and 0x9c
|
||||
st = r"(?:\u0007|\u001B\u005C|\u009C)"
|
||||
pattern = [
|
||||
rf"[\u001B\u009B][\[\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*|[a-zA-Z\d]+(?:;[-a-zA-Z\d/#&.:=?%@~_]*)*)?{st})",
|
||||
r"(?:(?:\d{1,4}(?:[;:]\d{0,4})*)?[\dA-PR-TZcf-nq-uy=><~]))",
|
||||
]
|
||||
ansi_escape = re.compile("|".join(pattern), re.UNICODE)
|
||||
return ansi_escape.sub("", text)
|
||||
|
||||
|
||||
def get_caller_frame(depth: int) -> Optional[FrameType]:
|
||||
"""
|
||||
Get the caller's stack frame at a specific depth.
|
||||
|
||||
This is used to provide better error messages and debugging information
|
||||
by tracking where template methods were called from in user code.
|
||||
|
||||
:param depth: The depth of the stack trace to retrieve
|
||||
|
||||
:return: The caller frame, or None if not available
|
||||
"""
|
||||
stack = inspect.stack()[1:]
|
||||
if len(stack) < depth + 1:
|
||||
return None
|
||||
return stack[depth].frame
|
||||
|
||||
|
||||
def get_caller_directory(depth: int) -> Optional[str]:
|
||||
"""
|
||||
Get the directory of the caller at a specific stack depth.
|
||||
|
||||
This is used to determine the file_context_path when creating a template,
|
||||
so file paths are resolved relative to the user's template file location.
|
||||
|
||||
:param depth: The depth of the stack trace
|
||||
|
||||
:return: The caller's directory path, or None if not available
|
||||
"""
|
||||
try:
|
||||
# Get the stack trace
|
||||
caller_frame = get_caller_frame(depth)
|
||||
if caller_frame is None:
|
||||
return None
|
||||
|
||||
caller_file = caller_frame.f_code.co_filename
|
||||
|
||||
# Return the directory of the caller file
|
||||
return os.path.dirname(os.path.abspath(caller_file))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def pad_octal(mode: int) -> str:
|
||||
"""
|
||||
Convert a numeric file mode to a zero-padded octal string.
|
||||
|
||||
:param mode: File mode as a number (e.g., 493 for 0o755)
|
||||
|
||||
:return: Zero-padded 4-digit octal string (e.g., "0755")
|
||||
|
||||
Example
|
||||
```python
|
||||
pad_octal(0o755) # Returns "0755"
|
||||
pad_octal(0o644) # Returns "0644"
|
||||
```
|
||||
"""
|
||||
return f"{mode:04o}"
|
||||
|
||||
|
||||
def get_build_step_index(step: str, stack_traces_length: int) -> int:
|
||||
"""
|
||||
Get the array index for a build step based on its name.
|
||||
|
||||
Special steps:
|
||||
- BASE_STEP_NAME: Returns 0 (first step)
|
||||
- FINALIZE_STEP_NAME: Returns the last index
|
||||
- Numeric strings: Converted to number
|
||||
|
||||
:param step: Build step name or number as string
|
||||
:param stack_traces_length: Total number of stack traces (used for FINALIZE_STEP_NAME)
|
||||
|
||||
:return: Index for the build step
|
||||
"""
|
||||
if step == BASE_STEP_NAME:
|
||||
return 0
|
||||
|
||||
if step == FINALIZE_STEP_NAME:
|
||||
return stack_traces_length - 1
|
||||
|
||||
return int(step)
|
||||
|
||||
|
||||
def read_gcp_service_account_json(
|
||||
context_path: str, path_or_content: Union[str, dict]
|
||||
) -> str:
|
||||
"""
|
||||
Read GCP service account JSON from a file or object.
|
||||
|
||||
:param context_path: Base directory for resolving relative file paths
|
||||
:param path_or_content: Either a path to a JSON file or a service account object
|
||||
|
||||
:return: Service account JSON as a string
|
||||
"""
|
||||
if isinstance(path_or_content, str):
|
||||
with open(
|
||||
os.path.join(context_path, path_or_content), "r", encoding="utf-8"
|
||||
) as f:
|
||||
return f.read()
|
||||
else:
|
||||
return json.dumps(path_or_content)
|
||||
Reference in New Issue
Block a user