wehub/googlecloudplatform--generative-ai
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:30:30 +08:00
commit 914fea506e
2793 changed files with 802106 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/format.sh
Executable
+126
View File
@@ -0,0 +1,126 @@
#!/bin/bash
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
# Check if uv is installed
if ! command -v uv &> /dev/null; then
echo "uv could not be found. Please install it to run the formatting script."
echo "See https://github.com/astral-sh/uv for installation instructions."
exit 1
fi
# Define the uv run command with all necessary dependencies
UV_RUN="uv run --with autoflake --with ruff --with nbqa --with nbformat>=5.10.4 --with git+https://github.com/tensorflow/docs"
# Sorting and de-duplicating spelling allow file
SPELLING_ALLOW_FILE=".github/actions/spelling/allow.txt"
SPELLING_EXPECT_FILE=".github/actions/spelling/expect.txt"
if [ -f "$SPELLING_EXPECT_FILE" ]; then
echo "Combining $SPELLING_EXPECT_FILE into $SPELLING_ALLOW_FILE"
cat "$SPELLING_EXPECT_FILE" >> "$SPELLING_ALLOW_FILE"
rm "$SPELLING_EXPECT_FILE"
fi
if [ -f "$SPELLING_ALLOW_FILE" ]; then
echo "Sorting and de-duplicating $SPELLING_ALLOW_FILE"
sort -u "$SPELLING_ALLOW_FILE" -o "$SPELLING_ALLOW_FILE"
fi
# Determine files to lint
FORMAT_ALL=false
UNSAFE_FIXES=false
PASSED_FILES=""
for arg in "$@"; do
if [ "$arg" == "--all" ]; then
FORMAT_ALL=true
elif [ "$arg" == "--unsafe-fixes" ]; then
UNSAFE_FIXES=true
elif [[ "$arg" == *.ipynb ]] || [[ "$arg" == *.py ]]; then
PASSED_FILES="$PASSED_FILES $arg"
fi
done
LINT_PATHS_NB=""
LINT_PATHS_PY=""
if [ -n "$PASSED_FILES" ]; then
for f in $PASSED_FILES; do
if [[ "$f" == *.ipynb ]]; then
LINT_PATHS_NB="$LINT_PATHS_NB $f"
elif [[ "$f" == *.py ]]; then
LINT_PATHS_PY="$LINT_PATHS_PY $f"
fi
done
elif [ "$FORMAT_ALL" = true ]; then
LINT_PATHS_NB=$(find . -name "*.ipynb" -not -path "*/.*")
LINT_PATHS_PY=$(find . -name "*.py" -not -path "*/.*" -not -name "noxfile.py")
else
TARGET_BRANCH="main"
# Get changed files
CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRTUXB "$TARGET_BRANCH" | sort -u)
STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMRTUXB "$TARGET_BRANCH" | sort -u)
COMMITTED_FILES=$(git diff HEAD "$TARGET_BRANCH" --name-only --diff-filter=ACMRTUXB | sort -u)
ALL_CHANGED=$(echo "$CHANGED_FILES $STAGED_FILES $COMMITTED_FILES" | tr ' ' '\n' | sort -u)
for f in $ALL_CHANGED; do
if [ -f "$f" ]; then
if [[ "$f" == *.ipynb ]]; then
LINT_PATHS_NB="$LINT_PATHS_NB $f"
elif [[ "$f" == *.py ]] && [ "$f" != "noxfile.py" ]; then
LINT_PATHS_PY="$LINT_PATHS_PY $f"
fi
fi
done
fi
# --- Format Notebooks ---
if [ -n "$LINT_PATHS_NB" ]; then
echo "Formatting notebooks..."
# Run custom notebook processor
$UV_RUN python3 scripts/notebook_processor.py $LINT_PATHS_NB
# Run nbqa tools
$UV_RUN nbqa autoflake $LINT_PATHS_NB -i -r --remove-all-unused-imports
UNSAFE_FLAG=""
if [ "$UNSAFE_FIXES" = true ]; then
UNSAFE_FLAG=" --unsafe-fixes"
fi
$UV_RUN nbqa "ruff check --fix-only$UNSAFE_FLAG" $LINT_PATHS_NB
$UV_RUN nbqa "ruff format" $LINT_PATHS_NB
$UV_RUN python3 -m tensorflow_docs.tools.nbfmt $LINT_PATHS_NB
else
echo "No notebooks to format."
fi
# --- Format Python Files ---
if [ -n "$LINT_PATHS_PY" ]; then
echo "Formatting Python files..."
$UV_RUN autoflake -i -r --remove-all-unused-imports $LINT_PATHS_PY
UNSAFE_FLAG=""
if [ "$UNSAFE_FIXES" = true ]; then
UNSAFE_FLAG=" --unsafe-fixes"
fi
$UV_RUN ruff check --fix-only$UNSAFE_FLAG $LINT_PATHS_PY
$UV_RUN ruff format $LINT_PATHS_PY
else
echo "No Python files to format."
fi
scripts/notebook_processor.py
+207
View File
@@ -0,0 +1,207 @@
# pylint: skip-file
# type: ignore
# -*- coding: utf-8 -*-
#
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import urllib.parse
import nbformat
DEFAULT_RUFF_LINE_LENGTH = 88
# --- Notebook Link Constants ---
LINK_PREFIXES = {
"colab_link": "https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/",
"colab_enterprise_link": "https://console.cloud.google.com/agent-platform/colab/import/",
"github_link": "https://github.com/GoogleCloudPlatform/generative-ai/blob/main/",
"workbench_link": "https://console.cloud.google.com/agent-platform/workbench/instances?download_url=",
"bigquery_studio_link": "https://console.cloud.google.com/bigquery/import?url=",
"linkedin_link": "https://www.linkedin.com/sharing/share-offsite/?url=",
"bluesky_link": "https://bsky.app/intent/compose?text=",
"twitter_link": "https://twitter.com/intent/tweet?url=",
"reddit_link": "https://reddit.com/submit?url=",
"facebook_link": "https://www.facebook.com/sharer/sharer.php?u=",
}
GITHUB_URL_PREFIX = LINK_PREFIXES["github_link"]
RAW_URL_PREFIX = (
"https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/"
)
def fix_markdown_links(
cell_source: str, relative_notebook_path: str
) -> tuple[str, bool]:
"""Fixes links in a markdown cell and returns the updated source."""
new_lines = []
changes_made = False
encoded_url = urllib.parse.quote(f"{GITHUB_URL_PREFIX}{relative_notebook_path}")
for line in cell_source.splitlines():
for key, prefix in LINK_PREFIXES.items():
if prefix not in line or "**NOTE:**" in line:
continue
start_index = line.find(prefix) + len(prefix)
end_index = line.find(".ipynb", start_index) + len(".ipynb")
correct_link = ""
if key in {"colab_link", "github_link"}:
correct_link = relative_notebook_path
elif key == "colab_enterprise_link":
correct_link = urllib.parse.quote(
f"{RAW_URL_PREFIX}{relative_notebook_path}",
safe=":",
)
elif key == "workbench_link":
correct_link = f"{RAW_URL_PREFIX}{relative_notebook_path}"
elif key == "bigquery_studio_link":
correct_link = f"{GITHUB_URL_PREFIX}{relative_notebook_path}"
elif key in {
"linkedin_link",
"bluesky_link",
"twitter_link",
"reddit_link",
"facebook_link",
}:
correct_link = encoded_url
if correct_link.lower() not in line.lower():
line = line.replace(line[start_index:end_index], correct_link)
changes_made = True
new_lines.append(line)
return "\n".join(new_lines), changes_made
def update_notebook_links(notebook_paths: list[str]) -> None:
"""Checks and fixes specific types of links in the provided list of notebooks."""
print("Checking notebook links...")
links_updated_count = 0
for notebook_path in notebook_paths:
# False positive
if "vector-search-2-intro" in notebook_path:
continue
try:
with open(notebook_path, encoding="utf-8") as f:
notebook = nbformat.read(f, as_version=4)
relative_notebook_path = os.path.relpath(
notebook_path, start=os.getcwd()
).lower()
notebook_modified = False
for cell in notebook.cells:
if (
cell.cell_type == "markdown"
and "<table" in cell.source
and "colab" in cell.source
):
updated_source, changes_made = fix_markdown_links(
cell.source, relative_notebook_path
)
if changes_made:
cell.source = updated_source
notebook_modified = True
if notebook_modified:
links_updated_count += 1
with open(notebook_path, "w", encoding="utf-8") as f:
nbformat.write(notebook, f)
print(f" -> Fixed links in {notebook_path}")
except Exception as e:
print(f"Could not check links in {notebook_path}. Error: {e}")
if links_updated_count > 0:
print(f"Fixed links in {links_updated_count} notebooks.")
def preprocess_notebook(
notebook_paths: list[str],
max_line_length: int = DEFAULT_RUFF_LINE_LENGTH,
) -> None:
"""Parses notebooks and wraps '@param' blocks with '# fmt: off/on'."""
print("Checking for '@param' blocks to wrap with '# fmt: off/on'...")
for path in notebook_paths:
try:
with open(path, encoding="utf-8") as f:
notebook = nbformat.read(f, as_version=4)
notebook_modified = False
for cell in notebook.cells:
if cell.cell_type != "code" or "@param" not in cell.source:
continue
source_lines = cell.source.split("\n")
clean_source_lines = [
line
for line in source_lines
if line.strip() not in ["# fmt: off", "# fmt: on"]
]
param_indices = [
i
for i, line in enumerate(clean_source_lines)
if "@param" in line and len(line) >= max_line_length
]
if not param_indices:
continue
first_param_index = param_indices[0]
last_param_index = param_indices[-1]
# Reconstruct the cell source with new directives
new_source_lines = []
for i, line in enumerate(clean_source_lines):
if i == first_param_index:
new_source_lines.append("# fmt: off")
new_source_lines.append(line)
if i == last_param_index:
new_source_lines.append("# fmt: on")
new_source = "\n".join(new_source_lines)
if new_source != cell.source:
cell.source = new_source
notebook_modified = True
if notebook_modified:
print(f" -> Wrapped '@param' blocks with '# fmt: off/on' in {path}")
with open(path, "w", encoding="utf-8") as f:
nbformat.write(notebook, f)
except Exception as e:
print(f"Could not process notebook {path}. Error: {e}")
if __name__ == "__main__":
if len(sys.argv) < 2:
print(
"Usage: python notebook_processor.py <notebook_path1> <notebook_path2> ..."
)
sys.exit(1)
notebook_paths = sys.argv[1:]
preprocess_notebook(notebook_paths)
update_notebook_links(notebook_paths)