wehub-resource-sync
159 lines
6.8 KiB
Python
159 lines
6.8 KiB
Python
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
|
|
def get_repo_root(file_path):
|
|
"""通过给定文件路径找到仓库根目录"""
|
|
current_dir = os.path.dirname(file_path)
|
|
while not os.path.exists(os.path.join(current_dir, ".git")):
|
|
parent_dir = os.path.abspath(os.path.join(current_dir, os.pardir))
|
|
if parent_dir == current_dir:
|
|
raise FileNotFoundError("Could not find .git directory")
|
|
current_dir = parent_dir
|
|
return current_dir
|
|
|
|
|
|
def find_dead_links(directory):
|
|
# 正则表达式，用于匹配Markdown和reStructuredText中的链接
|
|
markdown_link_pattern = r"\[([^\[\]]+)\]\(([^)]+)\)" # 修改正则表达式以捕获链接文本
|
|
rst_link_pattern = r"``([^`]+) <([^>]+)>`_" # reStructuredText链接
|
|
dead_links = []
|
|
|
|
for root, dirs, files in os.walk(directory):
|
|
if "third_party" in root:
|
|
continue
|
|
for file in files:
|
|
if file.endswith((".md", ".rst")):
|
|
file_path = os.path.join(root, file)
|
|
try:
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
# 查找Markdown链接
|
|
markdown_matches = re.findall(markdown_link_pattern, content)
|
|
for link_text, match in markdown_matches:
|
|
if match.startswith(("http:", "https:")):
|
|
# 忽略外部链接
|
|
continue
|
|
elif "#" in match:
|
|
# 这是一个锚点链接，忽略文件系统检查
|
|
continue
|
|
abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), match))
|
|
if not os.path.exists(abs_path):
|
|
dead_links.append((file_path, link_text, "Markdown Link: " + abs_path))
|
|
|
|
# 查找reStructuredText链接
|
|
rst_matches = re.findall(rst_link_pattern, content)
|
|
for text, url in rst_matches:
|
|
if not url.startswith(("http:", "https:")):
|
|
abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), url))
|
|
if not os.path.exists(abs_path):
|
|
dead_links.append((file_path, text, "reStructuredText Link: " + abs_path))
|
|
|
|
except Exception as e:
|
|
print(f"Error reading {file_path}: {e}")
|
|
|
|
return dead_links
|
|
|
|
|
|
def create_symlinks(root_dir, src_dir, tgt_dir, file_extension=".md"):
|
|
"""
|
|
Create corresponding folders in the tgt directory based on the src directory,
|
|
and create relative path symlinks for files of a specific type.
|
|
Also check if existing files in tgt have corresponding files in src, otherwise print a warning.
|
|
|
|
:param src_dir: Path to the source directory
|
|
:param tgt_dir: Path to the target directory
|
|
:param file_extension: File extension for which symlinks need to be created, default is ".md"
|
|
"""
|
|
tgt_dir = os.path.join(root_dir, tgt_dir)
|
|
src_dir = os.path.join(root_dir, src_dir)
|
|
|
|
# List all existing files in the tgt directory (including files in subdirectories)
|
|
existing_tgt_files = set()
|
|
for root, dirs, files in os.walk(tgt_dir):
|
|
for file in files:
|
|
existing_tgt_files.add(os.path.relpath(os.path.join(root, file), tgt_dir))
|
|
|
|
# Ensure the target directory exists
|
|
os.makedirs(tgt_dir, exist_ok=True)
|
|
|
|
count = 0
|
|
|
|
# Iterate over all files and folders in the source directory
|
|
for root, dirs, files in os.walk(src_dir):
|
|
# Create corresponding folder structure in the target directory
|
|
relative_path = os.path.relpath(root, src_dir)
|
|
tgt_path = os.path.join(tgt_dir, relative_path)
|
|
|
|
# Create symlinks for files of a specific type
|
|
for file in files:
|
|
if file.endswith(file_extension):
|
|
src_file_path = os.path.join(root, file)
|
|
relative_src_file_path = os.path.relpath(src_file_path, tgt_path)
|
|
tgt_file_path = os.path.join(tgt_path, file)
|
|
|
|
os.makedirs(tgt_path, exist_ok=True)
|
|
|
|
# If the target file already exists and is a symlink, delete it first
|
|
if os.path.exists(tgt_file_path) and os.path.islink(tgt_file_path):
|
|
existing_link_target = os.readlink(tgt_file_path)
|
|
if existing_link_target != relative_src_file_path:
|
|
os.unlink(tgt_file_path)
|
|
# Create the symlink
|
|
os.symlink(relative_src_file_path, tgt_file_path)
|
|
count += 1
|
|
|
|
elif not os.path.exists(tgt_file_path):
|
|
os.symlink(relative_src_file_path, tgt_file_path)
|
|
count += 1
|
|
else:
|
|
print(f"File already exists: {tgt_file_path}. Please remove it from {tgt_dir} and try again.")
|
|
sys.exit(1)
|
|
|
|
# Remove this processed file from the existing tgt files
|
|
existing_tgt_files.discard(os.path.relpath(tgt_file_path, tgt_dir))
|
|
|
|
# Check for remaining files in tgt (i.e., files that exist in tgt but not found in src)
|
|
for file in existing_tgt_files:
|
|
print(f"Warning: File exists in {tgt_dir} but not found in {src_dir}: {file}")
|
|
|
|
return count
|
|
|
|
|
|
def process_file(file_path):
|
|
# Default synchronization of the 'llm' and 'docs/zh/llm' folders
|
|
count = create_symlinks(file_path, "llm", "docs/zh/llm", file_extension=".md")
|
|
if count > 0:
|
|
print("New files were added to docs/zh/llm. Please check them.")
|
|
sys.exit(1)
|
|
|
|
dead_links = find_dead_links(file_path)
|
|
if len(dead_links) > 0:
|
|
print("Dead links found in", file_path)
|
|
for link in dead_links:
|
|
print("file path:", link[0], "- link text:", link[1], "- deal link:", link[2])
|
|
print("Please check the above dead links and fix them.")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) > 1:
|
|
repo_root = get_repo_root(os.path.realpath(sys.argv[1]))
|
|
process_file(repo_root)
|