chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,55 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def check_approval(count, required_reviewers):
|
||||
json_buff = ""
|
||||
for line in sys.stdin:
|
||||
json_buff = "".join([json_buff, line])
|
||||
json_resp = json.loads(json_buff)
|
||||
approves = 0
|
||||
approved_user_ids = []
|
||||
approved_user_logins = set()
|
||||
for review in json_resp:
|
||||
if review["state"] == "APPROVED":
|
||||
approves += 1
|
||||
approved_user_ids.append(review["user"]["id"])
|
||||
approved_user_logins.add(review["user"]["login"])
|
||||
|
||||
# convert to int
|
||||
required_reviewers_int = set()
|
||||
required_reviewers_login = set()
|
||||
for rr in required_reviewers:
|
||||
if rr.isdigit():
|
||||
required_reviewers_int.add(int(rr))
|
||||
else:
|
||||
required_reviewers_login.add(rr)
|
||||
|
||||
if (
|
||||
len(set(approved_user_ids) & required_reviewers_int) + len(approved_user_logins & required_reviewers_login)
|
||||
>= count
|
||||
):
|
||||
print("TRUE")
|
||||
else:
|
||||
print("FALSE")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1 and sys.argv[1].isdigit():
|
||||
check_approval(int(sys.argv[1]), sys.argv[2:])
|
||||
else:
|
||||
print("Usage: python check_pr_approval.py [count] [required reviewer id] ...")
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
failed_num=0
|
||||
echo_list=()
|
||||
approval_line=`curl -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/PaddlePaddle/PaddleNLP/pulls/${PR_ID}/reviews?per_page=10000`
|
||||
|
||||
function add_failed(){
|
||||
failed_num=`expr $failed_num + 1`
|
||||
echo_list="${echo_list[@]}$1"
|
||||
}
|
||||
|
||||
function check_approval(){
|
||||
person_num=`echo $@|awk '{for (i=2;i<=NF;i++)print $i}'`
|
||||
echo ${person_num}
|
||||
APPROVALS=`echo ${approval_line}|python check_pr_approval.py $1 $person_num`
|
||||
echo ${APPROVALS}
|
||||
if [[ "${APPROVALS}" == "FALSE" && "${echo_line}" != "" ]]; then
|
||||
add_failed "${failed_num}. ${echo_line}"
|
||||
fi
|
||||
}
|
||||
|
||||
for file_name in `git diff --numstat upstream/${BRANCH} |awk '{print $NF}'`;do
|
||||
arr_file_name=(${file_name//// })
|
||||
dir1=${arr_file_name[0]}
|
||||
dir2=${arr_file_name[1]}
|
||||
dir3=${arr_file_name[2]}
|
||||
dir4=${arr_file_name[3]}
|
||||
echo "file_name:"${file_name}, "dir1:"${dir1}, "dir2:"${dir2},"dir3:"${dir3},".xx:" ${file_name##*.}
|
||||
if [[ ${file_name} =~ "paddlenlp/trainer/training_args.py" ]] || [[ ${file_name} =~ "paddlenlp/trainer/trainer.py" ]] || [[ ${file_name} =~ "llm/run_pretrain.py" ]] || [[ ${file_name} =~ "llm/run_finetune.py" ]];then
|
||||
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of training_args.py/trainer.py/run_pretrain.py "
|
||||
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
|
||||
elif [[ ${dir1} =~ "paddlenlp" ]];then
|
||||
if [[ ${dir2} =~ "transformers" ]];then
|
||||
echo_line="You must have one RD (wawltor) approval for the changes of transformers "
|
||||
check_approval 1 wawltor
|
||||
elif [[ ${dir2} =~ "taskflow" ]];then
|
||||
echo_line="You must have one RD (w5688414(Recommend),DesmonDay,wawltor) approval for the changes of taskflow"
|
||||
check_approval 1 w5688414 DesmonDay wawltor
|
||||
elif [[ ${dir3} =~ "trainer" ]];then
|
||||
echo_line="You must have one RD (ZHUI(Recommend), wawltor) approval for the changes of trainer"
|
||||
check_approval 1 ZHUI wawltor
|
||||
elif [[ ${dir3} =~ "fast_transformer" ]] || [[ ${dir4} =~ "FasterTransformer" ]] ;then
|
||||
echo_line="You must have one RD (guoshengCS(Recommend), wawltor) approval for the changes of FT or FG"
|
||||
check_approval 1 guoshengCS wawltor
|
||||
elif [[ ${dir3} =~ "llama" ]] || [[ ${dir3} =~ "gpt" ]];then
|
||||
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of llm/llama/auto_parallel/ "
|
||||
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
|
||||
fi
|
||||
elif [[ ${dir1} =~ "model_zoo" ]];then #
|
||||
if [[ ${dir2} =~ "gpt-3" ]] ;then
|
||||
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of model_zoo/gpt-3 "
|
||||
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
|
||||
fi
|
||||
elif [[ ${dir1} =~ "llm" ]];then
|
||||
if [[ ${dir3} =~ "auto_parallel" ]] ;then
|
||||
echo_line="You must have two RD: one from(ZHUI, wawltor),one from(ForFishes,sneaxiy,zhiqiu) approval for the changes of llm/llama/auto_parallel/ "
|
||||
check_approval 2 ZHUI wawltor ForFishes sneaxiy zhiqiu
|
||||
else
|
||||
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of llm"
|
||||
check_approval 1 wj-Mcat lugimzzz DesmonDay wawltor
|
||||
fi
|
||||
elif [[ ${dir1} =~ "tests" ]];then
|
||||
if [[ ${dir2} =~ "transformers" ]] ;then
|
||||
echo_line="You must have one RD (wawltor) approval for the changes of transformers "
|
||||
check_approval 1 wawltor
|
||||
elif [[ ${dir2} =~ "taskflow" ]] || [[ ${dir2} =~ "prompt" ]];then
|
||||
echo_line="You must have one RD (w5688414(Recommend),, wawltor) approval for the changes of taskflow"
|
||||
check_approval 1 w5688414 wawltor
|
||||
elif [[ ${dir2} =~ "llm" ]] ;then
|
||||
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of tests/llm/"
|
||||
check_approval 1 wj-Mcat lugimzzz gongel wtmlon wawltor
|
||||
elif [[ ${dir2} =~ "trainer" ]] ;then
|
||||
echo_line="You must have one RD (ZHUI(Recommend), wawltor) approval for the changes of trainer"
|
||||
check_approval 1 ZHUI wawltor
|
||||
elif [[ ${dir2} =~ "ops" ]] || [[ ${dir2} =~ "embedding" ]];then
|
||||
echo_line="You must have one RD (guoshengCS(Recommend),, wawltor) approval for the changes of ops or embedding"
|
||||
check_approval 1 guoshengCS wawltor
|
||||
elif [[ ${dir2} =~ "model_zoo" ]] ;then
|
||||
echo_line="You must have one RD (wawltor(Recommend)) approval for the changes of model_zoo"
|
||||
check_approval 1 wawltor
|
||||
elif [[ ${dir2} =~ "cli" ]] || [[ ${dir2} =~ "generation" ]];then
|
||||
echo_line="You must have one RD (wj-Mcat(Recommend), wawltor) approval for the changes of llm/generation"
|
||||
check_approval 1 wj-Mcat wawltor
|
||||
elif [[ ${dir2} =~ "test_tipc" ]];then
|
||||
echo_line="You must have one RD (wj-Mcat(Recommend),lugimzzz, wawltor) approval for the changes of test_tipc"
|
||||
check_approval 1 wj-Mcat lugimzzz wawltor
|
||||
elif [[ ${dir2} =~ "dataaug" ]]|| [[ ${dir2} =~ "peft" ]];then
|
||||
echo_line="You must have one RD (lugimzzz(Recommend), wawltor) approval for the changes of dataaug/peft"
|
||||
check_approval 1 lugimzzz wawltor
|
||||
elif [[ ${dir2} =~ "data" ]]|| [[ ${dir2} =~ "dataset" ]];then
|
||||
echo_line="You must have one RD (KB-Ding(Recommend), wawltor) approval for the changes of data/dataset"
|
||||
check_approval 1 KB-Ding wawltor
|
||||
elif [[ ${dir2} =~ "layers" ]] || [[ ${dir2} =~ "metrics" ]] || [[ ${file_name} =~ "tests/llm/test_pretrain.py" ]];then
|
||||
echo_line="You must have one RD (ZHUI(Recommend),DesmonDay,wawltor) approval for the changes of layers/metrics/tests/llm/test_pretrain.py"
|
||||
check_approval 1 ZHUI DesmonDay wawltor
|
||||
elif [[ ${dir2} =~ "utils" ]] ;then
|
||||
echo_line="You must have one RD (wawltor(Recommend)) approval for the changes of tests/utils"
|
||||
check_approval 1 wawltor
|
||||
fi
|
||||
elif [[ ${dir1} =~ "pipelines" ]];then
|
||||
echo_line="You must have one RD (w5688414(Recommend), wawltor) approval for the changes of pipelines"
|
||||
check_approval 1 w5688414 junnyu wawltor
|
||||
elif [[ ${dir1} =~ "ppdiffusers" ]];then
|
||||
echo_line="You must have one RD (junnyu(Recommend), wawltor) approval for the changes of pipelines"
|
||||
check_approval 1 w5688414 junnyu wawltor
|
||||
else
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "${echo_list}" ];then
|
||||
echo "**************************************************************"
|
||||
echo "Please find RD for approval."
|
||||
echo -e "${echo_list[@]}"
|
||||
echo "There are ${failed_num} approved errors."
|
||||
echo "**************************************************************"
|
||||
exit 1
|
||||
else
|
||||
echo "**************************************************************"
|
||||
echo "CI APPROVAL PASSED."
|
||||
echo "**************************************************************"
|
||||
exit 0
|
||||
fi
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
failed_num=0
|
||||
echo_list=()
|
||||
approval_line=`curl -H "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/PaddlePaddle/PaddleNLP/pulls/${PR_ID}/reviews?per_page=10000`
|
||||
|
||||
function add_failed(){
|
||||
failed_num=`expr $failed_num + 1`
|
||||
echo_list="${echo_list[@]}$1"
|
||||
}
|
||||
|
||||
function check_approval(){
|
||||
person_num=`echo $@|awk '{for (i=2;i<=NF;i++)print $i}'`
|
||||
echo ${person_num}
|
||||
APPROVALS=`echo ${approval_line}|python check_pr_approval.py $1 $person_num`
|
||||
echo ${APPROVALS}
|
||||
if [[ "${APPROVALS}" == "FALSE" && "${echo_line}" != "" ]]; then
|
||||
add_failed "${failed_num}. ${echo_line}"
|
||||
fi
|
||||
}
|
||||
|
||||
echo_line="The PaddleNLP repository will be switched to the PaddleFormers repository soon, so the PR needs to be merged into PaddleFormers first and the PR link should be filled in the current PR description area. Then please contact From00 for approval."
|
||||
check_approval 1 From00
|
||||
|
||||
if [ -n "${echo_list}" ];then
|
||||
echo "**************************************************************"
|
||||
echo "Please find RD for approval."
|
||||
echo -e "${echo_list[@]}"
|
||||
echo "There are ${failed_num} approved errors."
|
||||
echo "**************************************************************"
|
||||
exit 1
|
||||
else
|
||||
echo "**************************************************************"
|
||||
echo "CI APPROVAL PASSED."
|
||||
echo "**************************************************************"
|
||||
exit 0
|
||||
fi
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def get_repo_root(file_path):
|
||||
"""通过给定文件路径找到仓库根目录"""
|
||||
current_dir = os.path.dirname(file_path)
|
||||
while not os.path.exists(os.path.join(current_dir, ".git")):
|
||||
parent_dir = os.path.abspath(os.path.join(current_dir, os.pardir))
|
||||
if parent_dir == current_dir:
|
||||
raise FileNotFoundError("Could not find .git directory")
|
||||
current_dir = parent_dir
|
||||
return current_dir
|
||||
|
||||
|
||||
def find_dead_links(directory):
|
||||
# 正则表达式，用于匹配Markdown和reStructuredText中的链接
|
||||
markdown_link_pattern = r"\[([^\[\]]+)\]\(([^)]+)\)" # 修改正则表达式以捕获链接文本
|
||||
rst_link_pattern = r"``([^`]+) <([^>]+)>`_" # reStructuredText链接
|
||||
dead_links = []
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
if "third_party" in root:
|
||||
continue
|
||||
for file in files:
|
||||
if file.endswith((".md", ".rst")):
|
||||
file_path = os.path.join(root, file)
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# 查找Markdown链接
|
||||
markdown_matches = re.findall(markdown_link_pattern, content)
|
||||
for link_text, match in markdown_matches:
|
||||
if match.startswith(("http:", "https:")):
|
||||
# 忽略外部链接
|
||||
continue
|
||||
elif "#" in match:
|
||||
# 这是一个锚点链接，忽略文件系统检查
|
||||
continue
|
||||
abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), match))
|
||||
if not os.path.exists(abs_path):
|
||||
dead_links.append((file_path, link_text, "Markdown Link: " + abs_path))
|
||||
|
||||
# 查找reStructuredText链接
|
||||
rst_matches = re.findall(rst_link_pattern, content)
|
||||
for text, url in rst_matches:
|
||||
if not url.startswith(("http:", "https:")):
|
||||
abs_path = os.path.abspath(os.path.join(os.path.dirname(file_path), url))
|
||||
if not os.path.exists(abs_path):
|
||||
dead_links.append((file_path, text, "reStructuredText Link: " + abs_path))
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error reading {file_path}: {e}")
|
||||
|
||||
return dead_links
|
||||
|
||||
|
||||
def create_symlinks(root_dir, src_dir, tgt_dir, file_extension=".md"):
|
||||
"""
|
||||
Create corresponding folders in the tgt directory based on the src directory,
|
||||
and create relative path symlinks for files of a specific type.
|
||||
Also check if existing files in tgt have corresponding files in src, otherwise print a warning.
|
||||
|
||||
:param src_dir: Path to the source directory
|
||||
:param tgt_dir: Path to the target directory
|
||||
:param file_extension: File extension for which symlinks need to be created, default is ".md"
|
||||
"""
|
||||
tgt_dir = os.path.join(root_dir, tgt_dir)
|
||||
src_dir = os.path.join(root_dir, src_dir)
|
||||
|
||||
# List all existing files in the tgt directory (including files in subdirectories)
|
||||
existing_tgt_files = set()
|
||||
for root, dirs, files in os.walk(tgt_dir):
|
||||
for file in files:
|
||||
existing_tgt_files.add(os.path.relpath(os.path.join(root, file), tgt_dir))
|
||||
|
||||
# Ensure the target directory exists
|
||||
os.makedirs(tgt_dir, exist_ok=True)
|
||||
|
||||
count = 0
|
||||
|
||||
# Iterate over all files and folders in the source directory
|
||||
for root, dirs, files in os.walk(src_dir):
|
||||
# Create corresponding folder structure in the target directory
|
||||
relative_path = os.path.relpath(root, src_dir)
|
||||
tgt_path = os.path.join(tgt_dir, relative_path)
|
||||
|
||||
# Create symlinks for files of a specific type
|
||||
for file in files:
|
||||
if file.endswith(file_extension):
|
||||
src_file_path = os.path.join(root, file)
|
||||
relative_src_file_path = os.path.relpath(src_file_path, tgt_path)
|
||||
tgt_file_path = os.path.join(tgt_path, file)
|
||||
|
||||
os.makedirs(tgt_path, exist_ok=True)
|
||||
|
||||
# If the target file already exists and is a symlink, delete it first
|
||||
if os.path.exists(tgt_file_path) and os.path.islink(tgt_file_path):
|
||||
existing_link_target = os.readlink(tgt_file_path)
|
||||
if existing_link_target != relative_src_file_path:
|
||||
os.unlink(tgt_file_path)
|
||||
# Create the symlink
|
||||
os.symlink(relative_src_file_path, tgt_file_path)
|
||||
count += 1
|
||||
|
||||
elif not os.path.exists(tgt_file_path):
|
||||
os.symlink(relative_src_file_path, tgt_file_path)
|
||||
count += 1
|
||||
else:
|
||||
print(f"File already exists: {tgt_file_path}. Please remove it from {tgt_dir} and try again.")
|
||||
sys.exit(1)
|
||||
|
||||
# Remove this processed file from the existing tgt files
|
||||
existing_tgt_files.discard(os.path.relpath(tgt_file_path, tgt_dir))
|
||||
|
||||
# Check for remaining files in tgt (i.e., files that exist in tgt but not found in src)
|
||||
for file in existing_tgt_files:
|
||||
print(f"Warning: File exists in {tgt_dir} but not found in {src_dir}: {file}")
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def process_file(file_path):
|
||||
# Default synchronization of the 'llm' and 'docs/zh/llm' folders
|
||||
count = create_symlinks(file_path, "llm", "docs/zh/llm", file_extension=".md")
|
||||
if count > 0:
|
||||
print("New files were added to docs/zh/llm. Please check them.")
|
||||
sys.exit(1)
|
||||
|
||||
dead_links = find_dead_links(file_path)
|
||||
if len(dead_links) > 0:
|
||||
print("Dead links found in", file_path)
|
||||
for link in dead_links:
|
||||
print("file path:", link[0], "- link text:", link[1], "- deal link:", link[2])
|
||||
print("Please check the above dead links and fix them.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
repo_root = get_repo_root(os.path.realpath(sys.argv[1]))
|
||||
process_file(repo_root)
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def add_spaces_between_chinese_and_english(text):
|
||||
# 正则表达式匹配中文字符后紧跟英文字符或英文字符后紧跟中文字符的情况
|
||||
pattern = r"([\u4e00-\u9fa5])([a-zA-Z])|([a-zA-Z])([\u4e00-\u9fa5])"
|
||||
|
||||
def replace_func(match):
|
||||
return match.group(1) + " " + match.group(2) if match.group(1) else match.group(3) + " " + match.group(4)
|
||||
|
||||
return re.sub(pattern, replace_func, text)
|
||||
|
||||
|
||||
def process_outside_codeblocks(text):
|
||||
# 正则表达式用于匹配Markdown代码块
|
||||
codeblock_pattern = r"```[\s\S]*?```"
|
||||
|
||||
# 找到所有的代码块并替换为占位符
|
||||
codeblocks = re.findall(codeblock_pattern, text)
|
||||
placeholders = []
|
||||
for i, block in enumerate(codeblocks):
|
||||
placeholder = f"CODEBLOCK_PLACEHOLDER_{i}"
|
||||
placeholders.append(placeholder)
|
||||
text = text.replace(block, placeholder, 1)
|
||||
|
||||
# 对非代码块文本处理中英文空格
|
||||
processed_text = add_spaces_between_chinese_and_english(text)
|
||||
|
||||
# 将占位符替换回原来的代码块内容
|
||||
for placeholder, block in zip(placeholders, codeblocks):
|
||||
processed_text = processed_text.replace(placeholder, block, 1)
|
||||
|
||||
return processed_text
|
||||
|
||||
|
||||
def process_file(file_path):
|
||||
with open(file_path, "r+", encoding="utf-8") as file:
|
||||
content = file.read()
|
||||
new_content = process_outside_codeblocks(content)
|
||||
if new_content != content:
|
||||
file.seek(0)
|
||||
file.write(new_content)
|
||||
file.truncate()
|
||||
print(f"Spaces added to {file_path} (excluding code blocks)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for file_path in sys.argv[1:]:
|
||||
process_file(file_path)
|
||||
@@ -0,0 +1,268 @@
|
||||
|
||||
|
||||
## 使用方法：
|
||||
|
||||
### 1构建modular__**.py文件
|
||||
|
||||
#### 1.1分析基础模型
|
||||
|
||||
在开始构建 `modular_xxx.py`文件之前，首先需要深入分析要基于的**基础模型**。这个基础模型通常是 paddle/ Transformers 库中已有的成熟模型。
|
||||
|
||||
##### 1.1.1 选择合适的基础模型
|
||||
|
||||
**选择标准：**
|
||||
|
||||
- **架构相似性**：新模型与基础模型的架构应尽可能相似
|
||||
- **任务类型**：基础模型应支持相同的任务（如文本生成、分类等）
|
||||
- **代码质量**：选择代码结构清晰、文档完善的模型
|
||||
|
||||
**常见基础模型选择：**
|
||||
|
||||
```
|
||||
# 基于BERT架构的模型
|
||||
基础模型：BertModel, RobertaModel, DebertaModel
|
||||
|
||||
# 基于GPT架构的模型
|
||||
基础模型：GPT2Model, LlamaModel, GPTNeoXModel
|
||||
|
||||
# 基于Encoder-Decoder架构的模型
|
||||
基础模型：T5Model, BartModel, PegasusModel
|
||||
```
|
||||
|
||||
##### 1.1.2 分析基础模型的关键组件
|
||||
|
||||
对于选定的基础模型，需要分析其核心组件：
|
||||
|
||||
###### **1. 配置文件 (`configuration_xxx.py`)**
|
||||
|
||||
```
|
||||
# 分析配置参数
|
||||
# 关注：hidden_size, num_attention_heads, num_hidden_layers,
|
||||
# vocab_size, max_position_embeddings 等关键参数
|
||||
```
|
||||
|
||||
###### **2. 模型架构 (`modeling_xxx.py`)**
|
||||
|
||||
```
|
||||
# 分析模型类结构
|
||||
import inspect
|
||||
from transformers import BertModel
|
||||
|
||||
# 查看类的方法和属性
|
||||
print(inspect.getmembers(BertModel, predicate=inspect.ismethod))
|
||||
# 重点关注：__init__, forward, 以及其他关键方法
|
||||
```
|
||||
|
||||
##### 1.1.3 识别需要修改的部分
|
||||
|
||||
基于分析结果，确定哪些部分需要自定义：
|
||||
|
||||
| 组件 | 是否需要修改 | 修改原因 |
|
||||
| :--------------- | :----------- | :------------------------- |
|
||||
| **配置参数** | ✅ 通常需要 | 调整模型尺寸、注意力头数等 |
|
||||
| **前向传播逻辑** | ✅ 通常需要 | 适配新的架构变化 |
|
||||
| **注意力机制** | ⚠️ 可能需要 | 如果使用不同的注意力机制 |
|
||||
| **位置编码** | ⚠️ 可能需要 | 如果使用不同的位置编码方案 |
|
||||
| **输出头** | ✅ 通常需要 | 适配不同的任务需求 |
|
||||
| **初始化方法** | ⚠️ 可能需要 | 如果使用不同的初始化策略 |
|
||||
|
||||
#### 1.2编写modular文件结构
|
||||
|
||||
在完成基础模型分析后，您需要创建一个结构清晰、符合规范的 `modular_xxx.py`文件。这个文件是代码生成器的模板，其结构直接决定了最终输出的 `modeling_xxx.py`文件的质量。
|
||||
|
||||
##### 1.2.1 文件基本结构
|
||||
|
||||
一个标准的 `modular_xxx.py`文件应包含以下部分，按顺序排列：
|
||||
|
||||
```
|
||||
# coding=utf-8
|
||||
# 版权声明 (可选)
|
||||
""" 新模型的简要文档字符串 (可选) """
|
||||
|
||||
# 1. 导入部分
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.utils.checkpoint
|
||||
from torch import nn
|
||||
from torch.nn import CrossEntropyLoss
|
||||
|
||||
# 从基础模型导入必要的组件
|
||||
from transformers.models.llama.modeling_llama import (
|
||||
LlamaConfig,
|
||||
LlamaModel,
|
||||
LlamaForCausalLM,
|
||||
LlamaDecoderLayer,
|
||||
# ... 其他需要继承或引用的组件
|
||||
)
|
||||
from transformers import PreTrainedModel, PreTrainedTokenizerBase
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
# 3. 注意力机制 (如果需要自定义)
|
||||
class MyNewAttention(nn.Module):
|
||||
"""自定义注意力机制"""
|
||||
def __init__(self, config: MyNewModelConfig):
|
||||
super().__init__()
|
||||
# 实现自定义注意力逻辑
|
||||
pass
|
||||
|
||||
def forward(self, hidden_states, attention_mask=None):
|
||||
# 实现前向传播
|
||||
pass
|
||||
|
||||
|
||||
# 4. 解码器层 (如果需要修改层结构)
|
||||
class MyNewDecoderLayer(LlamaDecoderLayer):
|
||||
"""
|
||||
自定义解码器层，继承自LlamaDecoderLayer
|
||||
重写需要修改的方法
|
||||
"""
|
||||
def __init__(self, config: MyNewModelConfig):
|
||||
super().__init__(config)
|
||||
# 替换或修改注意力机制
|
||||
if config.use_custom_attention:
|
||||
self.self_attn = MyNewAttention(config)
|
||||
|
||||
def forward(self, hidden_states, attention_mask=None):
|
||||
# 可以完全重写或部分修改父类逻辑
|
||||
if self.config.use_custom_attention:
|
||||
# 自定义逻辑
|
||||
return self._custom_forward(hidden_states, attention_mask)
|
||||
else:
|
||||
# 回退到父类逻辑
|
||||
return super().forward(hidden_states, attention_mask)
|
||||
|
||||
def _custom_forward(self, hidden_states, attention_mask):
|
||||
"""自定义前向传播实现"""
|
||||
pass
|
||||
|
||||
|
||||
# 5. 主模型类
|
||||
class MyNewModel(LlamaModel):
|
||||
"""
|
||||
我的新模型主类，继承自LlamaModel
|
||||
通常需要重写 __init__ 和 forward 方法
|
||||
"""
|
||||
def __init__(self, config: MyNewModelConfig):
|
||||
super().__init__(config)
|
||||
# 替换解码器层
|
||||
self.layers = nn.ModuleList([
|
||||
MyNewDecoderLayer(config) for _ in range(config.num_hidden_layers)
|
||||
])
|
||||
# 其他自定义初始化
|
||||
self.custom_layer = nn.Linear(config.hidden_size, config.custom_param)
|
||||
|
||||
def forward(self, input_ids, attention_mask=None):
|
||||
# 调用父类获取基础输出
|
||||
super().forward(input_ids, attention_mask)
|
||||
|
||||
# 添加自定义处理
|
||||
hidden_states = outputs[0]
|
||||
custom_output = self.custom_layer(hidden_states)
|
||||
|
||||
# 返回修改后的输出
|
||||
return (custom_output,) + outputs[1:]
|
||||
|
||||
|
||||
# 6. 任务特定模型 (如用于因果语言建模)
|
||||
class MyNewForCausalLM(LlamaForCausalLM):
|
||||
"""
|
||||
用于因果语言建模的我的新模型
|
||||
"""
|
||||
def __init__(self, config: MyNewModelConfig):
|
||||
super().__init__(config)
|
||||
# 替换主模型
|
||||
self.model = MyNewModel(config)
|
||||
|
||||
def forward(self, input_ids, attention_mask=None, labels=None):
|
||||
# 可以完全重写或扩展父类逻辑
|
||||
outputs = self.model(input_ids, attention_mask=attention_mask)
|
||||
|
||||
# 计算损失等
|
||||
loss = None
|
||||
if labels is not None:
|
||||
# 计算损失逻辑
|
||||
pass
|
||||
|
||||
return {"loss": loss, "logits": outputs[0]}
|
||||
|
||||
|
||||
# 8. 更新 __all__ 列表，声明哪些类应该被导出
|
||||
__all__ = [
|
||||
"MyNewModelConfig",
|
||||
"MyNewModel",
|
||||
"MyNewForCausalLM",
|
||||
"MyNewDecoderLayer",
|
||||
]
|
||||
```
|
||||
|
||||
##### 1.2.2 关键编写原则
|
||||
|
||||
**清晰的继承关系**
|
||||
|
||||
```
|
||||
# ✅ 正确：明确继承关系
|
||||
class MyNewModel(LlamaModel):
|
||||
pass
|
||||
|
||||
# ❌ 避免：直接继承过于通用的基类
|
||||
class MyNewModel(PreTrainedModel):
|
||||
pass # 这会导致需要实现大量抽象方法
|
||||
```
|
||||
|
||||
**最小化重写**
|
||||
|
||||
```
|
||||
# ✅ 正确：只重写需要修改的方法
|
||||
class MyNewDecoderLayer(LlamaDecoderLayer):
|
||||
def __init__(self, config):
|
||||
super().__init__(config) # 先调用父类初始化
|
||||
# 只修改需要定制的部分
|
||||
if config.use_custom_attention:
|
||||
self.self_attn = CustomAttention(config)
|
||||
|
||||
# ❌ 避免：完全重写整个类，除非必要
|
||||
```
|
||||
|
||||
**保持接口一致性**：
|
||||
|
||||
```
|
||||
def forward(self, input_ids, attention_mask=None, **kwargs):
|
||||
# 处理自定义逻辑
|
||||
result = custom_processing(input_ids)
|
||||
# 调用父类实现剩余逻辑
|
||||
super().forward(result, attention_mask, **kwargs)
|
||||
```
|
||||
|
||||
**充分利用现有组件**：
|
||||
|
||||
```
|
||||
# ✅ 正确：复用基础模型的组件
|
||||
from transformers.models.llama.modeling_llama import (
|
||||
LlamaRMSNorm,
|
||||
LlamaRotaryEmbedding,
|
||||
apply_rotary_pos_emb,
|
||||
)
|
||||
```
|
||||
|
||||
### 2.**执行转换命令**
|
||||
|
||||
通过一个用户主脚本main来驱动整个流程。其标准使用方式如下：
|
||||
|
||||
```
|
||||
#自动查找各个模型文件下的modular__**.py模块化构建代码，执行转换生成modeling__***.py文件
|
||||
python main.py
|
||||
```
|
||||
|
||||
### **自动化处理流水线**
|
||||
|
||||
<img src="https://raw.githubusercontent.com/hsz06/hsz/6d27682d692c0402095192c34ac245b1122adef3/process.png" style="zoom:33%;" />
|
||||
|
||||
### 最终输出
|
||||
|
||||
最终，在模型对应的目录下（如 `src/transformers/models/qwen2/`）会生成目标文件：
|
||||
|
||||
- **`modeling_qwen2.py`**：**这是唯一的输出文件，也是最终成果。** 它包含了：**模型架构**（如 `Qwen2Model`, `Qwen2ForCausalLM`）**内联的配置类**（如 `Qwen2Config`）**所有相关的函数、工具类和常量****正确的导入语句**（只导入标准库或 transformers 的通用组件）**文件顶部的警告注释**：明确告知开发者此文件为自动生成，不可手动编辑。
|
||||
@@ -0,0 +1,60 @@
|
||||
以下是针对Llama模型架构中各组件的功能解析，按模块分类说明其核心作用：
|
||||
|
||||
------
|
||||
|
||||
### **核心工具函数**
|
||||
|
||||
| 函数/常量 | 作用 | 与Qwen2的差异 |
|
||||
| :----------------------------: | :-----------------------------------: | :--------------------: |
|
||||
| `swiglu` | 实现SwiGLU激活函数：`x * silu(gate)` | Qwen2使用标准GLU |
|
||||
| `rms_norm_fused` | 启用融合的RMSNorm计算（CUDA优化） | 实现相同但配置参数不同 |
|
||||
| `__all__` | 定义模块的公开接口 | - |
|
||||
| `_get_interleave` | 生成交错注意力头索引（用于长序列） | Llama特有 |
|
||||
| `build_alibi_tensor` | 构建ALiBi位置偏置张量（相对位置编码） | Qwen2未使用 |
|
||||
| `get_triangle_upper_mask` | 生成因果上三角掩码 | 实现逻辑相同 |
|
||||
| `assign_kv_heads` | 分配KV头的索引（支持GQA/MQA） | - |
|
||||
| `parallel_matmul` | 并行矩阵乘法（张量并行） | - |
|
||||
| `scaled_dot_product_attention` | 核心注意力计算 | Llama支持更多掩码类型 |
|
||||
| `_make_causal_mask` | 动态生成因果掩码（考虑padding） | Qwen2更简化 |
|
||||
|
||||
### **归一化层**
|
||||
|
||||
| 类/函数 | 作用 | 差异点 |
|
||||
| :------------: | :-------------------: | :-------------: |
|
||||
| `LlamaRMSNorm` | 带融合优化的RMS归一化 | 与Qwen2实现相同 |
|
||||
|
||||
### **位置编码（核心差异）**
|
||||
|
||||
| 类 | 作用 | 特性 |
|
||||
| :-------------------------------------: | :------------------------: | :---------------: |
|
||||
| `LlamaRotaryEmbedding` | 基础RoPE实现 | - |
|
||||
| `LlamaLinearScalingRotaryEmbedding` | 线性缩放RoPE（扩展上下文） | Qwen2无此变体 |
|
||||
| `LlamaNTKScalingRotaryEmbedding` | NTK-aware缩放RoPE | 动态调整高频/低频 |
|
||||
| `LlamaDynamicNTKScalingRotaryEmbedding` | 动态NTK缩放（训练自适应） | Llama特有 |
|
||||
| `Llama3RotaryEmbedding` | Llama3专用RoPE | 改进的旋转策略 |
|
||||
|
||||
### **前馈网络**
|
||||
|
||||
| 类 | 作用 | 差异 |
|
||||
| :--------: | :-----------------: | :------------: |
|
||||
| `LlamaMLP` | 使用SwiGLU的门控FFN | Qwen2用普通GLU |
|
||||
|
||||
### **注意力机制**
|
||||
|
||||
| 类 | 核心改进 | 说明 |
|
||||
| :-----------------: | :----------------------------------------: | :------------: |
|
||||
| `LlamaAttention` | - 多版本RoPE支持 - ALiBi融合 - 动态NTK缩放 | 比Qwen2更复杂 |
|
||||
| `LlamaDecoderLayer` | 深度优化层实现 | 支持梯度检查点 |
|
||||
|
||||
### **预训练基础**
|
||||
|
||||
| 类 | 关键功能 | 扩展性 |
|
||||
| :--------------------: | :--------------------------: | :-----------: |
|
||||
| `LlamaPretrainedModel` | - 多设备加载 - FLOPs计算工具 | 比Qwen2更完善 |
|
||||
|
||||
### **任务模块**
|
||||
|
||||
| 类 | 用途 | 特色 |
|
||||
| :----------------: | :------------: | :---------------: |
|
||||
| `LlamaForCausalLM` | 语言建模 | 支持静态图导出 |
|
||||
| `ConcatMaskedLoss` | 多任务损失合并 | 处理padding的梯度 |
|
||||
@@ -0,0 +1,87 @@
|
||||
|
||||
|
||||
是Qwen2模型中各组件和函数的详细作用说明，按模块分类整理：
|
||||
|
||||
### **核心工具函数**
|
||||
|
||||
| 函数/常量 | 作用 |
|
||||
| :----------------------------: | :------------------------------------------------------: |
|
||||
| `__all__` | 定义模块的公开接口，控制`from module import *`时的可见性 |
|
||||
| `get_triangle_upper_mask` | 生成上三角因果注意力掩码（防止未来信息泄露） |
|
||||
| `assign_kv_heads` | 分配Key/Value头的索引（用于GQA/MQA） |
|
||||
| `parallel_matmul` | 并行矩阵乘法（支持张量并行） |
|
||||
| `scaled_dot_product_attention` | 实现缩放点积注意力核心计算 |
|
||||
| `masked_fill` | 按掩码填充张量（如将padding位置设为负无穷） |
|
||||
| `is_casual_mask` | 判断是否为因果注意力掩码 |
|
||||
| `_make_causal_mask` | 创建因果注意力掩码（考虑padding） |
|
||||
| `_expand_2d_mask` | 将2D掩码扩展为4D（适配多头注意力） |
|
||||
| `repeat_kv` | 重复Key/Value头（用于GQA/MQA） |
|
||||
|
||||
### **归一化层**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :------------: | :------------------------------: |
|
||||
| `Qwen2RMSNorm` | **RMS归一化层**（替代LayerNorm） |
|
||||
|
||||
### **位置编码**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :----------------------: | :--------------------------------: |
|
||||
| `Qwen2RotaryEmbedding` | **旋转位置编码(RoPE)** |
|
||||
| - `rotate_half` | 旋转向量的后半部分（RoPE核心操作） |
|
||||
| - `apply_rotary_pos_emb` | 将旋转位置编码应用到注意力分数 |
|
||||
|
||||
### **前馈网络**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :--------: | :---------------------------: |
|
||||
| `Qwen2MLP` | **门控线性单元(GLU)前馈网络** |
|
||||
|
||||
### **注意力机制**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :-----------------: | :------------------------------------------------: |
|
||||
| `Qwen2Attention` | **多头注意力机制** |
|
||||
| - `__init__` | 初始化Q/K/V投影层、输出层和RoPE |
|
||||
| - `forward` | 处理输入序列，计算注意力分数并聚合值向量 |
|
||||
| `Qwen2DecoderLayer` | **Transformer解码层** |
|
||||
| - `__init__` | 组合自注意力层和前馈网络 |
|
||||
| - `forward` | 执行：`LN -> Attention -> Add -> LN -> MLP -> Add` |
|
||||
|
||||
### **预训练基础**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :--------------------: | :--------------------------------: |
|
||||
| `Qwen2PretrainedModel` | **预训练模型基类** |
|
||||
| - `config_class` | 关联的配置类（Qwen2Config） |
|
||||
| - `_get_name_mappings` | 定义参数名称映射（用于加载检查点） |
|
||||
| - `_init_weights` | 参数初始化策略 |
|
||||
| - `_get_model_flops` | 计算模型FLOPs |
|
||||
|
||||
### **主干模型**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :---------------------------------: | :-----------------------: |
|
||||
| `Qwen2Model` | **模型主干架构** |
|
||||
| - `_prepare_decoder_attention_mask` | 生成解码器掩码 |
|
||||
| - `forward` | 执行完整的Transformer堆栈 |
|
||||
| `Qwen2ForCausalLM` | **因果语言模型** |
|
||||
| - `prepare_inputs_for_generation` | 处理生成时的输入格式 |
|
||||
| - `forward` | 计算语言建模损失 |
|
||||
|
||||
### **任务特定头部**
|
||||
|
||||
| 类 | 作用 |
|
||||
| :------------------------------: | :----------------------: |
|
||||
| `Qwen2LMHead` | 语言模型头部（词表投影） |
|
||||
| `Qwen2ForSequenceClassification` | 序列分类任务适配 |
|
||||
| `Qwen2ForTokenClassification` | 标记分类任务适配 |
|
||||
| `Qwen2SentenceEmbedding` | 句子向量提取 |
|
||||
|
||||
### **训练相关**
|
||||
|
||||
| 类/函数 | 作用 |
|
||||
| :-------------------------: | :------------------------: |
|
||||
| `Qwen2PretrainingCriterion` | 预训练损失计算 |
|
||||
| `recompute_training_full` | 激活重计算策略 |
|
||||
| `create_custom_forward` | 为梯度检查点创建自定义前向 |
|
||||
@@ -0,0 +1,150 @@
|
||||
## 项目报告：飞桨PaddleNLP-前沿模型模块化设计
|
||||
|
||||
### 项目信息
|
||||
|
||||
* 项目名称：飞桨PaddleNLP-前沿模型模块化设计
|
||||
* 方案描述：
|
||||
* 文件解析与并行处理：自动查找并并行处理 modular_*.py 文件。
|
||||
* 转换流程 (run_converter)：封装了单个模块化文件到独立模型文件的完整转换逻辑，包括动态确定模型名称和输出路径、收集并展开导入、重写子类并生成中间文件、移除导入并重写、全局重命名以及最终文件写入和临时文件清理。
|
||||
* 辅助工具 (until 目录)：包含 collect_import_modeling.py、rename_identifiers.py 和 rewrite_child_classes.py 等脚本，用于支持转换流程中的导入处理、标识符重命名和子类重写。
|
||||
* modular_qwen2.py：作为 convert 工具的输入，该文件继承了 Llama 模型的许多组件，并进行了 Qwen2 特有的修改和优化，如 Qwen2RMSNorm、Qwen2RotaryEmbedding、Qwen2MLP 和 Qwen2Attention。它还包含了大量与 PaddlePaddle 分布式训练（如张量并行、序列并行、重计算）和性能优化（如 Flash Attention、融合操作）相关的导入和逻辑。
|
||||
* configuration.py：定义了 Qwen2Config 类，存储 Qwen2 模型的所有配置参数，包括词汇表大小、隐藏层维度、注意力头数量、激活函数、最大位置嵌入长度等，并支持滑动窗口注意力等高级特性。
|
||||
* modeling__qwen2.py：是经过 convert 工具处理后生成的最终模型文件，包含了 Qwen2 模型的完整实现，包括核心模型类、组件实现、辅助函数以及分布式训练和性能优化相关的代码。
|
||||
|
||||
* 时间规划:
|
||||
|
||||
* 需求分析与方案设计(7月1日-7月15日)
|
||||
|
||||
详细调研和对比分析至少两种主流LLM（如Llama系列、Qwen系列）的架构细节、实现差异 和共通之处。 设计LLM的模块化组件体系，明确各模块的功能边界、输入输出接口、可配置参数以及模块 间的依赖关系。制定基于libcst的源码分析策略，确定需要识别的代码模式和转换规则。 初步规划模型并行能力的模块化方案和自动化集成思路
|
||||
|
||||
* 模块化核心功能开发（7月15日-8月15日）
|
||||
|
||||
利用libcst等工具，开发源码分析和转换工具的原型，能够解析现有模型代码并提取关键 结构信息，或根据配置生成初步的模块化代码片段。 搭建单元测试和集成测试框架，确保各模块和工具的正确性。
|
||||
|
||||
* Qwen2模型自动化构造（8月15日-9月7日）
|
||||
|
||||
以Llama模型结构为蓝本，利用阶段二开发的工具和模块库，自动化生成Qwen2模型的完整结构代码
|
||||
|
||||
* 精度对齐及模型并行能力验证（9月7日-9月21日）
|
||||
|
||||
对生成的Qwen2模型进行细致的功能测试和精度验证，通过在测试数据上与手动实现的 Qwen2模型进行效果对比，确保数值精度对齐。
|
||||
|
||||
* 文档撰写与项目总结（9月21日-9月30日）
|
||||
|
||||
编写详细的设计文档、用户手册、上手教程以及最佳实践案例。 整理项目代码，按照PaddleNLP社区规范准备Pull Request，将核心成果贡献给社区。完成项目总结报告。
|
||||
|
||||
### 项目进度
|
||||
|
||||
* 已完成工作：
|
||||
|
||||
对照项目申请书的方案，我完成了预定任务，主要工作成果如下：
|
||||
|
||||
* 核心转换流水线
|
||||
- 实现了`run_converter()`函数，执行完整的三阶段转换流程
|
||||
- 支持并行处理多个模型文件转换
|
||||
- 自动生成带警告标识的输出文件
|
||||
* 导入扩展系统
|
||||
- 实现`expand_modeling_imports()`函数进行递归依赖解析
|
||||
- 支持模块化导入的自动展开和集成
|
||||
* 标识符重命名系统
|
||||
- 实现`rename_identifiers()`函数进行智能重命名
|
||||
- 支持大小写保持的标识符转换
|
||||
* 类重写系统
|
||||
- 实现完整的类重写工具，支持继承关系扁平化
|
||||
- 集成依赖分析和合并引擎
|
||||
* 以Llama为蓝本的Qwen2模型自动化生成
|
||||
* 实现了基于llama的modular__qwen2.py
|
||||
* 通过转换系统自动生成完整的Qwen2模型实现
|
||||
* 对生成的模型进行了精度验证和并行能力验证
|
||||
* 与原代码进行精度对齐
|
||||
* 进行了并行能力验证
|
||||
* 项目文档
|
||||
* 转换工具的使用方法
|
||||
* 模块化构建的流程
|
||||
* 精度及并行能力验证报告
|
||||
|
||||
* 遇到的问题以及解决方案
|
||||
|
||||
* Import导入项收集的复杂性问题
|
||||
|
||||
存在的难点：
|
||||
|
||||
- （1）重复导入识别：同一个模块可能通过不同路径被多次导入，需要去重处理
|
||||
- （2）导入格式多样性：存在相对导入（`..modeling`）、绝对导入等多种格式，解析复杂
|
||||
- （3）循环依赖检测：模块间可能存在相互依赖，导致无限递归
|
||||
- （4）无效导入过滤：需要区分真正的modeling导入和其他类型的导入
|
||||
|
||||
针对以上问题，我提出了分层过滤解决方案：
|
||||
|
||||
- （1）专门的导入收集器： collect_import_modeling实现`ModelingImportCollector`专门识别包含"modeling"关键字的导入语句
|
||||
- （2）路径标准化处理： collect_import_modeling通过`resolve_file_path()`函数统一处理相对导入路径转换
|
||||
- （3）循环依赖避免机制： collect_import_modeling使用`seen`集合记录已处理的依赖项，防止无限递归
|
||||
- （4）严格模式过滤：filter_specific_modeling_imports()`只保留严格符合相对导入模式的modeling导入
|
||||
|
||||
* 大规模文件处理效率
|
||||
|
||||
存在的难点：
|
||||
|
||||
- （1）单线程处理效率低：大量模型文件的串行处理耗时过长
|
||||
- （2）内存占用过高：同时加载多个大型模型文件导致内存压力
|
||||
|
||||
针对以上问题，我构建了并行处理架构：
|
||||
|
||||
- （1）多进程并行化： main.py使用`multiprocessing.Pool`实现多进程并行转换，动态调整工作进程数量
|
||||
- （2）临时文件管理： main.py:65-68 处理完成后自动清理临时文件，减少内存占用
|
||||
|
||||
* 标识符重命名一致性
|
||||
|
||||
存在的难点：
|
||||
|
||||
- （1）大小写风格保持：需要保持原代码的命名风格（如llama→qwen2, Llama→Qwen2, LLAMA→QWEN2）
|
||||
- （2）误替换风险：字符串级别的替换容易产生误替换和语法错误
|
||||
- （3）冲突检测复杂：需要避免与已存在的标识符产生命名冲突
|
||||
|
||||
针对以上问题，我开发了AST级别的智能重命名系统：
|
||||
|
||||
- （1）大小写保持算法： rename_identifiers.py通过`_case_preserving_replace()`方法检测原标识符的大小写模式并应用到目标名称
|
||||
- （2）AST精确转换： rename_identifiers.py 使用`GenericRenamerTransformer`在AST层面进行精确替换，避免误替换
|
||||
- （3）智能冲突检测： rewrite_child_classes.py维护`existing_names`集合检测命名冲突，只注入不冲突的依赖项
|
||||
|
||||
* 注入依赖时的位置问题
|
||||
|
||||
存在的难点：
|
||||
|
||||
- （1）依赖注入顺序混乱：不同类型的依赖（方法、类）混合注入导致代码结构不清晰
|
||||
- （2）父子类位置关系错误：子类可能在父类定义之前被注入，导致引用错误
|
||||
- （3）代码可读性差：依赖项随意插入破坏了代码的逻辑结构和可维护性
|
||||
|
||||
针对以上问题，我实现了智能的分层注入策略：
|
||||
|
||||
- （1）依赖类型分类注入： 系统将注入的依赖分为方法和类两类，方法优先注入在imports之后，类按依赖关系分层注入
|
||||
- （2）父子类依赖关系排序： 通过分析类的继承关系，将有父类依赖的类和无父类依赖的类分开处理，确保父类先于子类定义
|
||||
- （3）动态位置插入机制： 在遍历主逻辑时，当遇到父类定义后立即插入其对应的子类，保证依赖关系的正确性和代码的逻辑连贯性
|
||||
|
||||
* 测试用例
|
||||
|
||||
* 模型转换正确性验证
|
||||
|
||||
测试用例的核心是验证从`modular_qwen2.py`转换生成的`modeling_qwen2.py`的功能正确性
|
||||
|
||||
* 双模型对比测试：加载原始的modular_qwen2模型和转换后的modeling_qwen2模型进行数值对比
|
||||
* 精度验证：使用相同的输入数据，两个模型输出的数值使用`numpy.allclose`进行数值对比，相对容差`rtol=1e-5`，绝对容差`atol=1e-3`；转换后的模型与原模型输出相同，模型转换正确。
|
||||
|
||||
* 精度对齐与并行能力验证
|
||||
|
||||
测试用例的核心是验证从`modular_qwen2.py`转换生成的`modeling_qwen2.py`的并行能力正确性
|
||||
|
||||
- 分布式训练兼容性：创建分布式并行环境，并使用paddle.distributed.launch进行启动运行，在张量并行度为2的配置下，转换后模型与原模型输出相对容差`rtol=1e-5`，绝对容差`atol=1e-3`
|
||||
- 对于相同的输入产生了完全相同的输出，分布式能力验证成功。
|
||||
|
||||
* 后续工作安排
|
||||
|
||||
* 对于大规模文件的处理依赖
|
||||
|
||||
对于多文件建立依赖关系图，从底层文件一次向上开始转换
|
||||
|
||||
* 完善pre-commit，实现自动化转换模块化文件并进行验证
|
||||
|
||||
* 扩展测试用例覆盖，完善精度对齐和并行能力验证文档中的测试场景
|
||||
|
||||
* 优化基础模型的构建，真正把基础模型标准化，模块化
|
||||
@@ -0,0 +1,20 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
# Copyright 2024 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from .configuration import *
|
||||
from .modeling import *
|
||||
from .modeling_pp import *
|
||||
from .tokenizer import *
|
||||
from .tokenizer_fast import *
|
||||
@@ -0,0 +1,213 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Qwen2 model configuration"""
|
||||
|
||||
from ..configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QWEN2_PRETRAINED_INIT_CONFIGURATION",
|
||||
"Qwen2Config",
|
||||
"QWEN2_PRETRAINED_RESOURCE_FILES_MAP",
|
||||
]
|
||||
QWEN2_PRETRAINED_INIT_CONFIGURATION = {
|
||||
# Hypothetical model weights (tiny-random-llama & micro-random-llama) for test only
|
||||
"__internal_testing__/micro-random-llama": {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"hidden_size": 64,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 1000,
|
||||
"max_position_embeddings": 2048,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 8,
|
||||
"num_hidden_layers": 1,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"vocab_size": 32000,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
},
|
||||
"__internal_testing__/tiny-random-llama": {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"hidden_size": 768,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 11008,
|
||||
"max_position_embeddings": 2048,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 8,
|
||||
"num_hidden_layers": 2,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"vocab_size": 32000,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
},
|
||||
}
|
||||
|
||||
# Hypothetical model weights (tiny-random-llama) for test only
|
||||
QWEN2_PRETRAINED_RESOURCE_FILES_MAP = {
|
||||
"model_state": {
|
||||
"__internal_testing__/micro-random-llama": "https://bj.bcebos.com/paddlenlp/models/community/__internal_testing__/micro-random-llama/model_state.pdparams",
|
||||
"__internal_testing__/tiny-random-llama": "https://bj.bcebos.com/paddlenlp/models/community/__internal_testing__/tiny-random-llama/model_state.pdparams",
|
||||
},
|
||||
}
|
||||
|
||||
class Qwen2Config(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`Qwen2Model`]. It is used to instantiate a
|
||||
Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of
|
||||
Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta).
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 151936):
|
||||
Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`Qwen2Model`]
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 22016):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 32):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 32):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
||||
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
||||
by meanpooling all the original heads within that group. For more details checkout [this
|
||||
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
||||
The non-linear activation function (function or string) in the decoder.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 32768):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the rms normalization layers.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied.
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
use_sliding_window (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use sliding window attention.
|
||||
sliding_window (`int`, *optional*, defaults to 4096):
|
||||
Sliding window attention (SWA) window size. If not specified, will default to `4096`.
|
||||
max_window_layers (`int`, *optional*, defaults to 28):
|
||||
The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen2Model, Qwen2Config
|
||||
|
||||
>>> # Initializing a Qwen2 style configuration
|
||||
>>> configuration = Qwen2Config()
|
||||
|
||||
>>> # Initializing a model from the Qwen2-7B style configuration
|
||||
>>> model = Qwen2Model(configuration)
|
||||
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "qwen2"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=151936,
|
||||
hidden_size=4096,
|
||||
intermediate_size=22016,
|
||||
num_hidden_layers=32,
|
||||
num_attention_heads=32,
|
||||
num_key_value_heads=32,
|
||||
hidden_act="silu",
|
||||
max_position_embeddings=32768,
|
||||
# seq_length=32768,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-6,
|
||||
use_cache=True,
|
||||
tie_word_embeddings=False,
|
||||
rope_theta=10000.0,
|
||||
pad_token_id=151643,
|
||||
bos_token_id=151643,
|
||||
eos_token_id=151643,
|
||||
use_sliding_window=False,
|
||||
sliding_window=4096,
|
||||
max_window_layers=28,
|
||||
use_flash_attention_for_generation=False,
|
||||
alibi=False,
|
||||
use_last_token_for_generation=False,
|
||||
attention_bias=True,
|
||||
attention_dropout=0.0,
|
||||
rope_scaling_factor=1.0,
|
||||
rope_scaling_type=None,
|
||||
dpo_config=None,
|
||||
use_fused_head_and_loss_fn=False,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
# self.seq_length = seq_length
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.use_sliding_window = use_sliding_window
|
||||
self.sliding_window = sliding_window
|
||||
self.max_window_layers = max_window_layers
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.hidden_act = hidden_act
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.rope_scaling_factor = rope_scaling_factor
|
||||
self.rope_scaling_type = rope_scaling_type
|
||||
|
||||
self.pad_token_id = pad_token_id
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
self.dpo_config = dpo_config
|
||||
self.use_flash_attention_for_generation = use_flash_attention_for_generation
|
||||
self.alibi = alibi
|
||||
self.use_fused_head_and_loss_fn = use_fused_head_and_loss_fn
|
||||
self.use_last_token_for_generation = use_last_token_for_generation
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,364 @@
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from typing import OrderedDict
|
||||
|
||||
import paddle
|
||||
import paddle.distributed.fleet as fleet
|
||||
import paddle.nn as nn
|
||||
from paddle.distributed.fleet.meta_parallel import (
|
||||
LayerDesc,
|
||||
PipelineLayer,
|
||||
SharedLayerDesc,
|
||||
)
|
||||
from paddle.distributed.fleet.recompute.recompute import recompute
|
||||
|
||||
from paddlenlp.transformers.refined_recompute import get_skip_recompute_ops
|
||||
from paddlenlp.transformers.refined_recompute import recompute as rr_recompute
|
||||
|
||||
from ...utils.tools import get_env_device
|
||||
from ..dpo_criterion import DPOCriterion
|
||||
from ..model_utils import PipelinePretrainedModel
|
||||
from .modeling import (
|
||||
Qwen2Config,
|
||||
Qwen2DecoderLayer,
|
||||
Qwen2LMHead,
|
||||
Qwen2Model,
|
||||
Qwen2PretrainedModel,
|
||||
Qwen2PretrainingCriterion,
|
||||
Qwen2RMSNorm,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Qwen2ForCausalLMPipe",
|
||||
]
|
||||
|
||||
|
||||
def parse_args(args):
|
||||
if isinstance(args, tuple):
|
||||
if len(args) == 4:
|
||||
hidden_states, attention_mask, attn_mask_startend_row_indices, position_ids = args
|
||||
elif len(args) == 3:
|
||||
hidden_states, attention_mask, attn_mask_startend_row_indices = args
|
||||
position_ids = None
|
||||
elif len(args) == 2:
|
||||
hidden_states, attention_mask = args
|
||||
attn_mask_startend_row_indices, position_ids = None, None
|
||||
else:
|
||||
hidden_states = args
|
||||
attention_mask, attn_mask_startend_row_indices, position_ids = None, None, None
|
||||
|
||||
if position_ids is not None:
|
||||
position_ids.stop_gradient = True
|
||||
|
||||
if attention_mask is not None:
|
||||
attention_mask.stop_gradient = True
|
||||
|
||||
if attn_mask_startend_row_indices is not None:
|
||||
attn_mask_startend_row_indices.stop_gradient = True
|
||||
|
||||
return hidden_states, attention_mask, attn_mask_startend_row_indices, position_ids
|
||||
|
||||
|
||||
def return_args(hidden_states, attention_mask=None, attn_mask_startend_row_indices=None, position_ids=None):
|
||||
ret = (hidden_states,)
|
||||
|
||||
if attention_mask is not None:
|
||||
ret += (attention_mask.clone(),)
|
||||
if attn_mask_startend_row_indices is not None:
|
||||
ret += (attn_mask_startend_row_indices.clone(),)
|
||||
if position_ids is not None:
|
||||
ret += (position_ids.clone(),)
|
||||
if len(ret) == 1:
|
||||
ret = ret[0]
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def get_attr(layer, name):
|
||||
if getattr(layer, name, None) is not None:
|
||||
return getattr(layer, name, None)
|
||||
else:
|
||||
return get_attr(layer._layer, name)
|
||||
|
||||
|
||||
class Qwen2EmbeddingPipe(nn.Layer):
|
||||
"""Extends QWenEmbeddings to forward attention_mask through the pipeline."""
|
||||
|
||||
def __init__(self, config: Qwen2Config):
|
||||
super(Qwen2EmbeddingPipe, self).__init__()
|
||||
self.config = config
|
||||
self.sequence_parallel = config.sequence_parallel
|
||||
self.hidden_size = config.hidden_size
|
||||
if config.tensor_parallel_degree > 1 and config.vocab_size % config.tensor_parallel_degree == 0:
|
||||
self.embed_tokens = fleet.meta_parallel.VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
weight_attr=paddle.ParamAttr(initializer=nn.initializer.XavierNormal()),
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
|
||||
|
||||
@property
|
||||
def embedding_weight(self):
|
||||
return get_attr(self.embed_tokens, "weight")
|
||||
|
||||
def forward(self, args):
|
||||
"""_summary_
|
||||
|
||||
Args:
|
||||
input (_type_): _description_
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
input_ids, attention_mask, attn_mask_startend_row_indices, position_ids = parse_args(args)
|
||||
input_embeds = self.embed_tokens(input_ids)
|
||||
if self.config.sequence_parallel:
|
||||
from paddlenlp.transformers import ScatterOp
|
||||
|
||||
# [bs, seq_len, num_head * head_dim] -> [bs * seq_len, num_head * head_dim]
|
||||
bs, seq_len, hidden_size = input_embeds.shape
|
||||
input_embeds = paddle.reshape_(input_embeds, [bs * seq_len, hidden_size])
|
||||
# [seq_len * bs / n, num_head * head_dim] (n is mp parallelism)
|
||||
input_embeds = ScatterOp.apply(input_embeds)
|
||||
|
||||
batch_size, seq_length = input_ids.shape
|
||||
|
||||
if attention_mask is not None:
|
||||
assert (
|
||||
attn_mask_startend_row_indices is None
|
||||
), "attention_mask and attn_mask_startend_row_indices can not be set at same time"
|
||||
|
||||
attention_mask = Qwen2Model._prepare_decoder_attention_mask(
|
||||
attention_mask, (batch_size, seq_length), 0, input_embeds.dtype
|
||||
)
|
||||
attention_mask.stop_gradient = True
|
||||
if get_env_device() == "npu":
|
||||
attention_mask = attention_mask.astype("bool")
|
||||
elif get_env_device() == "npu":
|
||||
attention_mask = paddle.tril(paddle.ones((seq_length, seq_length), dtype="bool"))
|
||||
attention_mask.stop_gradient = True
|
||||
|
||||
return return_args(input_embeds, attention_mask, attn_mask_startend_row_indices, position_ids)
|
||||
|
||||
|
||||
class Qwen2DecoderLayerPipe(Qwen2DecoderLayer):
|
||||
def forward(self, args):
|
||||
hidden_states, attention_mask, attn_mask_startend_row_indices, position_ids = parse_args(args)
|
||||
|
||||
has_gradient = not hidden_states.stop_gradient
|
||||
|
||||
if attention_mask is not None and attention_mask.dtype == paddle.int32:
|
||||
attention_mask, attn_mask_startend_row_indices, position_ids = (
|
||||
None,
|
||||
attention_mask,
|
||||
attn_mask_startend_row_indices,
|
||||
)
|
||||
elif attention_mask is not None and attention_mask.dtype == paddle.int64:
|
||||
attention_mask, attn_mask_startend_row_indices, position_ids = None, None, attention_mask
|
||||
elif attn_mask_startend_row_indices is not None and attn_mask_startend_row_indices.dtype == paddle.int64:
|
||||
attn_mask_startend_row_indices, position_ids = None, attn_mask_startend_row_indices
|
||||
|
||||
if self.enable_recompute and self.config.recompute_granularity == "full" and has_gradient:
|
||||
recompute_fn = rr_recompute if any(self.skip_recompute_ops.values()) else recompute
|
||||
if attention_mask is not None or attn_mask_startend_row_indices is not None:
|
||||
hidden_states = recompute_fn(
|
||||
super().forward,
|
||||
hidden_states,
|
||||
position_ids=position_ids,
|
||||
attention_mask=attention_mask,
|
||||
attn_mask_startend_row_indices=attn_mask_startend_row_indices,
|
||||
use_reentrant=False,
|
||||
)
|
||||
else:
|
||||
# for pretrain
|
||||
hidden_states = recompute_fn(
|
||||
super().forward,
|
||||
hidden_states,
|
||||
position_ids=position_ids,
|
||||
attn_mask_startend_row_indices=attn_mask_startend_row_indices,
|
||||
use_reentrant=self.config.recompute_use_reentrant,
|
||||
)
|
||||
else:
|
||||
hidden_states = super().forward(
|
||||
hidden_states,
|
||||
position_ids=position_ids,
|
||||
attention_mask=attention_mask,
|
||||
attn_mask_startend_row_indices=attn_mask_startend_row_indices,
|
||||
)
|
||||
|
||||
return return_args(hidden_states, attention_mask, attn_mask_startend_row_indices, position_ids)
|
||||
|
||||
|
||||
class Qwen2RMSNormPipe(nn.Layer):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.norm = Qwen2RMSNorm(config)
|
||||
|
||||
def forward(self, args):
|
||||
hidden_states, attention_mask, attn_mask_startend_row_indices, position_ids = parse_args(args)
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
class Qwen2LMHeadPipe(Qwen2LMHead):
|
||||
def __init__(self, config, transpose_y=False):
|
||||
super(Qwen2LMHeadPipe, self).__init__(config, transpose_y=transpose_y)
|
||||
|
||||
@property
|
||||
def embedding_weight(self):
|
||||
return get_attr(self, "weight")
|
||||
|
||||
|
||||
class Qwen2ForCausalLMPipe(PipelinePretrainedModel, PipelineLayer):
|
||||
"""QWenForPretraining adapted for pipeline parallelism.
|
||||
|
||||
The largest change is flattening the QWenModel class so we can express it as a
|
||||
sequence of layers including embedding, transformer layers, and output.
|
||||
"""
|
||||
|
||||
config_class = Qwen2Config
|
||||
|
||||
_get_tensor_parallel_mappings = Qwen2PretrainedModel._get_tensor_parallel_mappings
|
||||
_init_weights = Qwen2PretrainedModel._init_weights
|
||||
_keys_to_ignore_on_load_unexpected = Qwen2PretrainedModel._keys_to_ignore_on_load_unexpected
|
||||
_get_model_flops = Qwen2PretrainedModel._get_model_flops
|
||||
_get_hardware_flops = Qwen2PretrainedModel._get_hardware_flops
|
||||
|
||||
_tied_weights_keys = ["lm_head.weight"]
|
||||
|
||||
# DONOT Add base_model_prefix !!!!
|
||||
|
||||
@classmethod
|
||||
def _prepare_pipeline_inputs_func(cls, inputs):
|
||||
|
||||
first_stage_keys = ["input_ids", "attention_mask", "attn_mask_startend_row_indices", "position_ids"]
|
||||
last_stage_keys = ["labels"]
|
||||
|
||||
def get_expected_keys(inputs, keys):
|
||||
ret = tuple([inputs.pop(k) if k in inputs else None for k in keys])
|
||||
if len(ret) == 1:
|
||||
ret = ret[0]
|
||||
return ret
|
||||
|
||||
if type(inputs) is dict or type(inputs) is OrderedDict:
|
||||
return [
|
||||
get_expected_keys(inputs, first_stage_keys),
|
||||
get_expected_keys(inputs, last_stage_keys),
|
||||
]
|
||||
|
||||
keys = list(inputs[0].keys())
|
||||
inputs_batch = {key: [data.pop(key) for data in inputs] for key in keys}
|
||||
return [
|
||||
get_expected_keys(inputs_batch, first_stage_keys),
|
||||
get_expected_keys(inputs_batch, last_stage_keys),
|
||||
]
|
||||
|
||||
def __init__(self, config: Qwen2Config):
|
||||
self.config = config
|
||||
|
||||
# Note that we will actually perform a recompute only if both enable_recompute and layerwise_recompute are set to True
|
||||
# Enable_recompute defaults to False and is controlled by Trainer
|
||||
self.enable_recompute = False
|
||||
self.recompute_granularity = self.config.recompute_granularity
|
||||
self.pp_recompute_interval = self.config.pp_recompute_interval
|
||||
self.no_recompute_layers = config.no_recompute_layers if config.no_recompute_layers is not None else []
|
||||
if self.recompute_granularity == "full":
|
||||
assert len(self.no_recompute_layers) == 0, "for pp with full recompute, no_recompute_layers is not support"
|
||||
|
||||
virtual_pp_degree = getattr(self.config, "virtual_pp_degree", 1)
|
||||
|
||||
def get_hcg():
|
||||
return fleet.get_hybrid_communicate_group()
|
||||
|
||||
hcg = get_hcg()
|
||||
tensor_parallel_degree = max(hcg.get_model_parallel_world_size(), 1)
|
||||
tensor_parallel_rank = max(hcg.get_model_parallel_rank(), 0)
|
||||
|
||||
# TODO: fix tensor_parallel_degree rewrite in here
|
||||
config.tensor_parallel_degree = tensor_parallel_degree
|
||||
config.tensor_parallel_rank = tensor_parallel_rank
|
||||
|
||||
if config.tie_word_embeddings:
|
||||
self.add_sequential_layer(
|
||||
SharedLayerDesc(
|
||||
"qwen2_shared_weight", Qwen2EmbeddingPipe, shared_weight_attr="embedding_weight", config=config
|
||||
),
|
||||
"qwen2",
|
||||
)
|
||||
else:
|
||||
self.add_sequential_layer(LayerDesc(Qwen2EmbeddingPipe, config=config), "qwen2")
|
||||
|
||||
for i in range(config.num_hidden_layers):
|
||||
self.add_sequential_layer(
|
||||
LayerDesc(
|
||||
Qwen2DecoderLayerPipe,
|
||||
config=config,
|
||||
layerwise_recompute=i not in self.no_recompute_layers,
|
||||
skip_recompute_ops=get_skip_recompute_ops(config, i),
|
||||
),
|
||||
f"qwen2.layers.{i}",
|
||||
)
|
||||
self.add_sequential_layer(LayerDesc(Qwen2RMSNormPipe, config=config), "qwen2")
|
||||
|
||||
if config.tie_word_embeddings:
|
||||
self.add_sequential_layer(
|
||||
SharedLayerDesc(
|
||||
"qwen2_shared_weight",
|
||||
Qwen2LMHeadPipe,
|
||||
shared_weight_attr="embedding_weight",
|
||||
config=config,
|
||||
**{"transpose_y": True},
|
||||
),
|
||||
"lm_head",
|
||||
)
|
||||
else:
|
||||
self.add_sequential_layer(LayerDesc(Qwen2LMHeadPipe, config=config), "lm_head")
|
||||
|
||||
recompute_interval = 0
|
||||
if self.enable_recompute and self.recompute_granularity == "full":
|
||||
assert self.config.pp_recompute_interval <= config.num_hidden_layers // (
|
||||
virtual_pp_degree * get_hcg().topology().get_dim_size("pipe")
|
||||
), "pp recompute interval should smaller than num layers of each pp chunk"
|
||||
recompute_interval = self.config.pp_recompute_interval
|
||||
|
||||
seg_method = "layer:Qwen2DecoderLayer"
|
||||
if config.num_hidden_layers % get_hcg().topology().get_dim_size("pipe") != 0:
|
||||
seg_method = "uniform"
|
||||
|
||||
PipelineLayer.__init__(
|
||||
self,
|
||||
layers=self.get_sequential_layers(),
|
||||
loss_fn=self.get_loss_fn(config),
|
||||
topology=get_hcg().topology(),
|
||||
seg_method=seg_method,
|
||||
recompute_interval=recompute_interval,
|
||||
recompute_ctx={
|
||||
"mp_group": get_hcg().get_model_parallel_group(),
|
||||
"offload": False,
|
||||
"partition": False,
|
||||
},
|
||||
num_virtual_pipeline_stages=virtual_pp_degree,
|
||||
)
|
||||
# You should call init here, since there is a diamond inheritance problem
|
||||
self.apply(self._init_weights)
|
||||
# DON'T init PipelinePretrainedModel
|
||||
# PipelinePretrainedModel.__init__(self.super(), config=config)
|
||||
|
||||
def get_loss_fn(self, config):
|
||||
if config.dpo_config is not None:
|
||||
return DPOCriterion(config, use_infohub=True)
|
||||
else:
|
||||
return Qwen2PretrainingCriterion(config)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,448 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for Qwen2."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import unicodedata
|
||||
from functools import lru_cache
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import regex as re
|
||||
|
||||
from ...utils.log import logger
|
||||
from .. import AddedToken, PretrainedTokenizer
|
||||
|
||||
VOCAB_FILES_NAMES = {
|
||||
"vocab_file": "vocab.json",
|
||||
"merges_file": "merges.txt",
|
||||
}
|
||||
|
||||
__all__ = ["Qwen2Tokenizer"]
|
||||
|
||||
MAX_MODEL_INPUT_SIZES = {"__internal_testing__/tiny-random-qwen2": 32768}
|
||||
|
||||
PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
|
||||
characters the bpe code barfs on.
|
||||
|
||||
The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
|
||||
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
|
||||
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
|
||||
tables between utf-8 bytes and unicode strings.
|
||||
"""
|
||||
bs = (
|
||||
list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
|
||||
)
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8 + n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
|
||||
def get_pairs(word):
|
||||
"""
|
||||
Return set of symbol pairs in a word.
|
||||
|
||||
Word is represented as tuple of symbols (symbols being variable-length strings).
|
||||
"""
|
||||
pairs = set()
|
||||
prev_char = word[0]
|
||||
for char in word[1:]:
|
||||
pairs.add((prev_char, char))
|
||||
prev_char = char
|
||||
return pairs
|
||||
|
||||
|
||||
class Qwen2Tokenizer(PretrainedTokenizer):
|
||||
"""
|
||||
Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
|
||||
|
||||
Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
|
||||
be encoded differently whether it is at the beginning of the sentence (without space) or not:
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen2Tokenizer
|
||||
|
||||
>>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
|
||||
>>> tokenizer("Hello world")["input_ids"]
|
||||
[9707, 1879]
|
||||
|
||||
>>> tokenizer(" Hello world")["input_ids"]
|
||||
[21927, 1879]
|
||||
```
|
||||
This is expected.
|
||||
|
||||
You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
|
||||
|
||||
This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
|
||||
this superclass for more information regarding those methods.
|
||||
|
||||
Args:
|
||||
vocab_file (`str`):
|
||||
Path to the vocabulary file.
|
||||
merges_file (`str`):
|
||||
Path to the merges file.
|
||||
errors (`str`, *optional*, defaults to `"replace"`):
|
||||
Paradigm to follow when decoding bytes to UTF-8. See
|
||||
[bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
|
||||
unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
||||
token instead.
|
||||
bos_token (`str`, *optional*):
|
||||
The beginning of sequence token. Not applicable for this tokenizer.
|
||||
eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The end of sequence token.
|
||||
pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The token used for padding, for example when batching sequences of different lengths.
|
||||
clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not the model should cleanup the spaces that were added when splitting the input text during the
|
||||
tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
|
||||
split_special_tokens (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not the special tokens should be split during the tokenization process. The default behavior is
|
||||
to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
|
||||
['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
|
||||
'|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
|
||||
"""
|
||||
|
||||
resource_files_names = VOCAB_FILES_NAMES
|
||||
model_input_names = ["input_ids", "attention_mask", "attn_mask_startend_row_indices"]
|
||||
max_model_input_sizes = MAX_MODEL_INPUT_SIZES
|
||||
|
||||
pretrained_resource_files_map = {
|
||||
"vocab_file": {
|
||||
"__internal_testing__/tiny-random-qwen2": "https://bj.bcebos.com/paddlenlp/models/community/qwen2/vocab.json",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_file,
|
||||
merges_file,
|
||||
errors="replace",
|
||||
unk_token="<|endoftext|>",
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
clean_up_tokenization_spaces=False,
|
||||
split_special_tokens=False,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
if unk_token is None:
|
||||
logger.info("The `unk_token` parameter needs to be defined: we use `eos_token` by default.")
|
||||
unk_token = eos_token
|
||||
|
||||
# Qwen vocab does not contain control tokens; added tokens need to be special
|
||||
bos_token = (
|
||||
AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(bos_token, str)
|
||||
else bos_token
|
||||
)
|
||||
eos_token = (
|
||||
AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(eos_token, str)
|
||||
else eos_token
|
||||
)
|
||||
unk_token = (
|
||||
AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(unk_token, str)
|
||||
else unk_token
|
||||
)
|
||||
pad_token = (
|
||||
AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(pad_token, str)
|
||||
else pad_token
|
||||
)
|
||||
|
||||
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
||||
self.encoder = json.load(vocab_handle)
|
||||
self.decoder = {v: k for k, v in self.encoder.items()}
|
||||
self.errors = errors # how to handle errors in decoding
|
||||
self.byte_encoder = bytes_to_unicode()
|
||||
self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
|
||||
bpe_merges = []
|
||||
with open(merges_file, encoding="utf-8") as merges_handle:
|
||||
for i, line in enumerate(merges_handle):
|
||||
line = line.strip()
|
||||
if (i == 0 and line.startswith("#version:")) or not line:
|
||||
continue
|
||||
bpe_merges.append(tuple(line.split()))
|
||||
self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
|
||||
# NOTE: the cache can grow without bound and will get really large for long running processes
|
||||
# (esp. for texts of language that do not use space between word, e.g. Chinese); technically
|
||||
# not a memory leak but appears as one.
|
||||
# GPT2Tokenizer has the same problem, so let's be consistent.
|
||||
self.cache = {}
|
||||
|
||||
self.pat = re.compile(PRETOKENIZE_REGEX)
|
||||
|
||||
if kwargs.get("add_prefix_space", False):
|
||||
logger.warning_once(
|
||||
f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
errors=errors,
|
||||
bos_token=bos_token,
|
||||
eos_token=eos_token,
|
||||
pad_token=pad_token,
|
||||
unk_token=unk_token,
|
||||
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
||||
split_special_tokens=split_special_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return len(self.encoder)
|
||||
|
||||
def get_vocab(self):
|
||||
return dict(self.encoder, **self.added_tokens_encoder)
|
||||
|
||||
def bpe(self, token):
|
||||
if token in self.cache:
|
||||
return self.cache[token]
|
||||
word = tuple(token)
|
||||
pairs = get_pairs(word)
|
||||
|
||||
if not pairs:
|
||||
return token
|
||||
|
||||
while True:
|
||||
bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
|
||||
if bigram not in self.bpe_ranks:
|
||||
break
|
||||
first, second = bigram
|
||||
new_word = []
|
||||
i = 0
|
||||
while i < len(word):
|
||||
try:
|
||||
j = word.index(first, i)
|
||||
except ValueError:
|
||||
new_word.extend(word[i:])
|
||||
break
|
||||
else:
|
||||
new_word.extend(word[i:j])
|
||||
i = j
|
||||
|
||||
if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
|
||||
new_word.append(first + second)
|
||||
i += 2
|
||||
else:
|
||||
new_word.append(word[i])
|
||||
i += 1
|
||||
new_word = tuple(new_word)
|
||||
word = new_word
|
||||
if len(word) == 1:
|
||||
break
|
||||
else:
|
||||
pairs = get_pairs(word)
|
||||
word = " ".join(word)
|
||||
self.cache[token] = word
|
||||
return word
|
||||
|
||||
def _tokenize(self, text):
|
||||
"""Tokenize a string."""
|
||||
bpe_tokens = []
|
||||
for token in re.findall(self.pat, text):
|
||||
token = "".join(
|
||||
self.byte_encoder[b] for b in token.encode("utf-8")
|
||||
) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
|
||||
bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
|
||||
return bpe_tokens
|
||||
|
||||
def _convert_token_to_id(self, token):
|
||||
"""Converts a token (str) in an id using the vocab."""
|
||||
return self.encoder.get(token, self.added_tokens_encoder.get(token, len(self.encoder)))
|
||||
|
||||
def _convert_id_to_token(self, index):
|
||||
"""Converts an index (integer) in a token (str) using the vocab."""
|
||||
return self.decoder.get(index, self.added_tokens_decoder.get(index, self.unk_token))
|
||||
|
||||
def convert_tokens_to_string(self, tokens):
|
||||
"""Converts a sequence of tokens (string) in a single string."""
|
||||
text = "".join(tokens)
|
||||
text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
|
||||
return text
|
||||
|
||||
def _decode(
|
||||
self,
|
||||
token_ids,
|
||||
skip_special_tokens: bool = False,
|
||||
clean_up_tokenization_spaces: Optional[bool] = False,
|
||||
spaces_between_special_tokens: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
# `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
|
||||
# and cannot be configured elsewhere, but it should default to False for Qwen2Tokenizer
|
||||
return super()._decode(
|
||||
token_ids,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
|
||||
spaces_between_special_tokens=spaces_between_special_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
||||
if not os.path.isdir(save_directory):
|
||||
logger.error(f"Vocabulary path ({save_directory}) should be a directory")
|
||||
return
|
||||
vocab_file = os.path.join(
|
||||
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
|
||||
)
|
||||
merge_file = os.path.join(
|
||||
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
|
||||
)
|
||||
|
||||
with open(vocab_file, "w", encoding="utf-8") as f:
|
||||
f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
|
||||
|
||||
index = 0
|
||||
with open(merge_file, "w", encoding="utf-8") as writer:
|
||||
writer.write("#version: 0.2\n")
|
||||
for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
|
||||
if index != token_index:
|
||||
logger.warning(
|
||||
f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
|
||||
" Please check that the tokenizer is not corrupted!"
|
||||
)
|
||||
index = token_index
|
||||
writer.write(" ".join(bpe_tokens) + "\n")
|
||||
index += 1
|
||||
|
||||
return vocab_file, merge_file
|
||||
|
||||
def prepare_for_tokenization(self, text, **kwargs):
|
||||
text = unicodedata.normalize("NFC", text)
|
||||
return (text, kwargs)
|
||||
|
||||
def _encode_chat_inputs(
|
||||
self,
|
||||
conversations: List[List[str, str]],
|
||||
context_data: Dict[str, Any] = {},
|
||||
system: str = None,
|
||||
add_generation_prompt=True,
|
||||
):
|
||||
result = {}
|
||||
|
||||
# Some template do not support system msg, so we need to check it first.
|
||||
if system:
|
||||
try:
|
||||
self.chat_template.render(messages={"role": "system", "content": system})
|
||||
except Exception as e:
|
||||
raise ValueError("System is not supported in this tokenizer.", e)
|
||||
|
||||
# convert list msg to role dict msg
|
||||
conversation_dict = []
|
||||
origin_msg = []
|
||||
for round in conversations:
|
||||
round_role = [
|
||||
{"role": "user", "content": round[0]},
|
||||
{"role": "assistant", "content": round[1]},
|
||||
]
|
||||
origin_msg.extend(round_role)
|
||||
conversation_dict.append(round_role)
|
||||
|
||||
# Get system string in ChatTemplate
|
||||
# ChatTemplate contains three parts: system, user, and assistant.
|
||||
# However, the system string cannot be obtained directly with the chat_template.render() function.
|
||||
# Thus, three steps are needed to extract the system string.
|
||||
# Step 1: Obtain the combined system and user string in the first round.
|
||||
# Step 2: Obtain the special system string.
|
||||
# Step 3: Obtain the special combined system and user string in the first round.
|
||||
# Then, user string = (special system and user string) - (special system string)
|
||||
# And, system string = (initial system and user string) - (user string)
|
||||
|
||||
assert len(conversation_dict) > 0, "conversations is empty"
|
||||
|
||||
def replace_first_occurrence(original_string, to_find, to_replace):
|
||||
index = original_string.find(to_find)
|
||||
if index == -1: # to_find not found in original_string
|
||||
return original_string
|
||||
else:
|
||||
return original_string[:index] + to_replace + original_string[index + len(to_find) :]
|
||||
|
||||
if system:
|
||||
system_str = self.chat_template.render([system])
|
||||
else:
|
||||
# get system and user str
|
||||
round0_str = self.chat_template.render(
|
||||
messages=conversation_dict[0][:1], add_generation_prompt=False, **self.special_tokens_map
|
||||
)
|
||||
# get special system str
|
||||
round0_only_system_str = self.chat_template.render(
|
||||
messages=[{"role": "system", "content": ""}], add_generation_prompt=False, **self.special_tokens_map
|
||||
)
|
||||
# get special system and user str
|
||||
round0_system_user_str = self.chat_template.render(
|
||||
messages=[{"role": "system", "content": ""}] + conversation_dict[0][:1],
|
||||
add_generation_prompt=False,
|
||||
**self.special_tokens_map,
|
||||
)
|
||||
|
||||
# get user str = {special system and user str} - {special system str}
|
||||
user_str = replace_first_occurrence(round0_system_user_str, round0_only_system_str, "")
|
||||
# get system str = { system and user str} - {user str}
|
||||
system_str = round0_str.replace(user_str, "")
|
||||
|
||||
no_ans = []
|
||||
ans = []
|
||||
for conv in conversation_dict:
|
||||
roundi = [system] + conv if system else conv
|
||||
roundi_str = self.chat_template.render(
|
||||
messages=roundi, add_generation_prompt=False, **self.special_tokens_map
|
||||
)
|
||||
|
||||
roundi_no_ans = [system] + [conv[0]] if system else [conv[0]]
|
||||
roundi_no_ans_str = self.chat_template.render(
|
||||
messages=roundi_no_ans, add_generation_prompt=add_generation_prompt, **self.special_tokens_map
|
||||
)
|
||||
|
||||
roundi_ans_str = roundi_str[len(roundi_no_ans_str) :]
|
||||
ans.append(roundi_ans_str)
|
||||
|
||||
roundi_no_ans_no_system_str = replace_first_occurrence(roundi_no_ans_str, system_str, "")
|
||||
assert (
|
||||
roundi_no_ans_str == system_str + roundi_no_ans_no_system_str
|
||||
), f"the src string contains system str: {system_str}"
|
||||
no_ans.append(roundi_no_ans_no_system_str)
|
||||
|
||||
# the first round is special, we need to add system_str
|
||||
no_ans[0] = system_str + no_ans[0]
|
||||
|
||||
conversation_ids = []
|
||||
for i in range(len(no_ans)):
|
||||
conversation_ids.append(
|
||||
self.batch_encode(
|
||||
[no_ans[i], ans[i]],
|
||||
add_special_tokens=False,
|
||||
padding=False,
|
||||
)["input_ids"]
|
||||
)
|
||||
|
||||
result["conversations"] = conversation_ids
|
||||
return result
|
||||
@@ -0,0 +1,131 @@
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for Qwen2."""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from ..tokenizer_utils import AddedToken
|
||||
from ..tokenizer_utils_fast import PretrainedTokenizerFast
|
||||
from .tokenizer import Qwen2Tokenizer
|
||||
|
||||
VOCAB_FILES_NAMES = {
|
||||
"vocab_file": "vocab.json",
|
||||
"merges_file": "merges.txt",
|
||||
"tokenizer_file": "tokenizer.json",
|
||||
}
|
||||
|
||||
|
||||
MAX_MODEL_INPUT_SIZES = {"qwen/qwen-tokenizer": 32768}
|
||||
|
||||
|
||||
class Qwen2TokenizerFast(PretrainedTokenizerFast):
|
||||
"""
|
||||
Construct a "fast" Qwen2 tokenizer (backed by PaddleNLP's *tokenizers* library). Based on byte-level
|
||||
Byte-Pair-Encoding.
|
||||
|
||||
Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
|
||||
be encoded differently whether it is at the beginning of the sentence (without space) or not:
|
||||
|
||||
```python
|
||||
>>> from transformers import Qwen2TokenizerFast
|
||||
|
||||
>>> tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen-tokenizer")
|
||||
>>> tokenizer("Hello world")["input_ids"]
|
||||
[9707, 1879]
|
||||
|
||||
>>> tokenizer(" Hello world")["input_ids"]
|
||||
[21927, 1879]
|
||||
```
|
||||
This is expected.
|
||||
|
||||
This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
|
||||
refer to this superclass for more information regarding those methods.
|
||||
|
||||
Args:
|
||||
vocab_file (`str`, *optional*):
|
||||
Path to the vocabulary file.
|
||||
merges_file (`str`, *optional*):
|
||||
Path to the merges file.
|
||||
tokenizer_file (`str`, *optional*):
|
||||
Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
|
||||
contains everything needed to load the tokenizer.
|
||||
unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
|
||||
token instead. Not applicable to this tokenizer.
|
||||
bos_token (`str`, *optional*):
|
||||
The beginning of sequence token. Not applicable for this tokenizer.
|
||||
eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The end of sequence token.
|
||||
pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
|
||||
The token used for padding, for example when batching sequences of different lengths.
|
||||
"""
|
||||
|
||||
vocab_files_names = VOCAB_FILES_NAMES
|
||||
resource_files_names = VOCAB_FILES_NAMES
|
||||
model_input_names = ["input_ids", "attention_mask"]
|
||||
slow_tokenizer_class = Qwen2Tokenizer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_file=None,
|
||||
merges_file=None,
|
||||
tokenizer_file=None,
|
||||
unk_token="<|endoftext|>",
|
||||
bos_token=None,
|
||||
eos_token="<|endoftext|>",
|
||||
pad_token="<|endoftext|>",
|
||||
**kwargs,
|
||||
):
|
||||
# We need to at least pass vocab_file and merges_file to base class
|
||||
# in case a slow tokenizer needs to be initialized; other can be
|
||||
# configured through files.
|
||||
# following GPT2TokenizerFast, also adding unk_token, bos_token, and eos_token
|
||||
|
||||
bos_token = (
|
||||
AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(bos_token, str)
|
||||
else bos_token
|
||||
)
|
||||
eos_token = (
|
||||
AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(eos_token, str)
|
||||
else eos_token
|
||||
)
|
||||
unk_token = (
|
||||
AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(unk_token, str)
|
||||
else unk_token
|
||||
)
|
||||
pad_token = (
|
||||
AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
|
||||
if isinstance(pad_token, str)
|
||||
else pad_token
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
vocab_file=vocab_file,
|
||||
merges_file=merges_file,
|
||||
tokenizer_file=tokenizer_file,
|
||||
unk_token=unk_token,
|
||||
bos_token=bos_token,
|
||||
eos_token=eos_token,
|
||||
pad_token=pad_token,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Copied from transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast.save_vocabulary
|
||||
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
||||
files = self._tokenizer.model.save(save_directory, name=filename_prefix)
|
||||
return tuple(files)
|
||||
@@ -0,0 +1,158 @@
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import multiprocessing as mp
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from until.collect_import_modeling import expand_modeling_imports,remove_imports_and_rewrite,save_results_to_txt
|
||||
from until.rewrite_child_classes import rewrite_child_classes
|
||||
from until.rename_identifiers import rename_identifiers
|
||||
|
||||
AUTO_GENERATED_MESSAGE = """# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
||||
# This file was automatically generated from {relative_path}.
|
||||
# Do NOT edit this file manually as any edits will be overwritten by the generation of
|
||||
# the file from the modular. If any change should be done, please apply the change to the
|
||||
# {short_name} file directly. One of our CI enforces this.
|
||||
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
|
||||
"""
|
||||
|
||||
# --- 核心转换逻辑封装 ---
|
||||
def run_converter(file_to_parse_str: str):
|
||||
"""
|
||||
对单个 modular 文件执行完整的转换流程。
|
||||
这个函数是并行处理的基本单元。
|
||||
"""
|
||||
print(file_to_parse_str)
|
||||
file_to_parse = Path(file_to_parse_str)
|
||||
|
||||
# --- 动态确定模型名称和输出路径 ---
|
||||
# 假设：目标模型名可以从文件名推断，例如 "modular_qwen2.py" -> "qwen2"
|
||||
to_name = file_to_parse.stem.replace("modular_", "")
|
||||
# 假设：源模型名在模块化文件中有定义或约定俗成（这里我们先硬编码为 "llama" 作为示例）
|
||||
# 一个更健壮的实现会从 file_to_parse 文件内容中解析出 from_name
|
||||
|
||||
# 输出文件将与输入文件在同一目录下，例如 "modeling_qwen2.py"
|
||||
output_file = file_to_parse.parent / f"modeling_{to_name}.py"
|
||||
temp_merged_file = file_to_parse.parent / (output_file.stem + "_temp_merged.py")
|
||||
|
||||
print(f"--- 开始转换: {to_name} ---")
|
||||
print(f" 输入文件: '{file_to_parse}'")
|
||||
print(f" 最终输出: '{output_file}'")
|
||||
|
||||
# 步骤 1: 收集并展开 import
|
||||
expanded_code , dependencies,from_name= expand_modeling_imports(file_to_parse)
|
||||
#save_results_to_txt(dependencies, "modeling_imports_dependencies.txt")
|
||||
#save_results_to_txt(expanded_code, "modeling_imports_results.txt")
|
||||
#print(from_name)
|
||||
# 步骤 2: 重写子类并生成中间文件
|
||||
relative_path = re.search(
|
||||
r"(transformers/.*|examples/.*)", os.path.abspath(file_to_parse).replace("\\", "/")
|
||||
).group(1)
|
||||
formatted_message=AUTO_GENERATED_MESSAGE.format(relative_path=relative_path, short_name=os.path.basename(relative_path))
|
||||
rewrite_child_classes(expanded_code, file_to_parse,formatted_message, temp_merged_file,rename_map={
|
||||
"llama": "qwen2" # 只需要提供小写形式！
|
||||
})
|
||||
remove_imports_and_rewrite(temp_merged_file)
|
||||
# 步骤 3: 全局重命名
|
||||
try:
|
||||
merged_code = temp_merged_file.read_text(encoding="utf-8")
|
||||
final_code = rename_identifiers(merged_code, from_name, to_name)
|
||||
output_file.write_text(final_code, encoding="utf-8")
|
||||
print(f" ✅ 转换成功，最终代码已写入 '{output_file}'。")
|
||||
except FileNotFoundError:
|
||||
print(f" ❌ [错误] 找不到中间文件 '{temp_merged_file}'，无法进行重命名。")
|
||||
finally:
|
||||
# 清理临时文件
|
||||
if temp_merged_file.exists():
|
||||
temp_merged_file.unlink()
|
||||
|
||||
print(f"--- 转换结束: {to_name} ---\n")
|
||||
|
||||
|
||||
# --- 主执行逻辑 ---
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="将模块化的模型定义文件（modular_*.py）转换为独立的模型文件（modeling_*.py）。"
|
||||
)
|
||||
parser.add_argument(
|
||||
"files",
|
||||
nargs="*",
|
||||
help="要转换的模块化文件列表（可选的位置参数）。",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--files-to-parse", "-f",
|
||||
dest="files_to_parse", # 明确指定存储的目的地
|
||||
default=[], # 默认值改为空列表
|
||||
nargs="+",
|
||||
help="要转换的模块化文件列表。可使用 'all' 或 'examples' 关键字。",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_workers", "-w",
|
||||
default=-1,
|
||||
type=int,
|
||||
help="使用的进程数。默认为 -1，代表使用所有 CPU核心。",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# 合并位置参数和可选参数，以可选参数优先
|
||||
files_to_parse = args.files_to_parse if args.files_to_parse else args.files
|
||||
if not files_to_parse:
|
||||
files_to_parse = ["all"] # 如果未提供任何文件，则默认为 'all'
|
||||
|
||||
num_workers = mp.cpu_count() if args.num_workers == -1 else args.num_workers
|
||||
|
||||
# --- 解析文件路径 ---
|
||||
print(">>> 正在解析需要转换的文件...")
|
||||
if files_to_parse == ["all"]:
|
||||
from pathlib import Path
|
||||
# 确定项目根目录
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = SCRIPT_DIR.parent.parent.parent
|
||||
# 使用绝对路径进行搜索
|
||||
search_path = PROJECT_ROOT / "paddleformers/transformers/**/modular_*.py"
|
||||
files_to_parse = glob.glob(str(search_path), recursive=True)
|
||||
elif files_to_parse == ["examples"]:
|
||||
# 查找所有 examples 目录下的 modular 文件
|
||||
files_to_parse = glob.glob("examples/**/modular_*.py", recursive=True)
|
||||
else:
|
||||
# 将模型简称（如 qwen2）解析为完整路径
|
||||
resolved_files = []
|
||||
for model_name in files_to_parse:
|
||||
if not os.path.exists(model_name):
|
||||
# 尝试在 models 目录下构建路径
|
||||
full_path = os.path.join("src", "transformers", "models", model_name, f"modular_{model_name}.py")
|
||||
if not os.path.isfile(full_path):
|
||||
# 如果找不到，尝试在 examples 目录下构建
|
||||
full_path = os.path.join("examples", "modular-transformers", f"modular_{model_name}.py")
|
||||
|
||||
if not os.path.isfile(full_path):
|
||||
raise ValueError(f"无法为 '{model_name}' 找到模块化文件。请提供完整路径或确认文件名正确。")
|
||||
resolved_files.append(full_path)
|
||||
else:
|
||||
resolved_files.append(model_name)
|
||||
files_to_parse = resolved_files
|
||||
|
||||
if not files_to_parse:
|
||||
print("未找到任何需要转换的文件。")
|
||||
return
|
||||
|
||||
print(f"发现 {len(files_to_parse)} 个文件待处理。")
|
||||
|
||||
|
||||
ordered_files= [files_to_parse]
|
||||
print(ordered_files)
|
||||
|
||||
# --- 按依赖顺序并行处理 ---
|
||||
for i, dependency_level_files in enumerate(ordered_files):
|
||||
print(f"\n>>> 开始处理依赖层级 {i+1}/{len(ordered_files)} ({len(dependency_level_files)} 个文件)...")
|
||||
workers = min(num_workers, len(dependency_level_files))
|
||||
if workers > 0:
|
||||
with mp.Pool(workers) as pool:
|
||||
pool.map(run_converter, dependency_level_files)
|
||||
|
||||
print("\n--- 所有转换任务已完成 ---")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
"""模型组网正确性验证
|
||||
【基本流程】
|
||||
|
||||
定义原模型，加载权重，固定seed，基于numpy生成随机数，转换为PyTorch可以处理的tensor，送入网络，获取输出。
|
||||
|
||||
定义模块化转换后modeling模型，加载权重，固定seed，基于numpy生成随机数，转换为PaddlePaddle可以处理的tensor，送入网络，获取输出。
|
||||
|
||||
排查diff，小于阈值，即可完成自测。
|
||||
"""
|
||||
import numpy as np
|
||||
import paddle
|
||||
from paddleformers.transformers.qwen2 import Qwen2Config
|
||||
from paddleformers.transformers.qwen2.modeling import Qwen2ForCausalLM
|
||||
from paddleformers.transformers import Qwen2Config as Qwen2Config_hf
|
||||
from paddleformers.transformers import Qwen2ForCausalLM as Qwen2ForCausalLM_hf
|
||||
#from paddleformers.transformers.qwen2.test_model_expanded import Qwen2ForCausalLM as Qwen2ForCausalLM_hf
|
||||
|
||||
|
||||
|
||||
def eval_model_convert():
|
||||
paddle_input_ids = paddle.to_tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
|
||||
torch_input_ids = paddle.to_tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
|
||||
|
||||
# paddle model
|
||||
paddle_ckpt_path = "Qwen/Qwen2-0.5B"
|
||||
config_paddle = Qwen2Config.from_pretrained(paddle_ckpt_path)
|
||||
model_paddle = Qwen2ForCausalLM.from_pretrained(paddle_ckpt_path, config=config_paddle, dtype="float32")
|
||||
|
||||
# torch model
|
||||
|
||||
torch_ckpt_path = "Qwen/Qwen2-0.5B"
|
||||
config_torch = Qwen2Config_hf.from_pretrained(torch_ckpt_path)
|
||||
config_torch.dtype = "float32"
|
||||
model_torch = Qwen2ForCausalLM_hf.from_pretrained(torch_ckpt_path, config=config_torch, dtype="float32")
|
||||
|
||||
model_paddle.eval()
|
||||
model_torch.eval()
|
||||
|
||||
out_paddle = model_paddle(paddle_input_ids)[0]
|
||||
out_torch = model_torch(torch_input_ids, return_dict=False)[0]
|
||||
print(out_paddle)
|
||||
print(out_torch)
|
||||
assert np.allclose(out_paddle.numpy(), out_torch.detach().numpy(), rtol=1e-5, atol=1e-3)
|
||||
|
||||
eval_model_convert()
|
||||
@@ -0,0 +1,50 @@
|
||||
import numpy as np
|
||||
import paddle
|
||||
from paddle.distributed import fleet
|
||||
from paddleformers.transformers.qwen2 import Qwen2Config
|
||||
from paddleformers.transformers.qwen2.modeling import Qwen2ForCausalLM
|
||||
from paddleformers.transformers import Qwen2Config as Qwen2Config_hf
|
||||
from paddleformers.transformers import Qwen2ForCausalLM as Qwen2ForCausalLM_hf
|
||||
|
||||
def eval_model_convert_parallel(mp_degree=1):
|
||||
paddle_input_ids = paddle.to_tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
|
||||
torch_input_ids = paddle.to_tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
|
||||
|
||||
strategy = fleet.DistributedStrategy()
|
||||
strategy.hybrid_configs = {
|
||||
"dp_degree": 1,
|
||||
"mp_degree": mp_degree,
|
||||
"pp_degree": 1,
|
||||
"sharding_degree": 1,
|
||||
}
|
||||
fleet.init(is_collective=True, strategy=strategy)
|
||||
hcg = fleet.get_hybrid_communicate_group()
|
||||
|
||||
# paddle model
|
||||
paddle_ckpt_path = "Qwen/Qwen2-0.5B"
|
||||
config_paddle = Qwen2Config.from_pretrained(paddle_ckpt_path)
|
||||
config_paddle.tensor_parallel_degree = hcg.get_model_parallel_world_size()
|
||||
config_paddle.tensor_parallel_rank = hcg.get_model_parallel_rank()
|
||||
config_paddle.tensor_parallel_output = False
|
||||
model_paddle = Qwen2ForCausalLM.from_pretrained(paddle_ckpt_path, config=config_paddle, dtype="float32")
|
||||
|
||||
# torch model
|
||||
torch_ckpt_path = "Qwen/Qwen2-0.5B"
|
||||
config_torch = Qwen2Config_hf.from_pretrained(torch_ckpt_path)
|
||||
config_torch = Qwen2Config.from_pretrained(paddle_ckpt_path)
|
||||
config_torch.tensor_parallel_degree = hcg.get_model_parallel_world_size()
|
||||
config_torch.tensor_parallel_rank = hcg.get_model_parallel_rank()
|
||||
config_torch.tensor_parallel_output = False
|
||||
model_torch = Qwen2ForCausalLM_hf.from_pretrained(torch_ckpt_path, config=config_torch, dtype="float32")
|
||||
|
||||
model_paddle.eval()
|
||||
model_torch.eval()
|
||||
|
||||
# 手动验证
|
||||
out_paddle = model_paddle(paddle_input_ids)[0]
|
||||
out_torch = model_torch(torch_input_ids)[0]
|
||||
print(out_paddle)
|
||||
print(out_torch)
|
||||
assert np.allclose(out_paddle.numpy(), out_torch.detach().numpy(), rtol=1e-5, atol=1e-4)
|
||||
|
||||
eval_model_convert_parallel(mp_degree=2)
|
||||
@@ -0,0 +1,259 @@
|
||||
import libcst as cst
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Set, Union, List, Tuple
|
||||
|
||||
# ==============================================================================
|
||||
# 以下所有函数和类均保持您提供的原始版本，没有任何改动
|
||||
# ==============================================================================
|
||||
def get_unique_module_names(imports_dict: Dict[str, str]) -> Set[str]:
|
||||
"""
|
||||
从字典的值中提取出所有唯一的、纯净的模块名。
|
||||
它会移除前缀 '..' 和末尾的 '.'。
|
||||
"""
|
||||
unique_names = set()
|
||||
|
||||
for prefix_value in imports_dict.values():
|
||||
temp_name = prefix_value
|
||||
|
||||
# 1. 移除开头的 '..'
|
||||
if temp_name.startswith(".."):
|
||||
temp_name = temp_name[2:]
|
||||
|
||||
# 2. 移除末尾的 '.'
|
||||
final_name = temp_name.rstrip('.')
|
||||
|
||||
# 3. 将最终结果添加到集合中，自动保证唯一性
|
||||
if final_name:
|
||||
unique_names.add(final_name)
|
||||
|
||||
return unique_names
|
||||
def get_full_name(node: Union[cst.Name, cst.Attribute, cst.ImportFrom]) -> str:
|
||||
if isinstance(node, cst.Name):
|
||||
return node.value
|
||||
elif isinstance(node, cst.Attribute):
|
||||
return get_full_name(node.value) + "." + node.attr.value
|
||||
elif isinstance(node, cst.ImportFrom):
|
||||
module_parts = []
|
||||
if node.relative:
|
||||
module_parts.append("." * len(node.relative))
|
||||
if node.module:
|
||||
module_parts.append(get_full_name(node.module))
|
||||
return "".join(module_parts)
|
||||
else:
|
||||
return ""
|
||||
|
||||
class ModelingImportCollector(cst.CSTVisitor):
|
||||
def __init__(self):
|
||||
self.imports: Dict[str, str] = {} # name -> module_path
|
||||
self.prefixes_before_modeling: Dict[str, str] = {}
|
||||
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
|
||||
modname = get_full_name(node)
|
||||
if "modeling" in modname:
|
||||
modeling_index = modname.find("modeling")
|
||||
prefix = modname[:modeling_index]
|
||||
for alias in node.names:
|
||||
name_in_scope = alias.evaluated_name
|
||||
self.imports[alias.evaluated_name] = modname
|
||||
self.prefixes_before_modeling[name_in_scope] = prefix
|
||||
|
||||
class DependencyCollector(cst.CSTVisitor):
|
||||
def __init__(self):
|
||||
self.names: Set[str] = set()
|
||||
def visit_Name(self, node: cst.Name) -> None:
|
||||
self.names.add(node.value)
|
||||
|
||||
class ModuleInfoCollector(cst.CSTVisitor):
|
||||
def __init__(self):
|
||||
self.defs: Dict[str, Union[cst.ClassDef, cst.FunctionDef, cst.Assign]] = {}
|
||||
self.imports: Dict[str, Union[cst.Import, cst.ImportFrom]] = {}
|
||||
self.class_stack: List[str] = []
|
||||
def visit_ClassDef(self, node: cst.ClassDef) -> None:
|
||||
self.defs[node.name.value] = node
|
||||
self.class_stack.append(node.name.value)
|
||||
def leave_ClassDef(self, original_node: cst.ClassDef) -> None:
|
||||
self.class_stack.pop()
|
||||
def visit_FunctionDef(self, node: cst.FunctionDef) -> None:
|
||||
if not self.class_stack:
|
||||
self.defs[node.name.value] = node
|
||||
else:
|
||||
fullname = ".".join(self.class_stack + [node.name.value])
|
||||
self.defs[fullname] = node
|
||||
def visit_Assign(self, node: cst.Assign) -> None:
|
||||
if not self.class_stack:
|
||||
for target_wrapper in node.targets:
|
||||
if isinstance(target_wrapper.target, cst.Name):
|
||||
self.defs[target_wrapper.target.value] = node
|
||||
def visit_Import(self, node: cst.Import) -> None:
|
||||
for alias in node.names:
|
||||
name_in_scope = alias.asname.name.value if alias.asname else alias.name.value
|
||||
self.imports[name_in_scope] = node
|
||||
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
|
||||
for alias in node.names:
|
||||
name_in_scope = alias.asname.name.value if alias.asname else alias.name.value
|
||||
self.imports[name_in_scope] = node
|
||||
|
||||
def parse_file(file_path: str) -> Tuple[Dict, Dict, cst.Module]:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
code = f.read()
|
||||
module = cst.parse_module(code)
|
||||
collector = ModuleInfoCollector()
|
||||
module.visit(collector)
|
||||
return collector.defs, collector.imports, module
|
||||
|
||||
def collect_recursive(
|
||||
name: str, defs: Dict[str, cst.CSTNode], imports: Dict[str, cst.CSTNode],
|
||||
seen: Set[str], module: cst.Module,
|
||||
) -> Tuple[Dict[str, str], Set[str], Dict[str, List[str]]]:
|
||||
if name in seen or name not in defs:
|
||||
return {}, set(), {}
|
||||
seen.add(name)
|
||||
node = defs[name]
|
||||
dependencies = {name: []}
|
||||
dep_collector = DependencyCollector()
|
||||
node.visit(dep_collector)
|
||||
results = {name: module.code_for_node(node)}
|
||||
collected_imports = set()
|
||||
for dep in dep_collector.names:
|
||||
if dep in defs and dep not in seen:
|
||||
dep_results, dep_imports , dep_deps = collect_recursive(dep, defs, imports, seen, module)
|
||||
results.update(dep_results)
|
||||
collected_imports.update(dep_imports)
|
||||
dependencies.update(dep_deps)
|
||||
dependencies[name].append(dep) # 记录依赖关系 A -> B
|
||||
elif dep in imports:
|
||||
import_node = imports[dep]
|
||||
import_code = module.code_for_node(import_node)
|
||||
collected_imports.add(import_code)
|
||||
dependencies[name].append(dep)
|
||||
return results, collected_imports, dependencies
|
||||
|
||||
def resolve_file_path(current_file: str, modpath: str) -> Path:
|
||||
dots = len(modpath) - len(modpath.lstrip("."))
|
||||
parts = modpath.lstrip(".").split(".")
|
||||
cur_dir = Path(current_file).parent
|
||||
for _ in range(dots - 1):
|
||||
cur_dir = cur_dir.parent
|
||||
file_path = cur_dir.joinpath(*parts).with_suffix(".py")
|
||||
return file_path if file_path.exists() else None
|
||||
|
||||
def expand_modeling_imports(file_path: str) -> Dict[str, str]:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
code = f.read()
|
||||
module = cst.parse_module(code)
|
||||
imp_collector = ModelingImportCollector()
|
||||
module.visit(imp_collector)
|
||||
expanded_defs = {}
|
||||
all_imports = set()
|
||||
seen = set()
|
||||
dependencies = {}
|
||||
for name, modpath in imp_collector.imports.items():
|
||||
target_file = resolve_file_path(file_path, modpath)
|
||||
if not target_file: continue
|
||||
defs, imports, parsed_module = parse_file(str(target_file))
|
||||
if name in defs:
|
||||
new_defs, new_imports, new_deps = collect_recursive(name, defs, imports, seen, parsed_module)
|
||||
expanded_defs.update(new_defs)
|
||||
all_imports.update(new_imports)
|
||||
dependencies.update(new_deps)
|
||||
expanded = {}
|
||||
for i, import_code in enumerate(sorted(list(all_imports))):
|
||||
expanded[f"__import_{i}__"] = import_code
|
||||
expanded.update(expanded_defs)
|
||||
unique_modules = get_unique_module_names(imp_collector.prefixes_before_modeling)
|
||||
return expanded, dependencies,unique_modules # 返回代码和依赖关系
|
||||
|
||||
def save_results_to_txt(result: Dict[str, str], output_file: str):
|
||||
imports_to_write = []
|
||||
defs_to_write = {}
|
||||
for key, value in result.items():
|
||||
if key.startswith("__import_"):
|
||||
imports_to_write.append(value)
|
||||
else:
|
||||
defs_to_write[key] = value
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
if imports_to_write:
|
||||
f.write("### === Imports === ###\n")
|
||||
for imp in imports_to_write:
|
||||
f.write(f"{imp}\n")
|
||||
f.write("\n" + "="*50 + "\n\n")
|
||||
if defs_to_write:
|
||||
f.write("### === Definitions === ###\n")
|
||||
for k, v in sorted(defs_to_write.items()):
|
||||
f.write(f"=== {k} ===\n")
|
||||
f.write(f"{v}\n\n")
|
||||
|
||||
# ==============================================================================
|
||||
# ### NEW ### 以下是为“文件重写”这一新增功能而添加的全新、独立的模块
|
||||
# ==============================================================================
|
||||
|
||||
class ModelingImportNodeCollector(cst.CSTVisitor):
|
||||
"""一个专门用于收集待删除 import 节点的新 Visitor。"""
|
||||
def __init__(self):
|
||||
self.nodes_to_remove: Set[cst.ImportFrom] = set()
|
||||
|
||||
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
|
||||
modname = get_full_name(node)
|
||||
if "modeling" in modname:
|
||||
self.nodes_to_remove.add(node)
|
||||
|
||||
class ImportRemover(cst.CSTTransformer):
|
||||
"""一个独立的转换器，用于从语法树中删除指定的import节点。"""
|
||||
def __init__(self, nodes_to_remove: Set[cst.ImportFrom]):
|
||||
self.nodes_to_remove = nodes_to_remove
|
||||
|
||||
def leave_ImportFrom(
|
||||
self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom
|
||||
) -> Union[cst.ImportFrom, cst.RemovalSentinel]:
|
||||
if original_node in self.nodes_to_remove:
|
||||
return cst.RemoveFromParent()
|
||||
return updated_node
|
||||
|
||||
def remove_imports_and_rewrite(file_path: str):
|
||||
"""
|
||||
一个独立的函数，封装了文件读取、收集待删除节点、转换和重写的操作。
|
||||
"""
|
||||
# 1. 再次读取和解析文件，以启动独立的重写流程
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
code = f.read()
|
||||
module = cst.parse_module(code)
|
||||
|
||||
# 2. 收集需要删除的节点
|
||||
node_collector = ModelingImportNodeCollector()
|
||||
module.visit(node_collector)
|
||||
|
||||
nodes_to_remove = node_collector.nodes_to_remove
|
||||
if not nodes_to_remove:
|
||||
print(f"No 'modeling' imports found in '{file_path}' to remove.")
|
||||
return
|
||||
|
||||
# 3. 使用转换器生成修改后的代码
|
||||
print(f"Removing {len(nodes_to_remove)} 'modeling' import(s) from '{file_path}'...")
|
||||
remover = ImportRemover(nodes_to_remove)
|
||||
modified_tree = module.visit(remover)
|
||||
|
||||
# 4. 将修改后的代码写回原文件
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(modified_tree.code)
|
||||
print("File rewrite complete.")
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# ### MODIFIED ### 主程序块现在按顺序执行两个功能
|
||||
# ==============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_to_parse = "/home/hsz/PaddleFormers/PaddleFormers/paddleformers/transformers/convert/example/test_model.py"
|
||||
output_filename = "modeling_imports_results.txt"
|
||||
|
||||
# --- 步骤 1: 执行完整的原有功能 ---
|
||||
# 调用函数，其接口和返回值完全没有改变
|
||||
# 同时也修正了之前版本中解包错误的bug
|
||||
combined_results = expand_modeling_imports(file_to_parse)
|
||||
|
||||
# 保存结果，完成原有任务
|
||||
save_results_to_txt(combined_results, output_filename)
|
||||
print(f"Code extraction complete. Results saved to {output_filename}")
|
||||
|
||||
# --- 步骤 2: 在原有功能完成后，独立执行新增的功能 ---
|
||||
remove_imports_and_rewrite(file_to_parse)
|
||||
@@ -0,0 +1,109 @@
|
||||
import libcst as cst
|
||||
from libcst import CSTTransformer
|
||||
import re
|
||||
import os
|
||||
from typing import Set, List, Union
|
||||
|
||||
class GenericRenamerTransformer(CSTTransformer):
|
||||
"""
|
||||
一个通用的CST转换器，用于安全地将代码中的标识符从多个源名称替换为同一个目标名称，
|
||||
并能智能地保留原始名称的大小写风格。
|
||||
"""
|
||||
def __init__(self, from_names: Union[Set[str], List[str]], to_name: str):
|
||||
"""
|
||||
Args:
|
||||
from_names: 要被替换的源名称集合或列表 (例如 {'t5', 'llama', 'utils'})。
|
||||
to_name: 用于替换的目标名称 (例如 'qwen2')。
|
||||
"""
|
||||
self.to_name = to_name
|
||||
|
||||
# 1. 构建一个包含所有源名称的正则表达式 | (OR 逻辑)
|
||||
# - 使用 re.escape() 确保特殊字符被正确处理。
|
||||
# - 使用 | 符号连接所有名称，实现多选一匹配。
|
||||
# - 确保列表非空
|
||||
if not from_names:
|
||||
raise ValueError("from_names 列表不能为空。")
|
||||
|
||||
escaped_names = [re.escape(name) for name in from_names]
|
||||
pattern = "|".join(escaped_names)
|
||||
|
||||
# 2. 编译一个不区分大小写 (re.IGNORECASE) 的正则表达式
|
||||
self.regex = re.compile(pattern, re.IGNORECASE)
|
||||
|
||||
def _case_preserving_replace(self, match: re.Match) -> str:
|
||||
"""
|
||||
这是一个自定义的替换函数，它根据匹配到的字符串的大小写风格，
|
||||
来决定 to_name 应该使用哪种大小写形式。
|
||||
"""
|
||||
found_str = match.group(0)
|
||||
# 如果找到的是全大写 (e.g., LLAMA)
|
||||
if found_str.isupper():
|
||||
return self.to_name.upper()
|
||||
# 如果找到的是首字母大写 (e.g., Llama)
|
||||
if found_str.istitle():
|
||||
return self.to_name.title()
|
||||
# 默认情况，包括全小写 (e.g., llama)，返回全小写
|
||||
return self.to_name.lower()
|
||||
|
||||
def leave_Name(
|
||||
self, original_node: cst.Name, updated_node: cst.Name
|
||||
) -> cst.Name:
|
||||
"""
|
||||
当访问离开一个名称节点时，使用正则表达式和自定义替换函数执行重命名。
|
||||
"""
|
||||
# 使用 regex.sub() 和我们的自定义函数来进行替换
|
||||
new_name_str = self.regex.sub(self._case_preserving_replace, updated_node.value)
|
||||
|
||||
# 仅在名称确实发生改变时才创建一个新节点
|
||||
if new_name_str != updated_node.value:
|
||||
if not new_name_str.isidentifier():
|
||||
original_name = original_node.value
|
||||
# 警告，而不是跳过，因为这在依赖于上下文的重命名中可能是允许的。
|
||||
# 但对于 cst.Name 节点，它必须是有效标识符。
|
||||
print(f"警告：尝试将 '{original_name}' 重命名为无效标识符 '{new_name_str}'。跳过此重命名。")
|
||||
return updated_node
|
||||
return updated_node.with_changes(value=new_name_str)
|
||||
|
||||
return updated_node
|
||||
|
||||
def rename_identifiers(source_code: str, from_names: Union[Set[str], List[str]], to_name: str) -> str:
|
||||
"""
|
||||
接收一段Python源代码，将其中的所有 from_names 相关标识符安全地重命名为 to_name。
|
||||
|
||||
Args:
|
||||
source_code: 包含Python代码的字符串。
|
||||
from_names: 要被替换的源名称集合或列表 (例如 {"t5", "llama"})。
|
||||
to_name: 用于替换的目标名称 (例如 "qwen2")。
|
||||
|
||||
Returns:
|
||||
重构后的Python代码字符串。
|
||||
"""
|
||||
try:
|
||||
module = cst.parse_module(source_code)
|
||||
transformer = GenericRenamerTransformer(from_names, to_name)
|
||||
modified_module = module.visit(transformer)
|
||||
return modified_module.code
|
||||
except cst.ParserSyntaxError as e:
|
||||
print(f"Error: Failed to parse the source code. {e}")
|
||||
return source_code
|
||||
except ValueError as e:
|
||||
print(f"Error in rename process: {e}")
|
||||
return source_code
|
||||
|
||||
# --- 示例用法 ---
|
||||
# source_code = """
|
||||
# class LlamaModel(T5Model):
|
||||
# def forward(self, input_ids):
|
||||
# return self.llama_layer(input_ids)
|
||||
# LLAMA_CONFIG = 1
|
||||
# """
|
||||
# from_list = ['llama', 't5']
|
||||
# to_name = 'qwen2'
|
||||
|
||||
# new_code = rename_identifiers(source_code, from_list, to_name)
|
||||
# print(new_code)
|
||||
# # 预期输出:
|
||||
# # class Qwen2Model(Qwen2Model):
|
||||
# # def forward(self, input_ids):
|
||||
# # return self.qwen2_layer(input_ids)
|
||||
# # QWEN2_CONFIG = 1
|
||||
@@ -0,0 +1,649 @@
|
||||
import libcst as cst
|
||||
from typing import Dict, Optional, List, Set, Union
|
||||
from libcst import matchers as m
|
||||
import builtins
|
||||
import os
|
||||
|
||||
# ==============================================================================
|
||||
# SECTION 1: 智能类合并引擎
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
def get_node_code(node: cst.CSTNode) -> str:
|
||||
"""辅助函数，用于获取CST节点的代码字符串，以便比较。"""
|
||||
return cst.Module(body=[node]).code.strip()
|
||||
|
||||
def merge_parameters(
|
||||
child_params: cst.Parameters, parent_params: cst.Parameters
|
||||
) -> cst.Parameters:
|
||||
"""智能合并两个方法的参数列表。"""
|
||||
child_param_map = {p.name.value: p for p in child_params.params}
|
||||
|
||||
insertion_point = len(child_params.params)
|
||||
for i, p in enumerate(child_params.params):
|
||||
if p.star:
|
||||
insertion_point = i
|
||||
break
|
||||
|
||||
new_params_from_parent = []
|
||||
for p in parent_params.params:
|
||||
if p.name.value not in child_param_map and p.default is not None:
|
||||
new_params_from_parent.append(p)
|
||||
|
||||
final_params_list = list(child_params.params)
|
||||
final_params_list[insertion_point:insertion_point] = new_params_from_parent
|
||||
|
||||
return child_params.with_changes(params=tuple(final_params_list))
|
||||
|
||||
def _get_class_var_names(class_body: list) -> set:
|
||||
"""从类的 body 中提取所有类变量的名称。"""
|
||||
var_names = set()
|
||||
for stmt in class_body:
|
||||
if m.matches(stmt, m.SimpleStatementLine(body=[m.Assign()])):
|
||||
assign_node = stmt.body[0]
|
||||
for target in assign_node.targets:
|
||||
if isinstance(target.target, cst.Name):
|
||||
var_names.add(target.target.value)
|
||||
return var_names
|
||||
|
||||
def merge_parent_class_final(
|
||||
child_class: cst.ClassDef, parent_class: cst.ClassDef
|
||||
) -> cst.ClassDef:
|
||||
"""
|
||||
类合并主函数（最终智能版）：
|
||||
- 智能展开super()调用，避免代码冗余。
|
||||
- 智能合并方法的参数列表，防止运行时错误。
|
||||
- 正确处理类变量和未覆盖方法的继承。
|
||||
"""
|
||||
child_body_list = list(child_class.body.body)
|
||||
parent_body_map = {
|
||||
stmt.name.value: stmt
|
||||
for stmt in parent_class.body.body
|
||||
if hasattr(stmt, 'name') and isinstance(stmt.name, cst.Name)
|
||||
}
|
||||
|
||||
final_body = list(child_body_list)
|
||||
|
||||
# 1. 处理被子类覆盖的方法 (包括 __init__)
|
||||
for i, child_stmt in enumerate(child_body_list):
|
||||
if not isinstance(child_stmt, cst.FunctionDef):
|
||||
continue
|
||||
|
||||
method_name = child_stmt.name.value
|
||||
parent_method = parent_body_map.get(method_name)
|
||||
|
||||
if not parent_method or not isinstance(parent_method, cst.FunctionDef):
|
||||
continue
|
||||
|
||||
# 1a. 智能展开 super()
|
||||
child_method_body = list(child_stmt.body.body)
|
||||
parent_method_body = list(parent_method.body.body)
|
||||
|
||||
super_call_index = -1
|
||||
for j, stmt in enumerate(child_method_body):
|
||||
if m.matches(stmt, m.SimpleStatementLine(body=[m.Expr(value=m.Call(func=m.Attribute(value=m.Call(func=m.Name("super")))))]) ) \
|
||||
or m.matches(stmt, m.Return(value=m.Call(func=m.Attribute(value=m.Call(func=m.Name("super")))))):
|
||||
super_call_index = j
|
||||
break
|
||||
|
||||
new_method_body_stmts = child_method_body
|
||||
if super_call_index != -1:
|
||||
child_prefix_stmts = child_method_body[:super_call_index]
|
||||
child_suffix_stmts = child_method_body[super_call_index + 1:]
|
||||
child_prefix_codes = [get_node_code(s) for s in child_prefix_stmts]
|
||||
|
||||
divergence_index = 0
|
||||
for k, parent_stmt in enumerate(parent_method_body):
|
||||
if k < len(child_prefix_codes) and get_node_code(parent_stmt) == child_prefix_codes[k]:
|
||||
divergence_index += 1
|
||||
else:
|
||||
break
|
||||
|
||||
parent_suffix_stmts = parent_method_body[divergence_index:]
|
||||
new_method_body_stmts = child_prefix_stmts + parent_suffix_stmts + child_suffix_stmts
|
||||
|
||||
# 1b. 合并参数列表
|
||||
new_params = merge_parameters(child_stmt.params, parent_method.params)
|
||||
|
||||
# 1c. 创建最终的方法节点
|
||||
new_body_block = child_stmt.body.with_changes(body=tuple(new_method_body_stmts))
|
||||
final_method = child_stmt.with_changes(body=new_body_block, params=new_params)
|
||||
|
||||
final_body[i] = final_method
|
||||
|
||||
# 2. 添加父类中未被覆盖的成员
|
||||
child_member_names = {stmt.name.value for stmt in final_body if hasattr(stmt, 'name')}
|
||||
child_class_var_names = _get_class_var_names(final_body)
|
||||
|
||||
for parent_stmt in parent_class.body.body:
|
||||
if hasattr(parent_stmt, 'name') and parent_stmt.name.value in child_member_names:
|
||||
continue
|
||||
|
||||
if m.matches(parent_stmt, m.SimpleStatementLine(body=[m.Assign()])):
|
||||
parent_var_names = _get_class_var_names([parent_stmt])
|
||||
if not parent_var_names.isdisjoint(child_class_var_names):
|
||||
continue
|
||||
|
||||
final_body.append(parent_stmt)
|
||||
|
||||
# 3. 清理 pass 语句
|
||||
pass_matcher = m.SimpleStatementLine(body=[m.Pass()])
|
||||
non_pass_statements = [stmt for stmt in final_body if not m.matches(stmt, pass_matcher)]
|
||||
|
||||
if not non_pass_statements:
|
||||
cleaned_body = (cst.SimpleStatementLine(body=(cst.Pass(),)),)
|
||||
else:
|
||||
cleaned_body = tuple(non_pass_statements)
|
||||
|
||||
# 4. 返回最终结果
|
||||
return child_class.with_changes(
|
||||
bases=parent_class.bases,
|
||||
body=child_class.body.with_changes(body=cleaned_body)
|
||||
)
|
||||
|
||||
# ==============================================================================
|
||||
# SECTION 2:代码重构工具框架 (已集成新逻辑)
|
||||
# ==============================================================================
|
||||
|
||||
class ComprehensiveRenamer(cst.CSTTransformer):
|
||||
"""智能、大小写敏感地重命名所有匹配的名称。"""
|
||||
def __init__(self, rename_map: Dict[str, str]):
|
||||
self.rename_pairs = []
|
||||
for from_sub, to_sub in rename_map.items():
|
||||
self.rename_pairs.append((from_sub.lower(), to_sub.lower()))
|
||||
self.rename_pairs.append((from_sub.capitalize(), to_sub.capitalize()))
|
||||
self.rename_pairs.append((from_sub.upper(), to_sub.upper()))
|
||||
self.rename_pairs.sort(key=lambda x: len(x[0]), reverse=True)
|
||||
|
||||
def leave_Name(self, original_node: cst.Name, updated_node: cst.Name) -> cst.Name:
|
||||
for from_name, to_name in self.rename_pairs:
|
||||
if from_name in original_node.value:
|
||||
new_value = original_node.value.replace(from_name, to_name)
|
||||
return updated_node.with_changes(value=new_value)
|
||||
return updated_node
|
||||
|
||||
def get_base_class_name(base: cst.BaseExpression) -> Optional[str]:
|
||||
"""提取基类名称。"""
|
||||
if isinstance(base, cst.Name):
|
||||
return base.value
|
||||
elif isinstance(base, cst.Attribute):
|
||||
parts = []
|
||||
node = base
|
||||
while isinstance(node, cst.Attribute):
|
||||
parts.append(node.attr.value)
|
||||
node = node.value
|
||||
if isinstance(node, cst.Name):
|
||||
parts.append(node.value)
|
||||
return ".".join(reversed(parts))
|
||||
return None
|
||||
|
||||
def find_class_in_source(module_node: cst.Module) -> Optional[cst.ClassDef]:
|
||||
"""从模块节点中提取第一个类定义。"""
|
||||
for node in module_node.body:
|
||||
if isinstance(node, cst.ClassDef):
|
||||
return node
|
||||
return None
|
||||
|
||||
class DependencyVisitor(cst.CSTVisitor):
|
||||
"""扫描代码以查找所有潜在的外部引用。"""
|
||||
def __init__(self):
|
||||
self.scopes: List[Set[str]] = [set()]
|
||||
self.dependencies: Set[str] = set()
|
||||
self.builtins = set(dir(builtins))
|
||||
|
||||
def visit_FunctionDef(self, node: cst.FunctionDef) -> None:
|
||||
param_names = {p.name.value for p in node.params.params}
|
||||
self.scopes.append(param_names)
|
||||
|
||||
def leave_FunctionDef(self, original_node: cst.FunctionDef) -> None:
|
||||
self.scopes.pop()
|
||||
|
||||
def visit_Assign(self, node: cst.Assign) -> None:
|
||||
for target in node.targets:
|
||||
if isinstance(target.target, cst.Name):
|
||||
self.scopes[-1].add(target.target.value)
|
||||
|
||||
def visit_Name(self, node: cst.Name) -> None:
|
||||
is_local = any(node.value in scope for scope in self.scopes)
|
||||
if not is_local and node.value not in self.builtins:
|
||||
self.dependencies.add(node.value)
|
||||
|
||||
def find_usage_dependencies(node: Union[cst.ClassDef, cst.FunctionDef], expanded: Dict[str, str]) -> Set[str]:
|
||||
"""分析节点的CST，找出其使用到的其他实体。"""
|
||||
visitor = DependencyVisitor()
|
||||
node.visit(visitor)
|
||||
return {dep for dep in visitor.dependencies if dep in expanded}
|
||||
|
||||
def get_full_name(node: Union[cst.Name, cst.Attribute, cst.ImportFrom]) -> str:
|
||||
"""
|
||||
从CST节点递归获取完整名称，如 a.b.c 或 ..a.b
|
||||
"""
|
||||
if isinstance(node, cst.Name):
|
||||
return node.value
|
||||
elif isinstance(node, cst.Attribute):
|
||||
# 递归获取基础部分 (a.b)
|
||||
base_name = get_full_name(node.value)
|
||||
# 拼接当前属性 (.c)
|
||||
return f"{base_name}.{node.attr.value}" if base_name else node.attr.value
|
||||
elif isinstance(node, cst.ImportFrom):
|
||||
# 处理 from ... import ... 语句的模块路径
|
||||
module_parts = []
|
||||
if node.relative:
|
||||
module_parts.append("." * len(node.relative))
|
||||
if node.module:
|
||||
module_parts.append(get_full_name(node.module))
|
||||
return "".join(module_parts)
|
||||
return ""
|
||||
|
||||
def filter_specific_modeling_imports(
|
||||
import_nodes: Union[Dict[str, cst.BaseSmallStatement], List[cst.BaseSmallStatement]]
|
||||
) -> Dict[str, cst.BaseSmallStatement]:
|
||||
"""
|
||||
【修正版】只移除严格符合 `from ..***.modeling import ...` 模式的导入。
|
||||
|
||||
这个版本可以智能处理输入是字典或列表的情况，并且总是返回一个字典。
|
||||
"""
|
||||
kept_imports_dict: Dict[str, cst.BaseSmallStatement] = {}
|
||||
|
||||
# 【核心修正】: 检查输入类型，并确保我们总是遍历 CST 节点
|
||||
nodes_to_iterate = []
|
||||
if isinstance(import_nodes, dict):
|
||||
# 如果输入是字典，我们只关心它的值（CST 节点）
|
||||
nodes_to_iterate = list(import_nodes.values())
|
||||
elif isinstance(import_nodes, list):
|
||||
# 如果输入已经是列表，直接使用
|
||||
nodes_to_iterate = import_nodes
|
||||
|
||||
for node in nodes_to_iterate:
|
||||
should_keep = True
|
||||
|
||||
if isinstance(node, cst.ImportFrom):
|
||||
is_two_dots_relative = node.relative and len(node.relative) == 2
|
||||
|
||||
if is_two_dots_relative:
|
||||
module_path = get_full_name(node.module) if node.module else ""
|
||||
|
||||
if module_path.endswith(".modeling"):
|
||||
should_keep = False
|
||||
|
||||
if should_keep:
|
||||
kept_imports_dict[get_node_code(node)] = node
|
||||
|
||||
return kept_imports_dict
|
||||
|
||||
class EntityFinder(cst.CSTVisitor):
|
||||
"""
|
||||
A visitor to find the first ClassDef or FunctionDef node in a CST.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.found_node = None
|
||||
|
||||
def visit_ClassDef(self, node: cst.ClassDef) -> bool:
|
||||
# Found a class, store it and stop searching
|
||||
if self.found_node is None:
|
||||
self.found_node = node
|
||||
return False # Return False to stop traversing deeper
|
||||
|
||||
def visit_FunctionDef(self, node: cst.FunctionDef) -> bool:
|
||||
# Found a function, store it and stop searching
|
||||
if self.found_node is None:
|
||||
self.found_node = node
|
||||
return False # Return False to stop traversing deeper
|
||||
|
||||
def find_entity_in_source(source_cst_node: cst.Module) -> Optional[cst.CSTNode]:
|
||||
"""
|
||||
Parses a CST module to find the first class or function definition.
|
||||
|
||||
Args:
|
||||
source_cst_node: The parsed Concrete Syntax Tree of the source file.
|
||||
|
||||
Returns:
|
||||
The found ClassDef or FunctionDef node, or None if not found.
|
||||
"""
|
||||
if not isinstance(source_cst_node, cst.Module):
|
||||
# Ensure we have a valid CST to visit
|
||||
return None
|
||||
|
||||
finder = EntityFinder()
|
||||
source_cst_node.visit(finder)
|
||||
return finder.found_node
|
||||
|
||||
def rewrite_child_classes(
|
||||
expanded: Dict[str, str],
|
||||
target_file: str,
|
||||
template_comment: str,
|
||||
output_file: str,
|
||||
rename_map: Optional[Dict[str, str]] = None
|
||||
):
|
||||
"""完整的类重写工具 (已集成VFinal版合并引擎)。"""
|
||||
if rename_map is None: rename_map = {}
|
||||
|
||||
# --- 阶段一 & 二：解析代码 ---
|
||||
print("阶段一：正在预解析所有父类代码...")
|
||||
parsed_expanded: Dict[str, cst.Module] = {}
|
||||
imports_to_inject: Dict[str, cst.BaseSmallStatement] = {}
|
||||
for name, source in expanded.items():
|
||||
try:
|
||||
module_node = cst.parse_module(source)
|
||||
parsed_expanded[name] = module_node
|
||||
for node in module_node.body:
|
||||
if m.matches(node, m.SimpleStatementLine(body=[m.Import() | m.ImportFrom()])):
|
||||
imports_to_inject[module_node.code_for_node(node)] = node
|
||||
except Exception as e:
|
||||
print(f"警告：预解析 {name} 失败: {e}")
|
||||
|
||||
print("\n阶段二：正在分析目标文件...")
|
||||
with open(target_file, "r", encoding="utf-8") as f:
|
||||
module = cst.parse_module(f.read())
|
||||
|
||||
imports_from_target: Dict[str, cst.SimpleStatementLine] = {}
|
||||
body_statements: List[cst.BaseStatement] = []
|
||||
for stmt in module.body:
|
||||
# 匹配导入语句
|
||||
if m.matches(stmt, m.SimpleStatementLine(body=[m.Import() | m.ImportFrom()])):
|
||||
imports_from_target[module.code_for_node(stmt)] = stmt
|
||||
|
||||
# 匹配 try-except 块（通常用于可选导入）
|
||||
elif isinstance(stmt, cst.Try):
|
||||
imports_from_target[module.code_for_node(stmt)] = stmt
|
||||
|
||||
# 匹配 __all__ 定义
|
||||
elif m.matches(stmt, m.SimpleStatementLine(body=[m.Assign(targets=[m.AssignTarget(target=m.Name("__all__"))])])):
|
||||
imports_from_target[module.code_for_node(stmt)] = stmt
|
||||
|
||||
# 其他语句放入主体
|
||||
else:
|
||||
body_statements.append(stmt)
|
||||
imports_from_target=filter_specific_modeling_imports(imports_from_target)
|
||||
# --- 阶段三 & 四：依赖分析与合并 ---
|
||||
nodes_to_inject: Dict[str, Union[cst.ClassDef, cst.FunctionDef]] = {}
|
||||
existing_names: Set[str] = {stmt.name.value for stmt in body_statements if hasattr(stmt, 'name')}
|
||||
visiting: Set[str] = set()
|
||||
|
||||
def collect_dependencies(name: str):
|
||||
# 1. 边界检查 (完全不变)
|
||||
# 无论是类还是函数，这些检查（是否已解析、已收集、已存在、正在访问）都同样适用。
|
||||
if name not in parsed_expanded or name in nodes_to_inject or name in existing_names or name in visiting:
|
||||
return
|
||||
|
||||
# 2. 查找实体节点 (需要泛化)
|
||||
# find_entity_in_source 现在可以返回 ClassDef 或 FunctionDef 节点。
|
||||
entity_node = find_entity_in_source(parsed_expanded[name])
|
||||
if not entity_node:
|
||||
return
|
||||
|
||||
# 3. 标记正在访问 (完全不变)
|
||||
visiting.add(name)
|
||||
|
||||
# 4. 处理类特有的依赖：继承 (只对类执行)
|
||||
# 如果实体是类，才处理其父类依赖。函数没有继承，会自然跳过此块。
|
||||
if isinstance(entity_node, cst.ClassDef):
|
||||
for base in entity_node.bases:
|
||||
if base_name := get_base_class_name(base.value):
|
||||
collect_dependencies(base_name)
|
||||
|
||||
# 5. 处理通用依赖：使用关系 (对类和函数都执行)
|
||||
# 这里的 `find_usage_dependencies` 函数也必须是通用的，
|
||||
# 它需要能解析类和函数体内的依赖。
|
||||
# - 对于类: 查找成员变量的类型注解等。
|
||||
# - 对于函数: 查找参数的类型注解、返回值的类型注解、函数体内调用的其他函数、实例化的类等。
|
||||
for dep_name in find_usage_dependencies(entity_node, expanded):
|
||||
collect_dependencies(dep_name)
|
||||
|
||||
# 6. 完成处理，加入结果集 (完全不变)
|
||||
# 无论是类还是函数，都在其所有依赖项被处理完毕后，才将自身加入结果集。
|
||||
visiting.remove(name)
|
||||
nodes_to_inject[name] = entity_node
|
||||
print("\n阶段三：正在进行全局依赖扫描...")
|
||||
for stmt in body_statements:
|
||||
if isinstance(stmt, cst.ClassDef):
|
||||
for base in stmt.bases:
|
||||
if base_name := get_base_class_name(base.value):
|
||||
collect_dependencies(base_name)
|
||||
for dep_name in find_usage_dependencies(stmt, expanded):
|
||||
collect_dependencies(dep_name)
|
||||
|
||||
print("\n阶段四：正在执行类合并操作...")
|
||||
processed_body_statements = []
|
||||
merged_parents: Set[str] = set()
|
||||
for stmt in body_statements:
|
||||
if isinstance(stmt, cst.ClassDef) and stmt.bases:
|
||||
if base_name := get_base_class_name(stmt.bases[0].value):
|
||||
if base_name in parsed_expanded:
|
||||
parent_module = parsed_expanded[base_name]
|
||||
if parent_class_node := find_class_in_source(parent_module):
|
||||
print(f" > 正在合并 {base_name} -> {stmt.name.value}...")
|
||||
# <<<--- ★★★核心修改点：调用新的合并函数★★★
|
||||
stmt = merge_parent_class_final(stmt, parent_class_node)
|
||||
merged_parents.add(base_name)
|
||||
processed_body_statements.append(stmt)
|
||||
|
||||
# --- 阶段五：按正确顺序重新组装文件 ---
|
||||
print("\n阶段五：正在生成最终文件...")
|
||||
|
||||
nodes_to_inject_after_merge = {k: v for k, v in nodes_to_inject.items() if k not in merged_parents}
|
||||
main_defined_names = {stmt.name.value for stmt in processed_body_statements if hasattr(stmt, 'name')}
|
||||
|
||||
print(" > 正在应用智能重命名规则并检测冲突...")
|
||||
final_nodes_to_inject = {}
|
||||
renamer = ComprehensiveRenamer(rename_map)
|
||||
|
||||
for original_name, node in nodes_to_inject_after_merge.items():
|
||||
renamed_node = node.visit(renamer)
|
||||
new_name = renamed_node.name.value
|
||||
if new_name in main_defined_names:
|
||||
print(f" - 检测到主代码中已存在 '{new_name}'，将跳过注入 '{original_name}'")
|
||||
continue
|
||||
print(f" - 正在处理依赖 '{original_name}'...")
|
||||
final_nodes_to_inject[new_name] = renamed_node
|
||||
|
||||
final_imports = {**imports_from_target, **imports_to_inject}
|
||||
new_body = []
|
||||
new_header = []
|
||||
#加转换注释
|
||||
for line in template_comment.splitlines():
|
||||
stripped_line = line.strip()
|
||||
if stripped_line:
|
||||
comment_node = cst.Comment(stripped_line)
|
||||
new_header.append(cst.EmptyLine(
|
||||
comment=comment_node,
|
||||
indent=True,
|
||||
whitespace=cst.SimpleWhitespace(value="")
|
||||
))
|
||||
for item in module.header:
|
||||
if isinstance(item, cst.EmptyLine) and item.comment:
|
||||
new_header.append(item)
|
||||
elif isinstance(item, cst.TrailingWhitespace) and item.comment:
|
||||
new_header.append(item)
|
||||
|
||||
if final_imports:
|
||||
unique_imports = {module.code_for_node(n): n for n in final_imports.values()}
|
||||
new_body.extend(unique_imports.values())
|
||||
|
||||
injected_items = sorted(final_nodes_to_inject.values(), key=lambda n: n.name.value)
|
||||
# 2. 分类依赖项：方法和类
|
||||
methods_to_inject = []
|
||||
classes_to_inject = []
|
||||
for node in injected_items:
|
||||
if isinstance(node, cst.FunctionDef):
|
||||
print(node.name.value)
|
||||
methods_to_inject.append(node)
|
||||
elif isinstance(node, cst.ClassDef):
|
||||
classes_to_inject.append(node)
|
||||
else:
|
||||
print(f"警告：遇到未知类型的节点，无法分类: {type(node.name.value)}")
|
||||
# 3. 注入方法（放在 imports 之后，主逻辑之前）
|
||||
if methods_to_inject:
|
||||
new_body.extend([cst.EmptyLine(), cst.EmptyLine(comment=cst.Comment("# --- Injected Methods ---"))])
|
||||
new_body.extend(methods_to_inject)
|
||||
# 4. 处理类的注入顺序
|
||||
# 分组：有父类在主逻辑中的类 vs 没有的
|
||||
classes_with_parent_in_main = []
|
||||
classes_without_parent_in_main = []
|
||||
if classes_to_inject:
|
||||
# 获取主逻辑中的所有类名
|
||||
main_classes = {stmt.name.value for stmt in processed_body_statements if isinstance(stmt, cst.ClassDef)}
|
||||
|
||||
|
||||
|
||||
for cls_node in classes_to_inject:
|
||||
has_parent_in_main = False
|
||||
if isinstance(cls_node, cst.ClassDef) and cls_node.bases:
|
||||
for base in cls_node.bases:
|
||||
if base_name := get_base_class_name(base.value):
|
||||
if base_name in main_classes:
|
||||
has_parent_in_main = True
|
||||
break
|
||||
|
||||
if has_parent_in_main:
|
||||
classes_with_parent_in_main.append(cls_node)
|
||||
else:
|
||||
classes_without_parent_in_main.append(cls_node)
|
||||
|
||||
# 4.1 先注入没有父类依赖的类（放在 imports 之后）
|
||||
if classes_without_parent_in_main:
|
||||
new_body.extend([cst.EmptyLine(), cst.EmptyLine(comment=cst.Comment("# --- Injected Classes ---"))])
|
||||
new_body.extend(classes_without_parent_in_main)
|
||||
|
||||
|
||||
# 4. 动态遍历主逻辑，在父类定义后插入其子类
|
||||
if processed_body_statements:
|
||||
# 4.1 收集所有主逻辑的类名
|
||||
classes_with_parent_in_main = {
|
||||
cls for cls in classes_with_parent_in_main
|
||||
if isinstance(cls, cst.ClassDef)
|
||||
}
|
||||
|
||||
# 4.2 按顺序处理主逻辑的语句
|
||||
for stmt in processed_body_statements:
|
||||
new_body.append(stmt)
|
||||
|
||||
# 如果是类定义，检查是否有子类需要注入
|
||||
if isinstance(stmt, cst.ClassDef):
|
||||
parent_name = stmt.name.value
|
||||
# 查找依赖此父类的子类
|
||||
child_classes = [
|
||||
cls for cls in classes_with_parent_in_main
|
||||
if any(
|
||||
get_base_class_name(base.value) == parent_name
|
||||
for base in cls.bases
|
||||
)
|
||||
]
|
||||
# 注入子类
|
||||
if child_classes:
|
||||
new_body.extend([
|
||||
cst.EmptyLine(),
|
||||
cst.EmptyLine(comment=cst.Comment(f"# --- Children of {parent_name} ---")),
|
||||
*child_classes
|
||||
])
|
||||
# 从待注入列表中移除已处理的子类
|
||||
classes_with_parent_in_main = [
|
||||
cls for cls in classes_with_parent_in_main
|
||||
if cls not in child_classes
|
||||
]
|
||||
|
||||
# 5. 注入剩余未处理的依赖主逻辑的类（可能是跨文件的依赖）
|
||||
if classes_with_parent_in_main:
|
||||
new_body.extend([cst.EmptyLine(), cst.EmptyLine(comment=cst.Comment("# --- Remaining Injected Child Classes ---"))])
|
||||
new_body.extend(classes_with_parent_in_main)
|
||||
|
||||
"""
|
||||
if injected_items:
|
||||
new_body.extend([cst.EmptyLine(), cst.EmptyLine(comment=cst.Comment("# --- Injected Dependencies ---"))])
|
||||
new_body.extend(injected_items)
|
||||
|
||||
if processed_body_statements:
|
||||
new_body.extend([cst.EmptyLine(), cst.EmptyLine(comment=cst.Comment("# --- Main Application Logic ---"))])
|
||||
new_body.extend(processed_body_statements)
|
||||
"""
|
||||
new_module = module.with_changes(
|
||||
header=tuple(new_header), # 使用新的头部注释
|
||||
body=tuple(new_body) # 使用新的主体内容
|
||||
)
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
f.write(new_module.code)
|
||||
|
||||
print(f"\n成功生成合并后的文件: {output_file}")
|
||||
|
||||
# ==============================================================================
|
||||
# SECTION 3: 演示
|
||||
# ==============================================================================
|
||||
if __name__ == "__main__":
|
||||
|
||||
# --- 步骤1: 准备演示环境 ---
|
||||
# 创建一个虚拟的 child_class.py 文件供脚本读取
|
||||
child_class_content = """
|
||||
class MyChildClass(ParentClass):
|
||||
def __init__(self, config, child_param):
|
||||
# 与父类重复的语句
|
||||
if config.flag:
|
||||
self.param1 = config.param1
|
||||
else:
|
||||
self.param1 = config.default_param1
|
||||
|
||||
# 调用super
|
||||
super().__init__(config)
|
||||
|
||||
# 新增的属性和逻辑
|
||||
self.child_param = child_param
|
||||
print("Child class logic executed.")
|
||||
|
||||
def child_method(self):
|
||||
return "子类方法"
|
||||
"""
|
||||
with open("child_class.py", "w", encoding="utf-8") as f:
|
||||
f.write(child_class_content)
|
||||
|
||||
# --- 步骤2: 定义父类和祖父类源代码 ---
|
||||
expanded_parents = {
|
||||
"ParentClass": '''
|
||||
class ParentClass(GrandParentClass):
|
||||
def __init__(self, config):
|
||||
# 条件语句
|
||||
if config.flag:
|
||||
self.param1 = config.param1
|
||||
else:
|
||||
self.param1 = config.default_param1
|
||||
|
||||
# 循环语句
|
||||
for i in range(5):
|
||||
self.param2 = i
|
||||
|
||||
# 方法调用
|
||||
self.initialize(config)
|
||||
|
||||
# super调用（指向祖父类）
|
||||
super().__init__()
|
||||
|
||||
def initialize(self, config):
|
||||
self.param3 = config.param3
|
||||
|
||||
def parent_method(self):
|
||||
return "父类方法"
|
||||
''',
|
||||
"GrandParentClass": '''
|
||||
class GrandParentClass:
|
||||
def __init__(self):
|
||||
self.grand_param = "祖父参数"
|
||||
|
||||
def grand_method(self):
|
||||
return "祖父方法"
|
||||
'''
|
||||
}
|
||||
|
||||
# --- 步骤3: 运行重写工具 ---
|
||||
print("--- 开始运行代码重写工具 ---")
|
||||
rewrite_child_classes(
|
||||
expanded=expanded_parents,
|
||||
target_file="child_class.py",
|
||||
output_file="merged_class.py"
|
||||
)
|
||||
|
||||
# --- 步骤4: 打印结果 ---
|
||||
print("\n--- 查看生成的 merged_class.py 文件 ---")
|
||||
with open("merged_class.py", "r", encoding="utf-8") as f:
|
||||
print(f.read())
|
||||
|
||||
# --- 步骤5: 清理 ---
|
||||
os.remove("child_class.py")
|
||||
os.remove("merged_class.py")
|
||||
Executable
+4520
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,782 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
|
||||
export log_path=/workspace/case_logs
|
||||
export root_path=/workspace/PaddleNLP
|
||||
|
||||
export gpt_case_path=$root_path/slm/model_zoo/gpt-3
|
||||
export gpt_data_path=/fleetx_data
|
||||
|
||||
export llm_gpt_case_path=$root_path/llm
|
||||
export llm_gpt_data_path=/llm_gpt_data
|
||||
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
|
||||
function is_cuda123() {
|
||||
if [ $(nvcc -V|grep "cuda_12.3" |wc -l) -ne 0 ];then
|
||||
echo 1
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
IS_CUDA123=$(is_cuda123)
|
||||
|
||||
function track_case_status() {
|
||||
local case_name="$1"
|
||||
local prefix="$2"
|
||||
local original_path
|
||||
|
||||
original_path=$(pwd)
|
||||
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
|
||||
|
||||
total_count=$(ls -1 "$prefix"* 2>/dev/null | grep -Ev 'result\.log|functions\.txt' | wc -l)
|
||||
run_fail_count=$(ls -1 "$prefix"*_FAIL* 2>/dev/null | wc -l)
|
||||
loss_fail_count=$(grep 'check failed! ' result.log | awk -v prefix="$prefix" '{if ($2 ~ "^" prefix) print $2}'| wc -l)
|
||||
|
||||
echo -e "\033[31m ---- $case_name total tests : $total_count \033"
|
||||
if [ $run_fail_count -eq 0 ] && [ $loss_fail_count -eq 0 ]; then
|
||||
echo -e "\033[32m ---- all cases Success \033"
|
||||
else
|
||||
if [[ $run_fail_count -ne 0 ]] ; then
|
||||
echo -e "\033[31m ---- $case_name runtime failed test : $run_fail_count \033"
|
||||
ls -1 "$prefix"*_FAIL* 2>/dev/null | awk -v OFS="\t" '{print "\t" $0 "(failed)"}'
|
||||
fi
|
||||
if [[ $loss_fail_count -ne 0 ]] ; then
|
||||
echo -e "\033[31m ---- $case_name verification failed test : $loss_fail_count \033"
|
||||
grep 'check failed! ' result.log | awk -v prefix="$prefix" 'BEGIN {OFS="\t"} {if ($2 ~ "^" prefix) print "\t" $2 "(failed)"}'
|
||||
fi
|
||||
return 2
|
||||
fi
|
||||
cd "$original_path" || { echo "Failed to return to original path: $original_path"; return 1; }
|
||||
return 0
|
||||
}
|
||||
|
||||
function restore_func() {
|
||||
fun_list=$1
|
||||
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
|
||||
if [ -e "functions.txt" ]; then
|
||||
rm "functions.txt"
|
||||
echo "Deleted existing functions.txt"
|
||||
fi
|
||||
if [ ! -f "${log_path}/blacklist.csv" ]; then
|
||||
wget -q -P ${log_path}/ https://paddle-qa.bj.bcebos.com/Auto-Parallel/blacklist.csv --no-proxy || exit 101
|
||||
echo "\033 ---- wget blacklist.csv \033"
|
||||
fi
|
||||
blacklist_file=${log_path}/blacklist.csv
|
||||
mapfile -t blacklist < "$blacklist_file"
|
||||
for function in ${fun_list[@]};do
|
||||
if [[ " ${blacklist[@]} " == *" ${function} "* ]]; then
|
||||
echo "\033 ---- Function '$function' is blacklisted and will be skipped. \033"
|
||||
else
|
||||
echo "$function" >> functions.txt
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function gpt_case_list_dygraph() {
|
||||
fun_list=(
|
||||
# The test name must have "gpt_" as a prefix, which will
|
||||
# be used for tracking the execution status of the case.
|
||||
gpt_preprocess_data
|
||||
gpt_345M_single
|
||||
gpt_1.3B_dp
|
||||
gpt_6.7B_stage2_dp2_sharding4
|
||||
gpt_6.7B_stage3_dp2_sharding4
|
||||
gpt_6.7B_stage2_sharding8
|
||||
gpt_175B_DP1_MP4_PP2
|
||||
gpt_175B_DP1_MP4_PP2_sp
|
||||
gpt_175B_DP1_MP8_PP1
|
||||
gpt_175B_DP1_MP8_PP1_sp
|
||||
gpt_175B_DP1_MP1_PP8
|
||||
gpt_generation_345M_single
|
||||
gpt_generation_345M_hybrid
|
||||
# gpt_345M_mp8_qat
|
||||
# gpt_export_345M_mp1
|
||||
# gpt_export_345M_mp2
|
||||
# gpt_export_qat_345M
|
||||
# gpt_inference_345M_single
|
||||
# gpt_inference_345M_dp8
|
||||
gpt_345M_single_finetune
|
||||
gpt_eval_WikiText
|
||||
gpt_eval_LAMBADA
|
||||
)
|
||||
if [ $1 = "prepare_case" ]; then
|
||||
restore_func $fun_list
|
||||
elif [ $1 = "exec_case" ]; then
|
||||
for fun in "${fun_list[@]}"; do
|
||||
eval "$fun"
|
||||
done
|
||||
track_case_status $FUNCNAME "gpt_"
|
||||
else
|
||||
echo -e "\033[31m ---- Invalid status $1 \033[0m"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function llm_gpt_case_list_dygraph() {
|
||||
fun_list=(
|
||||
# The test name must have "llm_gpt_" as a prefix, which will
|
||||
# be used for tracking the execution status of the case.
|
||||
llm_gpt_recompute_bs32_bf16_MP2-SD4-stage1
|
||||
)
|
||||
if [ $1 = "prepare_case" ]; then
|
||||
restore_func $fun_list
|
||||
elif [ $1 = "exec_case" ]; then
|
||||
for fun in "${fun_list[@]}"; do
|
||||
eval "$fun"
|
||||
done
|
||||
track_case_status $FUNCNAME "llm_gpt_"
|
||||
else
|
||||
echo -e "\033[31m ---- Invalid status $1 \033[0m"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
############ case start ############
|
||||
function gpt_preprocess_data() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python ppfleetx/data/data_tools/gpt/raw_trans_to_json.py \
|
||||
--input_path ./dataset/wikitext_103_en \
|
||||
--output_path ./dataset/wikitext_103_en/wikitext_103_en \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python ppfleetx/data/data_tools/gpt/preprocess_data.py \
|
||||
--model_name gpt2 \
|
||||
--tokenizer_name GPTTokenizer \
|
||||
--data_format JSON \
|
||||
--input_path ./dataset/wikitext_103_en/wikitext_103_en.jsonl \
|
||||
--append_eos \
|
||||
--output_prefix ./dataset/wikitext_103_en/wikitext_103_en \
|
||||
--workers 40 \
|
||||
--log_interval 1000 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_345M_single() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python tools/train.py \
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_345M_single_card.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_1.3B_dp() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_6.7B_stage2_dp2_sharding4() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" \
|
||||
tools/train.py -c ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Distributed.sharding.sharding_degree=4 -o Distributed.sharding.sharding_stage=2 \
|
||||
-o Distributed.sharding.reduce_overlap=False -o Distributed.sharding.broadcast_overlap=False \
|
||||
-o Engine.logging_freq=5 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_6.7B_stage3_dp2_sharding4() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" \
|
||||
tools/train.py -c ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Distributed.sharding.sharding_degree=4 -o Distributed.sharding.sharding_stage=3 \
|
||||
-o Distributed.sharding.reduce_overlap=False -o Distributed.sharding.broadcast_overlap=False \
|
||||
-o Engine.logging_freq=5 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_6.7B_stage2_sharding8() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" \
|
||||
tools/train.py -c ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=20 -o Engine.eval_freq=20 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Distributed.sharding.sharding_degree=8 -o Distributed.sharding.sharding_stage=2 \
|
||||
-o Distributed.sharding.reduce_overlap=True -o Distributed.sharding.broadcast_overlap=True \
|
||||
-o Engine.logging_freq=5 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_175B_DP1_MP4_PP2() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml \
|
||||
-o Model.hidden_size=1024 -o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Global.local_batch_size=16 -o Global.micro_batch_size=2 \
|
||||
-o Distributed.mp_degree=4 -o Distributed.pp_degree=2 \
|
||||
-o Model.sequence_parallel=False \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_175B_DP1_MP4_PP2_sp() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml \
|
||||
-o Model.hidden_size=1024 -o Model.num_layers=4 -o Model.num_attention_heads=4 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Global.local_batch_size=16 -o Global.micro_batch_size=2 \
|
||||
-o Distributed.mp_degree=4 -o Distributed.pp_degree=2 -o Model.sequence_parallel=True \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_175B_DP1_MP8_PP1() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml \
|
||||
-o Model.hidden_size=1024 -o Model.num_layers=16 -o Model.num_attention_heads=16 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Global.local_batch_size=16 -o Global.micro_batch_size=2 \
|
||||
-o Distributed.mp_degree=8 -o Distributed.pp_degree=1 \
|
||||
-o Model.sequence_parallel=False \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_175B_DP1_MP8_PP1_sp() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml \
|
||||
-o Model.hidden_size=1024 -o Model.num_layers=16 -o Model.num_attention_heads=16 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Global.local_batch_size=16 -o Global.micro_batch_size=2 \
|
||||
-o Distributed.mp_degree=8 -o Distributed.pp_degree=1 -o Model.sequence_parallel=True \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_175B_DP1_MP1_PP8() {
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml \
|
||||
-o Model.hidden_size=1024 -o Model.num_layers=32 -o Model.num_attention_heads=16 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
-o Global.local_batch_size=16 -o Global.micro_batch_size=1 \
|
||||
-o Distributed.mp_degree=1 -o Distributed.pp_degree=8 \
|
||||
-o Model.virtual_pp_degree=2 -o Distributed.pp_recompute_interval=2 \
|
||||
-o Model.fused_linear=True -o Model.use_recompute=True \
|
||||
-o Model.sequence_parallel=False \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_345M_mp8_qat() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0,1,2,3,4,5,6,7" tools/train.py\
|
||||
-c ppfleetx/configs/nlp/gpt/qat_gpt_345M_mp8.yaml \
|
||||
-o Model.num_layers=4 -o Model.num_attention_heads=8 \
|
||||
-o Engine.max_steps=10 -o Engine.eval_freq=10 \
|
||||
-o Engine.eval_iters=5 -o Engine.save_load.save_steps=10 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_generation_345M_single() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python tasks/gpt/generation.py \
|
||||
-c ppfleetx/configs/nlp/gpt/generation_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826/ \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_generation_345M_hybrid() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python -m paddle.distributed.launch --devices "0" tasks/gpt/generation.py \
|
||||
-c ppfleetx/configs/nlp/gpt/generation_gpt_345M_dp8.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826/ \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_export_345M_mp1() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
log_dir=log_export
|
||||
rm -rf $log_dir
|
||||
rm -rf output
|
||||
|
||||
export PYTHONPATH=$root_path/slm/model_zoo/gpt-3:$PYTHONPATH
|
||||
export CUDA_VISIBLE_DEVICES=1
|
||||
python -m paddle.distributed.launch --log_dir $log_dir --devices "1" \
|
||||
./tools/auto_export.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/auto/generation_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./pretrained/inference_model \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python -m paddle.distributed.launch --devices "1" \
|
||||
projects/gpt/inference.py --mp_degree 1 --model_dir output \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_export_345M_mp2() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
log_dir=log_export
|
||||
rm -rf $log_dir
|
||||
rm -rf output
|
||||
|
||||
export PYTHONPATH=$root_path/slm/model_zoo/gpt-3:$PYTHONPATH
|
||||
export CUDA_VISIBLE_DEVICES=0,1
|
||||
python -m paddle.distributed.launch --devices "0,1" \
|
||||
./tools/auto_export.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/auto/generation_gpt_345M_mp2.yaml \
|
||||
-o Generation.use_topp_sampling=False \
|
||||
-o Engine.save_load.ckpt_dir=./pretrained/inference_model \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python -m paddle.distributed.launch --devices "0,1" \
|
||||
projects/gpt/inference.py --mp_degree 2 --model_dir output \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_export_qat_345M() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
log_dir=log_export
|
||||
rm -rf $log_dir
|
||||
rm -rf output
|
||||
|
||||
python ./tools/export.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/generation_qat_gpt_345M_single_card.yaml \
|
||||
-o Model.hidden_dropout_prob=0.0 \
|
||||
-o Model.attention_probs_dropout_prob=0.0 \
|
||||
-o Engine.save_load.ckpt_dir='./GPT_345M_QAT_wo_analysis/' \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python -m paddle.distributed.launch --devices "0" \
|
||||
projects/gpt/inference.py --mp_degree 1 --model_dir output \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_inference_345M_single() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
rm -rf output
|
||||
python tools/export.py \
|
||||
-c ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826/ \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python tasks/gpt/inference.py \
|
||||
-c ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_inference_345M_dp8() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
rm -rf output
|
||||
python -m paddle.distributed.launch --devices "0" tools/export.py \
|
||||
-c ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826/ \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
python -m paddle.distributed.launch --devices "0" \
|
||||
tasks/gpt/inference.py \
|
||||
-c ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_345M_single_finetune() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python ./tools/train.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/finetune_gpt_345M_single_card_glue.yaml \
|
||||
-o Engine.num_train_epochs=1 \
|
||||
-o Data.Train.dataset.name=WNLI \
|
||||
-o Data.Train.dataset.root=./dataset/WNLI/ \
|
||||
-o Data.Eval.dataset.name=WNLI \
|
||||
-o Data.Eval.dataset.root=./dataset/WNLI/ \
|
||||
-o Data.Eval.dataset.split=dev \
|
||||
-o Model.num_classes=2 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_eval_WikiText() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python ./tools/eval.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/eval_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826 \
|
||||
-o Offline_Eval.eval_path=./wikitext-103/wiki.valid.tokens \
|
||||
-o Offline_Eval.overlapping_eval=32 \
|
||||
-o Offline_Eval.batch_size=16 \
|
||||
-o Engine.max_steps=20 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function gpt_eval_LAMBADA() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
rm -rf log
|
||||
python ./tools/eval.py \
|
||||
-c ./ppfleetx/configs/nlp/gpt/eval_gpt_345M_single_card.yaml \
|
||||
-o Engine.save_load.ckpt_dir=./ckpt/PaddleFleetX_GPT_345M_220826 \
|
||||
-o Offline_Eval.eval_path=./lambada_test.jsonl \
|
||||
-o Offline_Eval.cloze_eval=True \
|
||||
-o Offline_Eval.batch_size=16 \
|
||||
-o Engine.max_steps=20 \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
check_result $FUNCNAME
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
|
||||
function llm_gpt_recompute_bs32_bf16_MP2-SD4-stage1() {
|
||||
echo "=========== $FUNCNAME run begin ==========="
|
||||
export FLAGS_cudnn_deterministic=1
|
||||
export FLAGS_embedding_deterministic=1
|
||||
export PYTHONPATH=$root_path/:$PYTHONPATH
|
||||
log_dir=mylog
|
||||
rm -rf $log_dir
|
||||
python -m paddle.distributed.launch --log_dir=./mylog --devices=0,1,2,3,4,5,6,7 run_pretrain.py \
|
||||
--model_name_or_path gpt2-medium-en \
|
||||
--tokenizer_name_or_path gpt2-medium-en \
|
||||
--input_dir ./data \
|
||||
--output_dir output \
|
||||
--sharding stage1 \
|
||||
--sharding_parallel_degree 4 \
|
||||
--tensor_parallel_degree 2 \
|
||||
--split 949,50,1 \
|
||||
--max_seq_length 1024 \
|
||||
--seed 1234 \
|
||||
--fuse_attention_qkv True \
|
||||
--use_flash_attention False \
|
||||
--bf16 False \
|
||||
--fp16 True \
|
||||
--fp16_opt_level O2 \
|
||||
--amp_master_grad True \
|
||||
--learning_rate 0.00001 \
|
||||
--min_learning_rate 0.000005 \
|
||||
--max_grad_norm 1.0 \
|
||||
--logging_steps 1 \
|
||||
--continue_training 0 \
|
||||
--dataloader_num_workers 1 \
|
||||
--eval_steps 1000 \
|
||||
--disable_tqdm True \
|
||||
--gradient_accumulation_steps 2 \
|
||||
--weight_decay 0.01 \
|
||||
--max_steps 30 \
|
||||
--save_steps 5000 \
|
||||
--device gpu \
|
||||
--skip_memory_metrics 0 \
|
||||
--warmup_ratio 0.01 \
|
||||
--scale_loss 32768 \
|
||||
--per_device_train_batch_size 4 \
|
||||
--do_train \
|
||||
--recompute True \
|
||||
>>${log_path}/$FUNCNAME 2>&1
|
||||
loss=`cat $log_dir/workerlog.0 | grep 'global_step: 30' | awk -F 'loss: ' '{print $2}' | awk -F ',' '{print $1}'`
|
||||
ips=`cat $log_dir/workerlog.0 | grep 'global_step: 30' | awk -F 'interval_samples_per_second: ' '{print $2}' | awk -F ',' '{print $1}'`
|
||||
mem=`cat $log_dir/workerlog.0 | grep 'global_step: 30' | awk -F 'gpu_max_memory_reserved: ' '{print $2}' | awk -F ',' '{print $1}'`
|
||||
echo "result: loss=$loss ips=$ips mem=$mem"
|
||||
if [ $IS_CUDA123 -ne 0 ];then
|
||||
loss_base=8.93676758
|
||||
else
|
||||
loss_base=8.93362999
|
||||
fi
|
||||
ips_base=64.75564390065037
|
||||
mem_base=8904
|
||||
check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
|
||||
echo "=========== $FUNCNAME run end ==========="
|
||||
}
|
||||
############ case end ############
|
||||
|
||||
function check_result() {
|
||||
echo -e "$1" >> ${log_path}/result.log
|
||||
if [ $? -ne 0 ];then
|
||||
echo -e "\033[31m $1 run failed! \033[0m" | tee -a ${log_path}/result.log
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ ! $1 =~ "llm" ]]; then
|
||||
echo -e "\033 $1 run successfully! \033" | tee -a ${log_path}/result.log
|
||||
elif [ $# -ne 7 ]; then
|
||||
echo -e "\033[31m $1 parameter transfer failed: $@ \033[0m" | tee -a ${log_path}/result.log
|
||||
return 0
|
||||
else
|
||||
diff_loss=$(echo $2 $3|awk '{printf "%0.2f\n", ($2-$1)/$1*100}')
|
||||
echo -e "loss_base: $2 loss_test: $3 loss_diff: $diff_loss%" | tee -a ${log_path}/result.log
|
||||
if [ $2 != $3 ];then
|
||||
echo -e "\033[31m $1 loss diff check failed! \033[0m" | tee -a ${log_path}/result.log
|
||||
exit 2
|
||||
fi
|
||||
|
||||
diff_ips=$(echo $4 $5|awk '{printf "%0.2f\n", ($2-$1)/$1*100}')
|
||||
echo -e "ips_base: $4 ips_test: $5 ips_diff: $diff_ips% " | tee -a $log_path/result.log
|
||||
v1=$(echo $diff_ips 5.0|awk '{print($1>=$2)?"0":"1"}')
|
||||
v2=$(echo $diff_ips -5.0|awk '{print($1<=$2)?"0":"1"}')
|
||||
if [[ $v1 == 0 ]];then
|
||||
echo -e "$1 IPS increase greater than 5%, not exit " | tee -a $log_path/result.log
|
||||
fi
|
||||
if [[ $v2 == 0 ]];then
|
||||
echo -e "\033[31m $1 IPS diff check failed! \033[0m" | tee -a $log_path/result.log
|
||||
exit 2
|
||||
fi
|
||||
|
||||
diff_mem=$(echo $6 $7|awk '{printf "%0.2f\n", ($2-$1)/$1*100}')
|
||||
echo -e "mem_base: $6 mem_test: $7 mem_diff: $diff_mem% " | tee -a $log_path/result.log
|
||||
w1=$(echo $diff_mem 5.0|awk '{print($1>=$2)?"0":"1"}')
|
||||
w2=$(echo $diff_mem -5.0|awk '{print($1<=$2)?"0":"1"}')
|
||||
if [[ $w1 == 0 ]];then
|
||||
echo -e "\033[31m $1 MEM diff check failed! \033[0m" | tee -a $log_path/result.log
|
||||
exit 2
|
||||
fi
|
||||
if [[ $w2 == 0 ]];then
|
||||
echo -e "$1 MEM decreases greater than 5%, not exit " | tee -a $log_path/result.log
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function before_hook_for_gpt() {
|
||||
echo -e "\033[31m ---- Set FLAGS for GPT dygraph cases \033[0m"
|
||||
cd ${gpt_case_path}
|
||||
env | grep FLAGS
|
||||
export http_proxy=${proxy}
|
||||
export https_proxy=${proxy}
|
||||
export no_proxy=bcebos.com
|
||||
if [[ $FLAGS_install_deps == 0 ]];then
|
||||
echo -e "\033[31m ---- Install requirements for GPT dygraph cases \033[0m"
|
||||
cp requirements.txt requirements_nlp.txt
|
||||
sed -i '/paddlenlp/d' requirements.txt
|
||||
python -m pip install -r requirements.txt --force-reinstall
|
||||
sed -i '/paddlenlp/!d' requirements_nlp.txt
|
||||
python -m pip install -r requirements_nlp.txt
|
||||
python -m pip install -r $root_path/requirements.txt
|
||||
python -m pip install -r $root_path/requirements-dev.txt
|
||||
python -m pip install --no-cache-dir https://paddlenlp.bj.bcebos.com/wheels/paddlenlp-ci-py3-none-any.whl --force-reinstall --no-dependencies
|
||||
python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)";
|
||||
else
|
||||
echo -e "\033[31m ---- Skip install requirements for GPT dygraph cases \033[0m"
|
||||
fi
|
||||
echo -e "\033[31m ---- Install ppfleetx/ops \033[0m"
|
||||
cd ppfleetx/ops && python setup_cuda.py install && cd ../..
|
||||
|
||||
unset http_proxy && unset https_proxy
|
||||
echo -e "\033[31m ---- download data for GPT dygraph cases \033[0m"
|
||||
rm -rf data
|
||||
if [[ -e ${gpt_data_path}/data ]]; then
|
||||
echo "data downloaded"
|
||||
else
|
||||
# download data for gpt
|
||||
mkdir ${gpt_data_path}/data;
|
||||
wget -q -O ${gpt_data_path}/data/gpt_en_dataset_300m_ids.npy https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_ids.npy;
|
||||
wget -q -O ${gpt_data_path}/data/gpt_en_dataset_300m_idx.npz https://bj.bcebos.com/paddlenlp/models/transformers/gpt/data/gpt_en_dataset_300m_idx.npz;
|
||||
fi
|
||||
cp -r ${gpt_data_path}/data ${gpt_case_path}/
|
||||
|
||||
echo -e "\033[31m ---- download other data \033[0m"
|
||||
rm -rf ckpt
|
||||
if [[ -e ${gpt_data_path}/ckpt/PaddleFleetX_GPT_345M_220826 ]]; then
|
||||
echo "ckpt/PaddleFleetX_GPT_345M_220826 downloaded"
|
||||
else
|
||||
# download ckpt for gpt
|
||||
mkdir -p ${gpt_data_path}/ckpt
|
||||
wget -q -O ${gpt_data_path}/ckpt/GPT_345M.tar.gz \
|
||||
https://paddlefleetx.bj.bcebos.com/model/nlp/gpt/GPT_345M.tar.gz
|
||||
tar -xzf ${gpt_data_path}/ckpt/GPT_345M.tar.gz -C ${gpt_data_path}/ckpt
|
||||
rm -rf ${gpt_data_path}/ckpt/GPT_345M.tar.gz
|
||||
fi
|
||||
|
||||
rm -rf dataset
|
||||
if [[ -e ${gpt_data_path}/dataset/wikitext_103_en ]]; then
|
||||
echo "dataset/wikitext_103_en downloaded"
|
||||
else
|
||||
# download dataset/wikitext_103_en
|
||||
mkdir -p ${gpt_data_path}/dataset/wikitext_103_en;
|
||||
wget -q -O ${gpt_data_path}/dataset/wikitext_103_en/wikitext-103-en.txt http://fleet.bj.bcebos.com/datasets/gpt/wikitext-103-en.txt
|
||||
fi
|
||||
|
||||
rm -rf wikitext-103
|
||||
if [[ -e ${gpt_data_path}/wikitext-103 ]]; then
|
||||
echo "wikitext-103 downloaded"
|
||||
else
|
||||
# download wikitext-103 for gpt eval
|
||||
wget -q -O ${gpt_data_path}/wikitext-103.zip https://paddlefleetx.bj.bcebos.com/data/wikitext-103.zip
|
||||
unzip -q ${gpt_data_path}/wikitext-103.zip -d ${gpt_data_path}/
|
||||
rm -rf ${gpt_data_path}/wikitext-103.zip
|
||||
fi
|
||||
|
||||
rm -rf lambada_test.jsonl
|
||||
if [[ -e ${gpt_data_path}/lambada_test.jsonl ]]; then
|
||||
echo "lambada_test.jsonl downloaded"
|
||||
else
|
||||
# download lambada_test.jsonl for gpt eval
|
||||
wget -q -O ${gpt_data_path}/lambada_test.jsonl https://raw.githubusercontent.com/cybertronai/bflm/master/lambada_test.jsonl
|
||||
fi
|
||||
|
||||
rm -rf pretrained
|
||||
if [[ -e ${gpt_data_path}/pretrained ]]; then
|
||||
echo "GPT_345M_FP16 downloaded"
|
||||
else
|
||||
# download GPT_345M_FP16 for gpt export
|
||||
wget -q -O ${gpt_data_path}/GPT_345M_FP16.tar.gz https://paddlefleetx.bj.bcebos.com/model/nlp/gpt/GPT_345M_FP16.tar.gz
|
||||
tar -zxvf ${gpt_data_path}/GPT_345M_FP16.tar.gz -C ${gpt_data_path}/
|
||||
rm -rf ${gpt_data_path}/GPT_345M_FP16.tar.gz
|
||||
fi
|
||||
|
||||
rm -rf GPT_345M_QAT_wo_analysis
|
||||
if [[ -e ${gpt_data_path}/GPT_345M_QAT_wo_analysis ]]; then
|
||||
echo "GPT_345M_QAT_wo_analysis downloaded"
|
||||
else
|
||||
# download GPT_345M_QAT_wo_analysis for gpt qat
|
||||
wget -q -O ${gpt_data_path}/GPT_345M_QAT_wo_analysis.tar https://paddlefleetx.bj.bcebos.com/model/nlp/gpt/GPT_345M_QAT_wo_analysis.tar
|
||||
tar xf ${gpt_data_path}/GPT_345M_QAT_wo_analysis.tar -C ${gpt_data_path}/
|
||||
rm -rf ${gpt_data_path}/GPT_345M_QAT_wo_analysis.tar
|
||||
fi
|
||||
|
||||
ln -s ${gpt_data_path}/ckpt ${gpt_case_path}/ckpt
|
||||
cp -r ${gpt_data_path}/dataset ${gpt_case_path}/
|
||||
ln -s ${gpt_data_path}/wikitext-103 ${gpt_case_path}/wikitext-103
|
||||
cp ${gpt_data_path}/lambada_test.jsonl ${gpt_case_path}/
|
||||
ln -s ${gpt_data_path}/pretrained ${gpt_case_path}/pretrained
|
||||
ln -s ${gpt_data_path}/GPT_345M_QAT_wo_analysis ${gpt_case_path}/GPT_345M_QAT_wo_analysis
|
||||
}
|
||||
|
||||
function before_hook_for_llm_gpt() {
|
||||
echo -e "\033[31m ---- Set FLAGS for llm GPT cases \033[0m"
|
||||
cd ${llm_gpt_case_path}
|
||||
export FLAGS_cudnn_deterministic=1
|
||||
export FLAGS_embedding_deterministic=1
|
||||
env | grep FLAGS
|
||||
export http_proxy=${proxy}
|
||||
export https_proxy=${proxy}
|
||||
export no_proxy=bcebos.com
|
||||
python -m pip install -r $root_path/requirements.txt
|
||||
python -m pip install -r $root_path/requirements-dev.txt
|
||||
unset http_proxy && unset https_proxy
|
||||
if [[ ! $FLAGS_download_data =~ "llm_gpt" ]];then
|
||||
echo -e "\033[31m ---- Download llm GPT data \033[0m"
|
||||
rm -rf data
|
||||
if [[ -e ${llm_gpt_data_path}/data ]]; then
|
||||
echo "llm GPT data downloaded"
|
||||
else
|
||||
# download data for llm GPT
|
||||
mkdir ${llm_gpt_data_path}/data;
|
||||
wget -q -O ${llm_gpt_data_path}/data/gpt2-en-mmap.bin https://paddlenlp.bj.bcebos.com/datasets/PDC_DATASETS/PRETRAIN/openwebtext2/gpt/mmap/gpt2-en-mmap.bin
|
||||
wget -q -O ${llm_gpt_data_path}/data/gpt2-en-mmap.idx https://paddlenlp.bj.bcebos.com/datasets/PDC_DATASETS/PRETRAIN/openwebtext2/gpt/mmap/gpt2-en-mmap.idx
|
||||
fi
|
||||
cp -r ${llm_gpt_data_path}/data ${llm_gpt_case_path}/
|
||||
else
|
||||
echo -e "\033[31m ---- Skip download llm GPT data \033[0m"
|
||||
fi
|
||||
}
|
||||
|
||||
export status=$1
|
||||
|
||||
if [[ $status = "prepare_case" ]];then
|
||||
export FLAGS_install_deps=$3
|
||||
export FLAGS_download_data=$4
|
||||
if [[ $2 = "gpt_case_list_dygraph" ]];then
|
||||
before_hook_for_gpt
|
||||
gpt_case_list_dygraph prepare_case
|
||||
elif [[ $2 = "llm_gpt_case_list_dygraph" ]];then
|
||||
before_hook_for_llm_gpt
|
||||
llm_gpt_case_list_dygraph prepare_case
|
||||
else
|
||||
echo -e "\033[31m ---- Invalid exec_case $2 \033[0m"
|
||||
fi
|
||||
elif [[ $status = "exec_case" ]];then
|
||||
export FLAGS_install_deps=$3
|
||||
export FLAGS_download_data=$4
|
||||
if [[ $2 =~ "llm_gpt" ]];then
|
||||
cd ${llm_gpt_case_path}
|
||||
elif [[ $2 =~ "gpt" ]];then
|
||||
cd ${gpt_case_path}
|
||||
fi
|
||||
$2
|
||||
else
|
||||
echo -e "\033[31m ---- Start executing $1 \033[0m"
|
||||
export exec_case=$1
|
||||
export FLAGS_install_deps=$2
|
||||
export FLAGS_download_data=$3
|
||||
|
||||
if [[ $exec_case =~ "llm_gpt" ]];then
|
||||
cd ${llm_gpt_case_path}
|
||||
before_hook_for_llm_gpt
|
||||
elif [[ $exec_case =~ "gpt" ]];then
|
||||
cd ${gpt_case_path}
|
||||
before_hook_for_gpt
|
||||
else
|
||||
echo -e "\033[31m ---- Invalid exec_case $exec_case \033[0m"
|
||||
fi
|
||||
|
||||
$1 exec_case
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
####################################
|
||||
export paddle=$1
|
||||
export nlp_dir=/workspace/PaddleNLP
|
||||
mkdir -p /workspace/case_logs
|
||||
export log_path=/workspace/case_logs
|
||||
export case_list=()
|
||||
|
||||
global_total_count=0
|
||||
global_success_count=0
|
||||
global_exit_250_arr=()
|
||||
global_runtime_fail_arr=()
|
||||
global_verification_fail_arr=()
|
||||
|
||||
target_lists_for_gpt=(
|
||||
"slm/model_zoo/gpt-3"
|
||||
"llm/auto_parallel/gpt-3"
|
||||
"paddlenlp/trainer/training_args.py"
|
||||
"paddlenlp/transformers/gpt"
|
||||
"scripts/distribute"
|
||||
".github/workflows/distribute.yml"
|
||||
)
|
||||
|
||||
target_lists_for_llama=(
|
||||
"llm/auto_parallel/llama"
|
||||
"paddlenlp/trainer/auto_trainer.py"
|
||||
"paddlenlp/trainer/training_args.py"
|
||||
"paddlenlp/transformers/llama"
|
||||
"scripts/distribute"
|
||||
".github/workflows/distribute.yml"
|
||||
)
|
||||
|
||||
target_lists_for_deepseek=(
|
||||
"llm/auto_parallel/deepseek-v3"
|
||||
"paddlenlp/trainer/auto_trainer.py"
|
||||
"paddlenlp/trainer/training_args.py"
|
||||
"paddlenlp/transformers/deepseek_v2/modeling_auto.py"
|
||||
"paddlenlp/transformers/deepseek_v2/modeling.py"
|
||||
"paddlenlp/transformers/deepseek_v3/modeling_auto.py"
|
||||
"paddlenlp/transformers/moe_layer_auto.py"
|
||||
"paddlenlp/transformers/moe_gate_auto.py"
|
||||
"scripts/distribute"
|
||||
".github/workflows/distribute.yml"
|
||||
)
|
||||
|
||||
target_path_for_ci_scripts="scripts/distribute"
|
||||
|
||||
####################################
|
||||
install_paddle(){
|
||||
echo -e "\033[31m ---- Install paddlepaddle-gpu \033"
|
||||
python -m pip install --no-cache-dir --user ${paddle} --force-reinstall --no-dependencies;
|
||||
python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
|
||||
}
|
||||
|
||||
install_paddlenlp(){
|
||||
echo -e "\033[31m ---- Install paddlenlp by set PYTHONPATH \033"
|
||||
export PYTHONPATH=${nlp_dir}:$PYTHONPATH
|
||||
# python -m pip install -r ${nlp_dir}/requirements.txt
|
||||
# export http_proxy=${proxy} && export https_proxy=${proxy}
|
||||
# python -m pip uninstall paddlenlp -y
|
||||
# rm -rf build/ && rm -rf paddlenlp.egg-info/ && rm -rf dist/
|
||||
# python -m pip install --ignore-installed -r requirements.txt
|
||||
# python -m pip install --ignore-installed -r requirements-dev.txt
|
||||
# python setup.py install
|
||||
# python setup.py build_ext
|
||||
# python setup.py bdist_wheel
|
||||
# unset http_proxy && unset https_proxy
|
||||
# cd -
|
||||
# python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)";
|
||||
}
|
||||
|
||||
install_external_ops(){
|
||||
echo -e "\033[31m ---- Install extern_ops \033"
|
||||
export PYTHONPATH=${nlp_dir}:$PYTHONPATH
|
||||
cd ${nlp_dir}/slm/model_zoo/gpt-3/external_ops
|
||||
python setup.py install
|
||||
python -c "import fused_ln;";
|
||||
}
|
||||
|
||||
function is_a100() {
|
||||
if [ $(nvidia-smi|grep A100|wc -l) -ne 0 ];then
|
||||
echo 1 # A100
|
||||
else
|
||||
echo 0 # not A100
|
||||
fi
|
||||
}
|
||||
|
||||
IS_A100=$(is_a100)
|
||||
|
||||
####################################
|
||||
get_diff_TO_case(){
|
||||
if [ -z "${AGILE_COMPILE_BRANCH}" ]; then
|
||||
# 定时任务回归测试
|
||||
case_list=("gpt-3_auto" "gpt-3_dygraph" "llama_auto" "deepseek_auto")
|
||||
else
|
||||
cd ${nlp_dir}
|
||||
if [ $IS_A100 -ne 0 ];then
|
||||
for file_name in `git diff --numstat ${AGILE_COMPILE_BRANCH} -- |awk '{print $NF}'`;do
|
||||
arr_file_name=(${file_name//// })
|
||||
dir1=${arr_file_name[0]}
|
||||
dir2=${arr_file_name[1]}
|
||||
dir3=${arr_file_name[2]}
|
||||
dir4=${arr_file_name[3]}
|
||||
file_item=$dir1/$dir2/$dir3/$dir4
|
||||
echo "file_name:"${file_name}, "path:"${file_item}
|
||||
if [ ! -f ${file_name} ];then # 针对pr删掉文件
|
||||
continue
|
||||
elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then
|
||||
continue
|
||||
else
|
||||
for ((i=0; i<${#target_lists_for_gpt[@]}; i++)); do
|
||||
if [[ ! ${dir3} =~ "benchmarks" ]] && [[ ${file_item} == *${target_lists_for_gpt[i]}* ]];then
|
||||
case_list[${#case_list[*]}]=gpt-3_auto
|
||||
case_list[${#case_list[*]}]=gpt-3_dygraph
|
||||
fi
|
||||
done
|
||||
for ((i=0; i<${#target_lists_for_llama[@]}; i++)); do
|
||||
if [[ ${file_item} == *${target_lists_for_llama[i]}* ]];then
|
||||
case_list[${#case_list[*]}]=llama_auto
|
||||
fi
|
||||
done
|
||||
for ((i=0; i<${#target_lists_for_deepseek[@]}; i++)); do
|
||||
if [[ ${file_item} == *${target_lists_for_deepseek[i]}* ]];then
|
||||
case_list[${#case_list[*]}]=deepseek_auto
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
else
|
||||
for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
|
||||
arr_file_name=(${file_name//// })
|
||||
dir1=${arr_file_name[0]}
|
||||
dir2=${arr_file_name[1]}
|
||||
dir3=${arr_file_name[2]}
|
||||
dir4=${arr_file_name[3]}
|
||||
file_item=$dir1/$dir2/$dir3/$dir4
|
||||
echo "file_name:"${file_name}, "path:"${file_item}
|
||||
if [ ! -f ${file_name} ];then # 针对pr删掉文件
|
||||
continue
|
||||
elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then
|
||||
continue
|
||||
else
|
||||
case_list[${#case_list[*]}]=gpt-3_auto
|
||||
case_list[${#case_list[*]}]=llama_auto
|
||||
case_list[${#case_list[*]}]=deepseek_auto
|
||||
for ((i=0; i<${#target_lists_for_gpt[@]}; i++)); do
|
||||
if [[ ! ${dir3} =~ "benchmarks" ]] && [[ ${file_item} == *${target_lists_for_gpt[i]}* ]];then
|
||||
case_list[${#case_list[*]}]=gpt-3_dygraph
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
}
|
||||
####################################
|
||||
function contain_case(){
|
||||
local e
|
||||
for e in "${@:2}";do
|
||||
if [[ "$e" == "$1" ]];then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
####################################
|
||||
function execute_func_list(){
|
||||
cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
|
||||
total_count=0
|
||||
success_count=0
|
||||
runtime_fail_count=0
|
||||
verification_fail_count=0
|
||||
exit_250_count=0
|
||||
while IFS= read -r func_name; do
|
||||
let total_count++
|
||||
let global_total_count++
|
||||
execute_num=1
|
||||
while true; do
|
||||
timeout 10m bash $1 exec_case $func_name $FLAGS_install_deps $FLAGS_download_data
|
||||
result=$?
|
||||
if [ $result -eq 0 ]; then
|
||||
echo -e "\033[32m test success!"
|
||||
let success_count++
|
||||
let global_success_count++
|
||||
elif [ $result -eq 1 ]; then
|
||||
if [ $execute_num -eq 1 ]; then
|
||||
echo -e "\033[31m first time execute failed, try again!"
|
||||
let execute_num++
|
||||
continue
|
||||
else
|
||||
echo -e "\033[31m second time execute failed, exit!"
|
||||
mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
|
||||
tail -15 ${log_path}/${func_name}_FAIL.log
|
||||
let runtime_fail_count++
|
||||
global_runtime_fail_arr+=("$func_name")
|
||||
fi
|
||||
elif [ $result -eq 2 ]; then
|
||||
echo -e "\033[31m verification failed!"
|
||||
let verification_fail_count++
|
||||
global_verification_fail_arr+=("$func_name")
|
||||
elif [ $result -eq 250 ]; then
|
||||
if [ $execute_num -eq 1 ]; then
|
||||
echo -e "\033[31m first time execute failed, try again!"
|
||||
let execute_num++
|
||||
continue
|
||||
else
|
||||
echo -e "\033[31m second time execute failed, exit!"
|
||||
mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
|
||||
tail -15 ${log_path}/${func_name}_FAIL.log
|
||||
let exit_250_count++
|
||||
global_exit_250_arr+=("$func_name")
|
||||
fi
|
||||
elif [ $result -eq 124 ]; then
|
||||
echo "\033[31m [failed-timeout] Test case execution was terminated after exceeding the 10m limit."
|
||||
mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
|
||||
tail -15 ${log_path}/${func_name}_FAIL.log
|
||||
let runtime_fail_count++
|
||||
global_runtime_fail_arr+=("$func_name")
|
||||
else
|
||||
echo "test failed!"
|
||||
mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
|
||||
tail -15 ${log_path}/${func_name}_FAIL.log
|
||||
let runtime_fail_count++
|
||||
global_runtime_fail_arr+=("$func_name")
|
||||
fi
|
||||
break
|
||||
done
|
||||
done < functions.txt
|
||||
echo -e "\033[31m $2 test case has complicated \033"
|
||||
echo -e "\033[31m $(printf '\t') total tests : $total_count \033"
|
||||
echo -e "\033[31m $(printf '\t') success tests : $success_count \033"
|
||||
echo -e "\033[31m $(printf '\t') runtime fail tests : $runtime_fail_count \033"
|
||||
echo -e "\033[31m $(printf '\t') verification fail tests : $verification_fail_count \033"
|
||||
echo -e "\033[31m $(printf '\t') exit 250 tests(intermittent issue) : $exit_250_count \033"
|
||||
}
|
||||
|
||||
function clean_file(){
|
||||
target_path=$1
|
||||
matching_data_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*data*")
|
||||
if [ -n "$matching_data_dirs" ]; then
|
||||
echo "cleaning data dirs:"
|
||||
echo $matching_data_dirs
|
||||
for dir in $matching_data_dirs; do
|
||||
rm -rf "$dir"
|
||||
echo "deleted $dir"
|
||||
done
|
||||
else
|
||||
echo "$target_path no data dirs found"
|
||||
fi
|
||||
|
||||
matching_output_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*output*")
|
||||
if [ -n "$matching_output_dirs" ]; then
|
||||
echo "cleaning output dirs:"
|
||||
echo $matching_output_dirs
|
||||
for dir in $matching_output_dirs; do
|
||||
rm -rf "$dir"
|
||||
echo "deleted $dir"
|
||||
done
|
||||
else
|
||||
echo "$target_path no output dirs found"
|
||||
fi
|
||||
}
|
||||
|
||||
####################################
|
||||
get_diff_TO_case # 获取待执行case列表
|
||||
case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]})) # 去重并将结果存储回原列表
|
||||
if [[ ${#case_list[*]} -ne 0 ]];then
|
||||
echo -e "\033[31m =======CI Check case========= \033"
|
||||
echo -e "\033[31m ---- case_list length: ${#case_list[*]}, cases: ${case_list[*]} \033"
|
||||
echo -e "\033[31m ============================= \033"
|
||||
set +e
|
||||
|
||||
# Install paddle
|
||||
install_paddle
|
||||
# Install paddlenlp
|
||||
install_paddlenlp
|
||||
# Install external_ops
|
||||
install_external_ops
|
||||
case_num=1
|
||||
export FLAGS_install_deps=0
|
||||
export FLAGS_download_data=""
|
||||
if [[ $(contain_case llama_auto ${case_list[@]}; echo $?) -eq 1 ]];then
|
||||
echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: llama_auto \033"
|
||||
cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh
|
||||
bash $cmd prepare_case llama_case_list_auto $FLAGS_install_deps $FLAGS_download_data
|
||||
execute_func_list $cmd llama_auto
|
||||
export FLAGS_install_deps=1
|
||||
export FLAGS_download_data="llama ""$FLAGS_download_data"
|
||||
let case_num++
|
||||
clean_file $nlp_dir/llm/auto_parallel/llama
|
||||
fi
|
||||
if [[ $(contain_case gpt-3_auto ${case_list[@]}; echo $?) -eq 1 ]];then
|
||||
echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: gpt-3_auto \033"
|
||||
cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh
|
||||
bash $cmd prepare_case llm_gpt_case_list_auto $FLAGS_install_deps $FLAGS_download_data
|
||||
execute_func_list $cmd gpt-3_auto
|
||||
export FLAGS_install_deps=1
|
||||
export FLAGS_download_data="gpt ""$FLAGS_download_data"
|
||||
let case_num++
|
||||
clean_file $nlp_dir/llm/auto_parallel/gpt-3
|
||||
fi
|
||||
if [[ $(contain_case deepseek_auto ${case_list[@]}; echo $?) -eq 1 ]];then
|
||||
echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: deepseek_auto \033"
|
||||
cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh
|
||||
bash $cmd prepare_case deepseek_case_list_auto $FLAGS_install_deps $FLAGS_download_data
|
||||
execute_func_list $cmd deepseek_auto
|
||||
export FLAGS_install_deps=1
|
||||
export FLAGS_download_data="deepseek ""$FLAGS_download_data"
|
||||
let case_num++
|
||||
clean_file $nlp_dir/llm/auto_parallel/deepseek-v3
|
||||
fi
|
||||
|
||||
if [[ $(contain_case gpt-3_dygraph ${case_list[@]}; echo $?) -eq 1 ]];then
|
||||
echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: gpt-3_dygraph \033"
|
||||
cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_dy.sh
|
||||
bash $cmd prepare_case gpt_case_list_dygraph $FLAGS_install_deps $FLAGS_download_data
|
||||
execute_func_list $cmd gpt-3_dygraph
|
||||
export FLAGS_install_deps=1
|
||||
export FLAGS_download_data="gpt ""$FLAGS_download_data"
|
||||
let case_num++
|
||||
clean_file $nlp_dir/slm/model_zoo/gpt-3
|
||||
fi
|
||||
echo -e "\033[31m ---- end run case \033"
|
||||
|
||||
echo -e "\033[31m ---- total tests : $global_total_count \033"
|
||||
if [ ${#global_exit_250_arr[@]} -ne 0 ]; then
|
||||
echo -e "\033[32m ---- exit 250 test : ${#global_exit_250_arr[@]} \033"
|
||||
for case in "${global_exit_250_arr[@]}"; do
|
||||
echo -e "\t$case(exit 250)"
|
||||
done
|
||||
fi
|
||||
|
||||
if [ ${#global_runtime_fail_arr[@]} -eq 0 ] && [ ${#global_verification_fail_arr[@]} -eq 0 ]; then
|
||||
echo -e "\033[32m ---- all cases Success \033"
|
||||
EXCODE=0
|
||||
else
|
||||
echo -e "\033[32m ---- runtime failed test : ${#global_runtime_fail_arr[@]} \033"
|
||||
for case in "${global_runtime_fail_arr[@]}"; do
|
||||
echo -e "\t$case(failed)"
|
||||
done
|
||||
echo -e "\033[32m ---- verification failed test : ${#global_verification_fail_arr[@]} \033"
|
||||
for case in "${global_verification_fail_arr[@]}"; do
|
||||
echo -e "\t$case(failed)"
|
||||
done
|
||||
EXCODE=1
|
||||
fi
|
||||
else
|
||||
echo -e "\033[32m Changed Not CI case, Skips \033"
|
||||
EXCODE=0
|
||||
fi
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,28 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
fork_point_sha = subprocess.check_output("git merge-base develop HEAD".split()).decode("utf-8")
|
||||
modified_files = (
|
||||
subprocess.check_output(f"git diff --diff-filter=d --name-only {fork_point_sha}".split()).decode("utf-8").split()
|
||||
)
|
||||
|
||||
valid_dirs = "|".join(sys.argv[1:])
|
||||
regex = re.compile(rf"^({valid_dirs}).*?\.(py|md)$")
|
||||
|
||||
relevant_modified_files = [x for x in modified_files if regex.match(x)]
|
||||
print(" ".join(relevant_modified_files), end="")
|
||||
@@ -0,0 +1,11 @@
|
||||
## PaddleNLP CI
|
||||
|
||||
主要代码结构及说明：
|
||||
```
|
||||
./
|
||||
├── run_ci.sh # CI pr级别 执行入口
|
||||
├── run_release.sh # 天级别回归&发版 执行入口
|
||||
├── ci_case.sh # CI 核心case
|
||||
├── ci_normal_case.py # 规范模型case 执行脚本
|
||||
└── requirements_ci.txt # 依赖库
|
||||
```
|
||||
@@ -0,0 +1,667 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
export nlp_dir=${PWD}
|
||||
export log_path=${nlp_dir}/model_logs
|
||||
export cudaid1=$2
|
||||
export cudaid2=$3
|
||||
export C_COMPILER_PATH=$(which gcc)
|
||||
export CXX_COMPILER_PATH=$(which g++)
|
||||
export CC=$(which gcc)
|
||||
export CXX=$(which g++)
|
||||
|
||||
export PADDLE_INFERENCE_MODEL_SUFFIX=$(python -c "
|
||||
import paddle
|
||||
try:
|
||||
from paddle.base.framework import use_pir_api
|
||||
pir_enabled = use_pir_api()
|
||||
except ImportError:
|
||||
pir_enabled = False
|
||||
model_suffix = '.json' if pir_enabled else '.pdmodel'
|
||||
print(model_suffix)
|
||||
")
|
||||
|
||||
if [ ! -d "model_logs" ]; then
|
||||
mkdir model_logs
|
||||
fi
|
||||
|
||||
print_info() {
|
||||
if [ $1 -ne 0 ]; then
|
||||
cat ${log_path}/$2.log | grep -v "SKIPPED" | grep -v "PASSED" > ${log_path}/$2_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/$2_FAIL \033[0m"
|
||||
cat ${log_path}/$2_FAIL.log
|
||||
if [ -n "${AGILE_JOB_BUILD_ID}" ]; then
|
||||
cp ${log_path}/$2_FAIL.log ${PPNLP_HOME}/upload/$2_FAIL.log.${AGILE_PIPELINE_BUILD_ID}.${AGILE_JOB_BUILD_ID}
|
||||
cd ${PPNLP_HOME} && python upload.py ${PPNLP_HOME}/upload 'paddlenlp/PaddleNLP_CI/PaddleNLP_CI'
|
||||
rm -rf upload/* && cd -
|
||||
fi
|
||||
if [ $1 -eq 124 ]; then
|
||||
echo -e "\033[31m [failed-timeout] Test case execution was terminated after exceeding the time limit. \033[0m"
|
||||
fi
|
||||
else
|
||||
tail -n 1 ${log_path}/$2.log
|
||||
echo -e "\033[32m ${log_path}/$2_SUCCESS \033[0m"
|
||||
fi
|
||||
}
|
||||
# case list
|
||||
# 2 msra_ner （不可控，内置）
|
||||
msra_ner() {
|
||||
cd ${nlp_dir}/slm/examples/information_extraction/msra_ner/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
## train
|
||||
time (python -m paddle.distributed.launch ./train.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-multilingual-uncased \
|
||||
--dataset msra_ner \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 16 \
|
||||
--learning_rate 2e-5 \
|
||||
--num_train_epochs 1 \
|
||||
--logging_steps 1 \
|
||||
--max_steps 2 \
|
||||
--save_steps 2 \
|
||||
--output_dir ./tmp/msra_ner/ \
|
||||
--device gpu >${log_path}/msra_ner_train.log) >>${log_path}/msra_ner_train.log 2>&1
|
||||
print_info $? msra_ner_train
|
||||
## eval
|
||||
time (python -u ./eval.py \
|
||||
--model_name_or_path bert-base-multilingual-uncased \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 16 \
|
||||
--device gpu \
|
||||
--init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_eval.log) >>${log_path}/msra_ner_eval.log 2>&1
|
||||
print_info $? msra_ner_eval
|
||||
## predict
|
||||
time (python -u ./predict.py \
|
||||
--model_name_or_path bert-base-multilingual-uncased \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 16 \
|
||||
--device gpu \
|
||||
--init_checkpoint_path ./tmp/msra_ner/model_2.pdparams >${log_path}/msra_ner_predict.log) >>${log_path}/msra_ner_predict.log 2>&1
|
||||
print_info $? msra_ner_predict
|
||||
}
|
||||
# 3 glue
|
||||
glue() {
|
||||
cd ${nlp_dir}/slm/examples/benchmark/glue/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
## TASK_SST-2
|
||||
export TASK_NAME=SST-2
|
||||
time (python -u run_glue.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--task_name $TASK_NAME \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 128 \
|
||||
--learning_rate 3e-5 \
|
||||
--max_steps 1 \
|
||||
--logging_steps 1 \
|
||||
--save_steps 1 \
|
||||
--output_dir ./$TASK_NAME/ \
|
||||
--device gpu >${log_path}/glue_${TASK_NAME}_train.log) >>${log_path}/glue_${TASK_NAME}_train.log 2>&1
|
||||
print_info $? glue_${TASK_NAME}_train
|
||||
}
|
||||
# 4 bert
|
||||
bert() {
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
# cd ${nlp_dir}/slm/model_zoo/bert/
|
||||
# wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/bert.tar.gz
|
||||
# tar -xzvf bert.tar.gz
|
||||
python -c "import datasets;from datasets import load_dataset; train_dataset=load_dataset('glue', 'sst2', split='train')"
|
||||
cd ${nlp_dir}/slm/model_zoo/bert/data/
|
||||
wget -q https://bj.bcebos.com/paddlenlp/models/transformers/bert/data/training_data.hdf5
|
||||
cd ../
|
||||
# pretrain
|
||||
time (python -m paddle.distributed.launch run_pretrain.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--max_predictions_per_seq 20 \
|
||||
--batch_size 16 \
|
||||
--learning_rate 1e-4 \
|
||||
--weight_decay 1e-2 \
|
||||
--adam_epsilon 1e-6 \
|
||||
--warmup_steps 10000 \
|
||||
--input_dir data/ \
|
||||
--output_dir pretrained_models/ \
|
||||
--logging_steps 1 \
|
||||
--save_steps 1 \
|
||||
--max_steps 1 \
|
||||
--device gpu \
|
||||
--use_amp False >${log_path}/bert_pretrain.log) >>${log_path}/bert_pretrain.log 2>&1
|
||||
print_info $? bert_pretrain
|
||||
time (python -m paddle.distributed.launch run_glue_trainer.py \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--task_name SST2 \
|
||||
--max_seq_length 128 \
|
||||
--per_device_train_batch_size 32 \
|
||||
--per_device_eval_batch_size 32 \
|
||||
--learning_rate 2e-5 \
|
||||
--num_train_epochs 3 \
|
||||
--logging_steps 1 \
|
||||
--save_steps 1 \
|
||||
--max_steps 1 \
|
||||
--output_dir ./tmp/ \
|
||||
--device gpu \
|
||||
--fp16 False\
|
||||
--do_train \
|
||||
--do_eval >${log_path}/bert_fintune.log) >>${log_path}/bert_fintune.log 2>&1
|
||||
print_info $? bert_fintune
|
||||
time (python -u ./export_model.py \
|
||||
--model_type bert \
|
||||
--model_path bert-base-uncased \
|
||||
--output_path ./infer_model/model >${log_path}/bert_export.log) >>${log_path}/bert_export.log 2>&1
|
||||
print_info $? bert_export
|
||||
}
|
||||
# 5 skep (max save 不可控 内置)
|
||||
skep() {
|
||||
cd ${nlp_dir}/slm/examples/sentiment_analysis/skep/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
## train_sentence
|
||||
time (python -m paddle.distributed.launch train_sentence.py \
|
||||
--batch_size 16 \
|
||||
--epochs 1 \
|
||||
--model_name "skep_ernie_1.0_large_ch" \
|
||||
--device gpu --save_dir ./checkpoints >${log_path}/skep_train_sentence.log) >>${log_path}/skep_train_sentence.log 2>&1
|
||||
print_info $? skep_train_sentence
|
||||
## train_aspect
|
||||
time (python -m paddle.distributed.launch train_aspect.py \
|
||||
--batch_size 4 \
|
||||
--epochs 1 --device gpu \
|
||||
--save_dir ./aspect_checkpoints >${log_path}/skep_train_aspect.log) >>${log_path}/skep_train_aspect.log 2>&1
|
||||
print_info $? skep_train_aspect
|
||||
# # train_opinion
|
||||
time (python -m paddle.distributed.launch train_opinion.py \
|
||||
--batch_size 4 \
|
||||
--epochs 1 \
|
||||
--device gpu \
|
||||
--save_dir ./opinion_checkpoints >${log_path}/skep_train_opinion.log) >>${log_path}/skep_train_opinion.log 2>&1
|
||||
print_info $? skep_train_opinion
|
||||
# predict_sentence
|
||||
time (python predict_sentence.py \
|
||||
--model_name "skep_ernie_1.0_large_ch" \
|
||||
--ckpt_dir checkpoints/model_100 >${log_path}/skep_predict_sentence.log) >>${log_path}/skep_predict_sentence.log 2>&1
|
||||
print_info $? skep_predict_sentence
|
||||
## predict_aspect
|
||||
time (python predict_aspect.py \
|
||||
--device 'gpu' \
|
||||
--ckpt_dir ./aspect_checkpoints/model_100 >${log_path}/skep_predict_aspect.log) >>${log_path}/skep_predict_aspect.log 2>&1
|
||||
print_info $? skep_predict_aspect
|
||||
# # predict_opinion
|
||||
time (python predict_opinion.py \
|
||||
--device 'gpu' \
|
||||
--ckpt_dir ./opinion_checkpoints/model_100 >${log_path}/skep_predict_opinion.log) >>${log_path}/skep_predict_opinion.log 2>&1
|
||||
print_info $? skep_predict_opinion
|
||||
}
|
||||
# 6 bigbird
|
||||
bigbird(){
|
||||
cd ${nlp_dir}/slm/model_zoo/bigbird/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
time (python -m paddle.distributed.launch --log_dir log run_pretrain.py \
|
||||
--model_name_or_path bigbird-base-uncased \
|
||||
--input_dir "./data" \
|
||||
--output_dir "output" \
|
||||
--batch_size 4 \
|
||||
--weight_decay 0.01 \
|
||||
--learning_rate 1e-5 \
|
||||
--max_steps 1 \
|
||||
--save_steps 1 \
|
||||
--logging_steps 1 \
|
||||
--max_encoder_length 512 \
|
||||
--max_pred_length 75 >${log_path}/bigbird_pretrain.log) >>${log_path}/bigbird_pretrain.log 2>&1
|
||||
print_info $? bigbird_pretrain
|
||||
}
|
||||
# 9 ernie
|
||||
ernie(){
|
||||
#data process
|
||||
cd ${nlp_dir}/slm/model_zoo/ernie-1.0/
|
||||
|
||||
if [ -d "data_ernie_3.0" ];then
|
||||
rm -rf data_ernie_3.0
|
||||
fi
|
||||
|
||||
mkdir data_ernie_3.0
|
||||
cd data_ernie_3.0
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_ids.npy
|
||||
wget https://bj.bcebos.com/paddlenlp/models/transformers/data_tools/wudao_200g_sample_ernie-3.0-base-zh_idx.npz
|
||||
cd ../
|
||||
|
||||
# pretrain_trainer
|
||||
time (python -u -m paddle.distributed.launch \
|
||||
--log_dir "output/trainer_log" \
|
||||
run_pretrain_trainer.py \
|
||||
--model_type "ernie" \
|
||||
--model_name_or_path "ernie-3.0-base-zh" \
|
||||
--tokenizer_name_or_path "ernie-3.0-base-zh" \
|
||||
--input_dir "./data_ernie_3.0" \
|
||||
--output_dir "output/trainer_log" \
|
||||
--split 949,50,1 \
|
||||
--max_seq_length 512 \
|
||||
--per_device_train_batch_size 16 \
|
||||
--per_device_eval_batch_size 32 \
|
||||
--fp16 \
|
||||
--fp16_opt_level "O2" \
|
||||
--learning_rate 0.0001 \
|
||||
--min_learning_rate 0.00001 \
|
||||
--max_steps 2 \
|
||||
--save_steps 2 \
|
||||
--weight_decay 0.01 \
|
||||
--warmup_ratio 0.01 \
|
||||
--max_grad_norm 1.0 \
|
||||
--logging_steps 1\
|
||||
--dataloader_num_workers 4 \
|
||||
--eval_steps 1000 \
|
||||
--report_to "visualdl" \
|
||||
--disable_tqdm true \
|
||||
--do_train \
|
||||
--device "gpu" >${log_path}/ernie_1.0_pretrain_trainer.log) >>${log_path}/ernie_1.0_pretrain_trainer.log 2>&1
|
||||
print_info $? ernie_1.0_pretrain_trainer
|
||||
}
|
||||
# 11 ofa
|
||||
ofa(){
|
||||
cd ${nlp_dir}/slm/examples/model_compression/ofa/
|
||||
cd ../../benchmark/glue/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
# finetuing
|
||||
time (python -u run_glue.py \
|
||||
--model_type bert \
|
||||
--model_name_or_path bert-base-uncased \
|
||||
--task_name SST-2 \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 32 \
|
||||
--learning_rate 2e-5 \
|
||||
--num_train_epochs 1 \
|
||||
--max_steps 1 \
|
||||
--logging_steps 1 \
|
||||
--save_steps 1 \
|
||||
--output_dir ./ \
|
||||
--device gpu >${log_path}/ofa_pretrain.log) >>${log_path}/ofa_pretrain.log 2>&1
|
||||
print_info $? ofa_pretrain
|
||||
mv sst-2_ft_model_1.pdparams/ ${nlp_dir}/slm/examples/model_compression/ofa/
|
||||
cd -
|
||||
#model slim
|
||||
# export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
# time (python -m paddle.distributed.launch run_glue_ofa.py \
|
||||
# --model_type bert \
|
||||
# --model_name_or_path ./sst-2_ft_model_1.pdparams/ \
|
||||
# --task_name SST-2 --max_seq_length 128 \
|
||||
# --batch_size 32 \
|
||||
# --learning_rate 2e-5 \
|
||||
# --num_train_epochs 1 \
|
||||
# --max_steps 1 \
|
||||
# --logging_steps 1 \
|
||||
# --save_steps 1 \
|
||||
# --output_dir ./ofa/SST-2 \
|
||||
# --device gpu \
|
||||
# --width_mult_list 1.0 0.8333333333333334 0.6666666666666666 0.5 >${log_path}/ofa_slim) >>${log_path}/ofa_slim 2>&1
|
||||
# print_info $? ofa_slim
|
||||
}
|
||||
# 12 albert
|
||||
albert() {
|
||||
cd ${nlp_dir}/slm/examples/benchmark/glue/
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
time (python -m paddle.distributed.launch run_glue.py \
|
||||
--model_type albert \
|
||||
--model_name_or_path albert-base-v2 \
|
||||
--task_name SST-2 \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 32 \
|
||||
--learning_rate 1e-5 \
|
||||
--max_steps 1 \
|
||||
--warmup_steps 1256 \
|
||||
--logging_steps 1 \
|
||||
--save_steps 1 \
|
||||
--output_dir ./albert/SST-2/ \
|
||||
--device gpu >${log_path}/albert_sst-2_train.log) >>${log_path}/albert_sst-2_train.log 2>&1
|
||||
print_info $? albert_sst-2_train
|
||||
}
|
||||
# 13 squad
|
||||
# squad() {
|
||||
# cd ${nlp_dir}/slm/examples/machine_reading_comprehension/SQuAD/
|
||||
# export CUDA_VISIBLE_DEVICES=${cudaid1}
|
||||
# # finetune
|
||||
# time (python -m paddle.distributed.launch run_squad.py \
|
||||
# --model_type bert \
|
||||
# --model_name_or_path bert-base-uncased \
|
||||
# --max_seq_length 384 \
|
||||
# --batch_size 12 \
|
||||
# --learning_rate 3e-5 \
|
||||
# --num_train_epochs 1 \
|
||||
# --max_steps 1 \
|
||||
# --logging_steps 1 \
|
||||
# --save_steps 1 \
|
||||
# --warmup_proportion 0.1 \
|
||||
# --weight_decay 0.01 \
|
||||
# --output_dir ./tmp/squad/ \
|
||||
# --device gpu \
|
||||
# --do_train \
|
||||
# --do_predict >${log_path}/squad_train) >>${log_path}/squad_train 2>&1
|
||||
# print_info $? squad_train
|
||||
# # export model
|
||||
# time (python -u ./export_model.py \
|
||||
# --model_type bert \
|
||||
# --model_path ./tmp/squad/model_1/ \
|
||||
# --output_path ./infer_model/model >${log_path}/squad_export) >>${log_path}/squad_export 2>&1
|
||||
# print_info $? squad_export
|
||||
# predict
|
||||
# time (python -u deploy/python/predict.py \
|
||||
# --model_type bert \
|
||||
# --model_name_or_path ./infer_model/model \
|
||||
# --batch_size 2 \
|
||||
# --max_seq_length 384 >${log_path}/squad_predict) >>${log_path}/squad_predict 2>&1
|
||||
# print_info $? squad_predict
|
||||
# }
|
||||
# 15 lexical_analysis
|
||||
lexical_analysis(){
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
cd ${nlp_dir}/slm/examples/lexical_analysis/
|
||||
#train
|
||||
time (python download.py --data_dir ./ )
|
||||
time (python -m paddle.distributed.launch train.py \
|
||||
--data_dir ./lexical_analysis_dataset_tiny \
|
||||
--model_save_dir ./save_dir \
|
||||
--epochs 1 \
|
||||
--save_steps 15 \
|
||||
--logging_steps 1\
|
||||
--batch_size 32 \
|
||||
--device gpu >${log_path}/lexical_analysis_train.log) >>${log_path}/lexical_analysis_train.log 2>&1
|
||||
print_info $? lexical_analysis_train
|
||||
#export
|
||||
time (python export_model.py \
|
||||
--data_dir=./lexical_analysis_dataset_tiny \
|
||||
--params_path=./save_dir/model_15.pdparams \
|
||||
--output_path=./infer_model/static_graph_params >${log_path}/lexical_analysis_export.log) >>${log_path}/lexical_analysis_export.log 2>&1
|
||||
print_info $? lexical_analysis_export
|
||||
# predict
|
||||
time (python predict.py --data_dir ./lexical_analysis_dataset_tiny \
|
||||
--init_checkpoint ./save_dir/model_15.pdparams \
|
||||
--batch_size 32 \
|
||||
--device gpu >${log_path}/lexical_analysis_predict.log) >>${log_path}/lexical_analysis_predict.log 2>&1
|
||||
print_info $? lexical_analysis_predict
|
||||
# deploy
|
||||
time (python deploy/predict.py \
|
||||
--model_file=infer_model/static_graph_params${PADDLE_INFERENCE_MODEL_SUFFIX} \
|
||||
--params_file=infer_model/static_graph_params.pdiparams \
|
||||
--data_dir lexical_analysis_dataset_tiny >${log_path}/lexical_analysis_deploy.log) >>${log_path}/lexical_analysis_deploy.log 2>&1
|
||||
print_info $? lexical_analysis_deploy
|
||||
}
|
||||
|
||||
# 22 transformer
|
||||
transformer() {
|
||||
cd ${nlp_dir}/slm/examples/machine_translation/transformer/
|
||||
wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/WMT14.en-de.partial.tar.gz
|
||||
tar -xzvf WMT14.en-de.partial.tar.gz
|
||||
time (
|
||||
sed -i "s/save_step: 10000/save_step: 1/g" configs/transformer.base.yaml
|
||||
sed -i "s/print_step: 100/print_step: 1/g" configs/transformer.base.yaml
|
||||
sed -i "s/epoch: 30/epoch: 1/g" configs/transformer.base.yaml
|
||||
sed -i "s/max_iter: None/max_iter: 2/g" configs/transformer.base.yaml
|
||||
sed -i "s/batch_size: 4096/batch_size: 1000/g" configs/transformer.base.yaml
|
||||
|
||||
python train.py --config ./configs/transformer.base.yaml \
|
||||
--train_file ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.en ${PWD}/WMT14.en-de.partial/train.tok.clean.bpe.de \
|
||||
--dev_file ${PWD}/WMT14.en-de.partial/dev.tok.bpe.en ${PWD}/WMT14.en-de.partial/dev.tok.bpe.de \
|
||||
--vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
|
||||
--unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" >${log_path}/transformer_train.log
|
||||
) >>${log_path}/transformer_train.log 2>&1
|
||||
print_info $? transformer_train
|
||||
#predict
|
||||
time (
|
||||
sed -i 's#init_from_params: "./trained_models/step/"#init_from_params: "./trained_models/step_final/"#g' configs/transformer.base.yaml
|
||||
python predict.py --config ./configs/transformer.base.yaml \
|
||||
--test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
|
||||
--without_ft \
|
||||
--vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
|
||||
--unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" >${log_path}/transformer_predict.log
|
||||
) >>${log_path}/transformer_predict.log 2>&1
|
||||
print_info $? transformer_predict
|
||||
#export
|
||||
time (
|
||||
python export_model.py --config ./configs/transformer.base.yaml \
|
||||
--vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
|
||||
--bos_token "<s>" --eos_token "<e>" >${log_path}/transformer_export.log
|
||||
) >>${log_path}/transformer_export.log 2>&1
|
||||
print_info $? transformer_export
|
||||
#infer
|
||||
time (
|
||||
python ./deploy/python/inference.py --config ./configs/transformer.base.yaml \
|
||||
--profile \
|
||||
--test_file ${PWD}/WMT14.en-de.partial/test.tok.bpe.en ${PWD}/WMT14.en-de.partial/test.tok.bpe.de \
|
||||
--vocab_file ${PWD}/WMT14.en-de.partial/vocab_all.bpe.33708 \
|
||||
--unk_token "<unk>" --bos_token "<s>" --eos_token "<e>" >${log_path}/transformer_infer.log
|
||||
) >>${log_path}/transformer_infer.log 2>&1
|
||||
print_info $? transformer_infer
|
||||
|
||||
# fast_transformer
|
||||
}
|
||||
#28 question_matching
|
||||
question_matching() {
|
||||
cd ${nlp_dir}/slm/examples/text_matching/question_matching/
|
||||
wget -q https://paddle-qa.bj.bcebos.com/paddlenlp/data_v4.tar.gz
|
||||
tar -xvzf data_v4.tar.gz
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
#train
|
||||
time (
|
||||
python -u -m paddle.distributed.launch train.py \
|
||||
--train_set ./data_v4/train/ALL/train \
|
||||
--dev_set ./data_v4/train/ALL/dev \
|
||||
--device gpu \
|
||||
--eval_step 10 \
|
||||
--max_steps 10 \
|
||||
--save_dir ./checkpoints \
|
||||
--train_batch_size 32 \
|
||||
--learning_rate 2E-5 \
|
||||
--epochs 1 \
|
||||
--rdrop_coef 0.0 >${log_path}/question_matching_train.log) >>${log_path}/question_matching_train.log 2>&1
|
||||
print_info $? question_matching_train
|
||||
#predict
|
||||
time (
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid1}
|
||||
python -u \
|
||||
predict.py \
|
||||
--device gpu \
|
||||
--params_path "./checkpoints/model_10/model_state.pdparams" \
|
||||
--batch_size 128 \
|
||||
--input_file ./data_v4/test/public_test_A \
|
||||
--result_file 0.0_predict_public_result_test_A_re >${log_path}/question_matching_predict.log) >>${log_path}/question_matching_predict.log 2>&1
|
||||
print_info $? question_matching_predict
|
||||
}
|
||||
# 29 ernie-csc
|
||||
ernie-csc() {
|
||||
export CUDA_VISIBLE_DEVICES=${cudaid2}
|
||||
cd ${nlp_dir}/slm/examples/text_correction/ernie-csc
|
||||
#dowdnload data
|
||||
python download.py --data_dir ./extra_train_ds/ --url https://github.com/wdimmy/Automatic-Corpus-Generation/raw/master/corpus/train.sgml
|
||||
#trans xml txt
|
||||
python change_sgml_to_txt.py -i extra_train_ds/train.sgml -o extra_train_ds/train.txt
|
||||
#2卡训练
|
||||
python -m paddle.distributed.launch train.py --batch_size 32 --logging_steps 100 --epochs 1 --learning_rate 5e-5 --model_name_or_path ernie-1.0-base-zh --output_dir ./checkpoints/ --extra_train_ds_dir ./extra_train_ds/ >${log_path}/ernie-csc_train.log 2>&1
|
||||
print_info $? ernie-csc_train
|
||||
#predict
|
||||
sh run_sighan_predict.sh >${log_path}/ernie-csc_predict.log 2>&1
|
||||
print_info $? ernie-csc_predict
|
||||
#export model
|
||||
python export_model.py --params_path ./checkpoints/best_model.pdparams --output_path ./infer_model/static_graph_params >${log_path}/ernie-csc_export.log 2>&1
|
||||
print_info $? ernie-csc_export
|
||||
#python deploy
|
||||
python predict.py --model_file infer_model/static_graph_params${PADDLE_INFERENCE_MODEL_SUFFIX} --params_file infer_model/static_graph_params.pdiparams >${log_path}/ernie-csc_deploy.log 2>&1
|
||||
print_info $? ernie-csc_deploy
|
||||
}
|
||||
|
||||
clue() {
|
||||
cd ${nlp_dir}/slm/examples/benchmark/clue/classification
|
||||
time (python -u ./run_clue_classifier_trainer.py \
|
||||
--model_name_or_path ernie-3.0-base-zh \
|
||||
--dataset "clue afqmc" \
|
||||
--max_seq_length 128 \
|
||||
--per_device_train_batch_size 32 \
|
||||
--per_device_eval_batch_size 32 \
|
||||
--learning_rate 1e-5 \
|
||||
--num_train_epochs 3 \
|
||||
--logging_steps 1 \
|
||||
--seed 42 \
|
||||
--save_steps 3 \
|
||||
--warmup_ratio 0.1 \
|
||||
--weight_decay 0.01 \
|
||||
--adam_epsilon 1e-8 \
|
||||
--output_dir ./tmp \
|
||||
--device gpu \
|
||||
--do_train \
|
||||
--do_eval \
|
||||
--metric_for_best_model "eval_accuracy" \
|
||||
--load_best_model_at_end \
|
||||
--save_total_limit 1 \
|
||||
--max_steps 1 >${log_path}/clue-trainer_api.log) >>${log_path}/clue-trainer_api.log 2>&1
|
||||
print_info $? clue-tranier_api
|
||||
time (python -u run_clue_classifier.py \
|
||||
--model_name_or_path ernie-3.0-base-zh \
|
||||
--task_name afqmc \
|
||||
--max_seq_length 128 \
|
||||
--batch_size 16 \
|
||||
--learning_rate 3e-5 \
|
||||
--num_train_epochs 3 \
|
||||
--logging_steps 100 \
|
||||
--seed 42 \
|
||||
--save_steps 1 \
|
||||
--warmup_proportion 0.1 \
|
||||
--weight_decay 0.01 \
|
||||
--adam_epsilon 1e-8 \
|
||||
--output_dir ./output/afqmc \
|
||||
--device gpu \
|
||||
--max_steps 1 \
|
||||
--do_train >${log_path}/clue-class.log) >>${log_path}/clue-class.log 2>&1
|
||||
print_info $? clue-class
|
||||
# cd ${nlp_dir}/slm/examples/benchmark/clue/mrc
|
||||
# export CUDA_VISIBLE_DEVICES=${cudaid1}
|
||||
# python -m paddle.distributed.launch run_cmrc2018.py \
|
||||
# --model_name_or_path ernie-3.0-base-zh \
|
||||
# --batch_size 16 \
|
||||
# --learning_rate 3e-5 \
|
||||
# --max_seq_length 512 \
|
||||
# --num_train_epochs 2 \
|
||||
# --do_train \
|
||||
# --do_predict \
|
||||
# --warmup_proportion 0.1 \
|
||||
# --weight_decay 0.01 \
|
||||
# --gradient_accumulation_steps 2 \
|
||||
# --max_steps 1 \
|
||||
# --output_dir ./tmp >${log_path}/clue-mrc >>${log_path}/clue-mrc 2>&1
|
||||
# print_info $? clue-mrc
|
||||
}
|
||||
#33 taskflow
|
||||
taskflow (){
|
||||
cd ${nlp_dir}
|
||||
timeout 10m python -m pytest scripts/regression/test_taskflow.py >${log_path}/taskflow.log 2>&1
|
||||
print_info $? taskflow
|
||||
}
|
||||
ernie-3.0(){
|
||||
cd ${nlp_dir}/slm/model_zoo/ernie-3.0/
|
||||
#训练
|
||||
python run_seq_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset afqmc --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_seq_cls.log 2>&1
|
||||
print_info $? ernie-3.0_train_seq_cls
|
||||
python run_token_cls.py --model_name_or_path ernie-3.0-medium-zh --dataset msra_ner --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_token_cls.log 2>&1
|
||||
print_info $? ernie-3.0_train_token_cls
|
||||
python run_qa.py --model_name_or_path ernie-3.0-medium-zh --dataset cmrc2018 --output_dir ./best_models --export_model_dir best_models/ --do_train --do_eval --do_export --config=configs/default.yml --max_steps=2 --save_step=2 >${log_path}/ernie-3.0_train_qa.log 2>&1
|
||||
print_info $? ernie-3.0_train_qa
|
||||
# 预测
|
||||
python run_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_seq_cls.log 2>&1
|
||||
print_info $? ernie-3.0_predict_seq_cls
|
||||
python run_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_token_cls.log 2>&1
|
||||
print_info $? ernie-3.0_predict_token_cls
|
||||
python run_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models --do_predict --config=configs/default.yml >${log_path}/ernie-3.0_predict_qa.log 2>&1
|
||||
print_info $? ernie-3.0_predict_qa
|
||||
#压缩 skip for paddleslim api error https://github.com/PaddlePaddle/PaddleSlim/blob/9f3e9b2f0f9948b780900d1299f2c3fe47322deb/paddleslim/nas/ofa/layers.py#L1301C32-L1302
|
||||
# python compress_seq_cls.py --model_name_or_path best_models/afqmc/ --dataset afqmc --output_dir ./best_models/afqmc --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_seq_cls >>${log_path}/ernie-3.0_compress_seq_cls 2>&1
|
||||
# print_info $? ernie-3.0_compress_seq_cls
|
||||
# python compress_token_cls.py --model_name_or_path best_models/msra_ner/ --dataset msra_ner --output_dir ./best_models/msra_ner --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_token_cls >>${log_path}/ernie-3.0_compress_token_cls 2>&1
|
||||
# print_info $? ernie-3.0_compress_token_cls
|
||||
# python compress_qa.py --model_name_or_path best_models/cmrc2018/ --dataset cmrc2018 --output_dir ./best_models/cmrc2018 --config=configs/default.yml --max_steps 10 --eval_steps 5 --save_steps 5 --algo_list mse --batch_size_list 4 >${log_path}/ernie-3.0_compress_qa >>${log_path}/ernie-3.0_compress_qa 2>&1
|
||||
# print_info $? ernie-3.0_compress_qa
|
||||
}
|
||||
uie(){
|
||||
cd ${nlp_dir}/slm/model_zoo/uie/
|
||||
mkdir data && cd data && wget https://bj.bcebos.com/paddlenlp/datasets/uie/doccano_ext.json && cd ../
|
||||
python doccano.py --doccano_file ./data/doccano_ext.json --task_type ext --save_dir ./data --splits 0.8 0.2 0 --schema_lang ch >${log_path}/uie_doccano.log 2>&1
|
||||
print_info $? uie_doccano
|
||||
python -u -m paddle.distributed.launch finetune.py --device gpu --logging_steps 2 --save_steps 2 --eval_steps 2 --seed 42 \
|
||||
--model_name_or_path uie-base --output_dir ./checkpoint/model_best --train_path data/train.txt --dev_path data/dev.txt \
|
||||
--max_seq_length 512 --per_device_eval_batch_size 16 --per_device_train_batch_size 16 --num_train_epochs 100 --learning_rate 1e-5 \
|
||||
--do_train --do_eval --do_export --export_model_dir ./checkpoint/model_best --label_names start_positions end_positions \
|
||||
--overwrite_output_dir --disable_tqdm True --metric_for_best_model eval_f1 --load_best_model_at_end True \
|
||||
--save_total_limit 1 --max_steps 2 >${log_path}/uie_train.log 2>&1
|
||||
print_info $? uie_train
|
||||
python evaluate.py --model_path ./checkpoint/model_best --test_path ./data/dev.txt --batch_size 16 --max_seq_len 512 >${log_path}/uie_eval.log 2>&1
|
||||
print_info $? uie_eval
|
||||
}
|
||||
ernie-layout(){
|
||||
cd ${nlp_dir}/slm/model_zoo/ernie-layout/
|
||||
# train ner
|
||||
python -u run_ner.py --model_name_or_path ernie-layoutx-base-uncased --output_dir ./ernie-layoutx-base-uncased/models/funsd/ \
|
||||
--dataset_name funsd --do_train --do_eval --max_steps 2 --eval_steps 2 --save_steps 2 --save_total_limit 1 --seed 1000 --overwrite_output_dir \
|
||||
--load_best_model_at_end --pattern ner-bio --preprocessing_num_workers 4 --overwrite_cache false --doc_stride 128 --target_size 1000 \
|
||||
--per_device_train_batch_size 4 --per_device_eval_batch_size 4 --learning_rate 2e-5 --lr_scheduler_type constant --gradient_accumulation_steps 1 \
|
||||
--metric_for_best_model eval_f1 --greater_is_better true >${log_path}/ernie-layout_train.log 2>&1
|
||||
print_info $? ernie-layout_train
|
||||
# export ner
|
||||
python export_model.py --task_type ner --model_path ./ernie-layoutx-base-uncased/models/funsd/ --output_path ./ner_export >${log_path}/ernie-layout_export.log 2>&1
|
||||
print_info $? ernie-layout_export
|
||||
# deploy ner
|
||||
cd ${nlp_dir}/slm/model_zoo/ernie-layout/deploy/python
|
||||
wget https://bj.bcebos.com/paddlenlp/datasets/document_intelligence/images.zip && unzip images.zip
|
||||
python infer.py --model_path_prefix ../../ner_export/inference --task_type ner --lang "en" --batch_size 8 >${log_path}/ernie-layout_deploy.log 2>&1
|
||||
print_info $? ernie-layout_deploy
|
||||
}
|
||||
ernie-1.0(){
|
||||
ernie
|
||||
}
|
||||
|
||||
ernie_layout(){
|
||||
ernie-layout
|
||||
}
|
||||
|
||||
ernie_csc(){
|
||||
ernie-csc
|
||||
}
|
||||
|
||||
segment_parallel_utils(){
|
||||
cd ${nlp_dir}
|
||||
echo "test segment_parallel_utils, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
|
||||
if [[ ${cudaid1} != ${cudaid2} ]]; then
|
||||
time (python -m paddle.distributed.launch tests/transformers/test_segment_parallel_utils.py >${log_path}/segment_parallel_utils.log) >>${log_path}/segment_parallel_utils.log 2>&1
|
||||
print_info $? segment_parallel_utils
|
||||
else
|
||||
echo "only one gpu:${cudaid1} is set, skip test"
|
||||
fi
|
||||
|
||||
}
|
||||
ring_flash_attention(){
|
||||
cd ${nlp_dir}
|
||||
echo "test ring_flash_attention, cudaid1:${cudaid1}, cudaid2:${cudaid2}"
|
||||
if [[ ${cudaid1} != ${cudaid2} ]]; then
|
||||
time (python -m paddle.distributed.launch tests/transformers/test_ring_flash_attention.py >${log_path}/ring_flash_attention.log) >>${log_path}/ring_flash_attention.log 2>&1
|
||||
print_info $? ring_flash_attention
|
||||
else
|
||||
echo "only one gpu:${cudaid1} is set, skip test"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
llm(){
|
||||
export http_proxy=${proxy} && export https_proxy=${proxy}
|
||||
echo ' Testing all LLMs '
|
||||
cd ${nlp_dir}
|
||||
timeout 50m python -m pytest tests/llm/test_*.py -vv --timeout=300 --alluredir=result >${log_path}/llm.log 2>&1
|
||||
print_info $? llm
|
||||
}
|
||||
|
||||
$1
|
||||
@@ -0,0 +1,175 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
RUN PaddleNLP CI Case
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def get_mode_info(case_path):
|
||||
"""
|
||||
Return: model_info{path,exec_file_list}
|
||||
Examples:
|
||||
pegasus = {
|
||||
"path": "applications/text_summarization/pegasus/"
|
||||
"deploy_path": "deploy/paddle_inference/"
|
||||
"prepare": "run_prepare.py"
|
||||
"train_exec_file": "train.py"
|
||||
"eval_exec_file": None
|
||||
"predict_exec_file": predict.py
|
||||
“export_exec_file”: export_model.py
|
||||
"infer_exec_file": inference_pegasus.py
|
||||
}
|
||||
"""
|
||||
model_info = {
|
||||
"path": case_path,
|
||||
"deploy_path": None,
|
||||
"prepare_exec_file": None,
|
||||
"train_exec_file": [],
|
||||
"eval_exec_file": None,
|
||||
"predict_exec_file": None,
|
||||
"export_exec_file": None,
|
||||
"infer_exec_file": None,
|
||||
}
|
||||
for root, dirs, files in os.walk(case_path):
|
||||
infer_deploy_path = case_path + "/deploy/paddle_inference"
|
||||
python_deploy_path = case_path + "/deploy/python"
|
||||
|
||||
if files and root == case_path:
|
||||
for file in files:
|
||||
# TODO .sh file incompatible windows
|
||||
if file.split(".")[-1] == "py":
|
||||
if re.compile("prepare.py").findall(file):
|
||||
model_info["prepare_exec_file"] = file
|
||||
|
||||
elif re.compile("train.py").findall(file):
|
||||
model_info["train_exec_file"].append(file)
|
||||
|
||||
elif re.compile("finetune").findall(file):
|
||||
model_info["train_exec_file"].append(file)
|
||||
|
||||
elif re.compile("eval.py").findall(file):
|
||||
model_info["eval_exec_file"] = file
|
||||
|
||||
elif re.compile("predict.py").findall(file):
|
||||
model_info["predict_exec_file"] = file
|
||||
|
||||
elif re.compile("export_model.py").findall(file):
|
||||
model_info["export_exec_file"] = file
|
||||
|
||||
elif re.compile("run_").findall(file):
|
||||
model_info["train_exec_file"].append(file)
|
||||
else:
|
||||
continue
|
||||
elif files and root == infer_deploy_path:
|
||||
for file in files:
|
||||
if file.split(".")[-1] == "py":
|
||||
model_info["deploy_path"] = "deploy/paddle_inference"
|
||||
model_info["infer_exec_file"] = file
|
||||
elif files and root == python_deploy_path:
|
||||
for file in files:
|
||||
if file.split(".")[-1] == "py":
|
||||
model_info["deploy_path"] = "deploy/python"
|
||||
model_info["infer_exec_file"] = file
|
||||
|
||||
print("model_info", model_info)
|
||||
return model_info
|
||||
|
||||
|
||||
def save_log(exit_code, output, case_name, file_name):
|
||||
"""
|
||||
save model log
|
||||
"""
|
||||
root_path = "/workspace/PaddleNLP"
|
||||
# root_path = '/ssd1/paddlenlp/zhangjunjun/PaddleNLP'
|
||||
if exit_code == 0:
|
||||
log_file = root_path + "/model_logs/" + os.path.join(case_name + "_" + file_name + "_SUCCESS.log")
|
||||
print("{} SUCCESS".format(file_name))
|
||||
with open(log_file, "a") as flog:
|
||||
flog.write("%s" % (output))
|
||||
else:
|
||||
log_file = root_path + "/model_logs/" + os.path.join(case_name + "_" + file_name + "_FAIL.log")
|
||||
print("{} FAIL".format(file_name))
|
||||
with open(log_file, "a") as flog:
|
||||
flog.write("%s" % (output))
|
||||
|
||||
|
||||
def run_normal_case(case_path):
|
||||
"""
|
||||
Run new normal case
|
||||
params:
|
||||
case_path: model path based PaddleNLP from git diff
|
||||
"""
|
||||
case_name = case_path.split("/")[-1]
|
||||
model_info = get_mode_info(case_path)
|
||||
deploy_path = model_info["deploy_path"]
|
||||
prepare_exec_file = model_info["prepare_exec_file"]
|
||||
eval_exec_file = model_info["eval_exec_file"]
|
||||
predict_exec_file = model_info["predict_exec_file"]
|
||||
export_exec_file = model_info["export_exec_file"]
|
||||
infer_exec_file = model_info["infer_exec_file"]
|
||||
|
||||
os.chdir(case_path)
|
||||
|
||||
if prepare_exec_file:
|
||||
prepare_output = subprocess.getstatusoutput("python %s " % (prepare_exec_file))
|
||||
save_log(prepare_output[0], prepare_output[1], case_name, prepare_exec_file.split(".")[0])
|
||||
|
||||
if model_info["train_exec_file"]:
|
||||
for train_file in model_info["train_exec_file"]:
|
||||
train_output = subprocess.getstatusoutput(
|
||||
"python -m paddle.distributed.launch %s --device gpu --max_steps 2 \
|
||||
--save_steps 2 --output_dir ./output/"
|
||||
% (train_file)
|
||||
)
|
||||
save_log(train_output[0], train_output[1], case_name, train_file.split(".")[0])
|
||||
else:
|
||||
print("Train Skipped")
|
||||
|
||||
if eval_exec_file:
|
||||
eval_output = subprocess.getstatusoutput("python %s --init_checkpoint_dir ./output/" % (eval_exec_file))
|
||||
save_log(eval_output[0], eval_output[1], case_name, eval_exec_file.split(".")[0])
|
||||
else:
|
||||
print("Evaluation Skipped")
|
||||
if predict_exec_file:
|
||||
predict_output = subprocess.getstatusoutput("python %s --init_checkpoint_dir ./output/" % (predict_exec_file))
|
||||
save_log(predict_output[0], predict_output[1], case_name, predict_exec_file.split(".")[0])
|
||||
else:
|
||||
print("Predict Skipped")
|
||||
if export_exec_file:
|
||||
export_output = subprocess.getstatusoutput(
|
||||
"python %s --export_output_dir ./inference_model/" % (export_exec_file)
|
||||
)
|
||||
save_log(export_output[0], export_output[1], case_name, export_exec_file.split(".")[0])
|
||||
else:
|
||||
print("Export model Skipped")
|
||||
if infer_exec_file:
|
||||
infer_output = subprocess.getstatusoutput(
|
||||
"cd %s && python %s --inference_model_dir ../../inference_model/" % (deploy_path, infer_exec_file)
|
||||
)
|
||||
save_log(infer_output[0], infer_output[1], case_name, infer_exec_file.split(".")[0])
|
||||
else:
|
||||
print("python inference Skipped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# path ="applications/text_summarization/pegasus"
|
||||
path = sys.argv[1]
|
||||
if os.path.isdir(path):
|
||||
run_normal_case(path)
|
||||
else:
|
||||
print("not model file path, skepped ")
|
||||
@@ -0,0 +1,109 @@
|
||||
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
import wget
|
||||
|
||||
|
||||
def gen_allure_report():
|
||||
"""
|
||||
generate allure report
|
||||
"""
|
||||
# install allure
|
||||
exit_code, output = subprocess.getstatusoutput("allure --version")
|
||||
if exit_code == 0:
|
||||
print("allure version is:{}".format(output))
|
||||
allure_bin = "allure"
|
||||
else:
|
||||
if os.path.exists("allure-2.19.0.zip") is False:
|
||||
bin_src = "https://xly-devops.bj.bcebos.com/tools/allure-2.19.0.zip"
|
||||
bin_file = wget.download(bin_src)
|
||||
zip_file = zipfile.ZipFile(bin_file, "a")
|
||||
zip_file.extractall()
|
||||
allure_bin_f = "%s/allure-2.19.0/bin/allure" % (os.getcwd())
|
||||
st = os.stat(allure_bin_f)
|
||||
os.chmod(allure_bin_f, st.st_mode | stat.S_IEXEC)
|
||||
allure_bin = "%s/allure-2.19.0/bin/allure" % (os.getcwd())
|
||||
exit_code, output = subprocess.getstatusoutput("java -version")
|
||||
if exit_code == 0:
|
||||
print("java version is:{}".format(output))
|
||||
else: # install java
|
||||
if os.path.exists("java_linux.tar.gz") is False:
|
||||
java_src = "https://paddle-qa.bj.bcebos.com/java/java_linux.tar.gz"
|
||||
wget.download(java_src)
|
||||
tf = tarfile.open("java_linux.tar.gz")
|
||||
tf.extractall(os.getcwd())
|
||||
os.environ["JAVA_HOME"] = os.path.join(os.getcwd(), "jdk1.8.0_351")
|
||||
os.environ["JRE_HOME"] = os.path.join(os.getenv("JAVA_HOME"), "jre")
|
||||
os.environ["CLASSPATH"] = os.path.join(os.getenv("JAVA_HOME"), "lib")
|
||||
os.environ["PATH"] += os.pathsep + os.path.join(os.getenv("JAVA_HOME"), "bin")
|
||||
exit_code, output = subprocess.getstatusoutput("java -version")
|
||||
print("java version is:{}".format(output))
|
||||
exit_code, output = subprocess.getstatusoutput("%s --version" % allure_bin)
|
||||
if exit_code == 0:
|
||||
print("allure version is:{}".format(output))
|
||||
cmd = "%s generate result -o report" % allure_bin
|
||||
ret = os.system(cmd)
|
||||
if ret:
|
||||
print("allure generate report failed")
|
||||
else:
|
||||
print("allure generate report success")
|
||||
os.environ["REPORT_SERVER_USERNAME"] = os.getenv("REPORT_SERVER_USERNAME")
|
||||
os.environ["REPORT_SERVER_PASSWORD"] = os.getenv("REPORT_SERVER_PASSWORD")
|
||||
os.environ["REPORT_SERVER"] = os.getenv("REPORT_SERVER")
|
||||
job_build_id = os.getenv("AGILE_JOB_BUILD_ID")
|
||||
REPORT_SERVER = os.getenv("REPORT_SERVER")
|
||||
|
||||
cmd = "curl -s {}/report/upload.sh | bash -s ./report {} report".format(REPORT_SERVER, job_build_id)
|
||||
|
||||
if job_build_id:
|
||||
# upload allure report
|
||||
cmd = "curl -s {}/report/upload.sh | bash -s ./report {} report".format(REPORT_SERVER, job_build_id)
|
||||
print("upload cmd is {}".format(cmd))
|
||||
ret = os.system(cmd)
|
||||
else:
|
||||
print("非流水线任务，请补充9位数字流水线任务id")
|
||||
|
||||
if os.path.exists("allure-2.19.0.zip"):
|
||||
time.sleep(1)
|
||||
try:
|
||||
os.remove("allure-2.19.0.zip")
|
||||
except:
|
||||
print("#### can not remove allure-2.19.0.zip")
|
||||
if os.path.exists("java_linux.tar.gz"):
|
||||
time.sleep(1)
|
||||
try:
|
||||
os.remove("java_linux.tar.gz")
|
||||
except:
|
||||
print("#### can not remove java_linux.tar.gz")
|
||||
if os.path.exists("bos_new.tar.gz"):
|
||||
time.sleep(1)
|
||||
try:
|
||||
os.remove("bos_new.tar.gz")
|
||||
except:
|
||||
print("#### can not remove bos_new.tar.gz")
|
||||
return ret
|
||||
else:
|
||||
print("allure is not config correctly:{}, please config allure manually!".format(output))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
gen_allure_report()
|
||||
@@ -0,0 +1,109 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Get PaddleNLP develop model list && before merge pr """
|
||||
import io
|
||||
import os
|
||||
|
||||
|
||||
def get_model_list():
|
||||
"""
|
||||
get model list from
|
||||
<https://github.com/PaddlePaddle/PaddleNLP/model_zoo/>
|
||||
<https://github.com/PaddlePaddle/PaddleNLP/slm/examples/>
|
||||
"""
|
||||
|
||||
CI_MODEL_LIST = [
|
||||
"DuEE",
|
||||
"DuReader-robust",
|
||||
"DuReader-yesno",
|
||||
"SQuAD",
|
||||
"albert",
|
||||
"bert",
|
||||
"bigbird",
|
||||
"clue",
|
||||
"couplet",
|
||||
"doc",
|
||||
"elmo",
|
||||
"ernie",
|
||||
"ernie-1.0",
|
||||
"ernie-csc",
|
||||
"ernie_matching",
|
||||
"few_shot",
|
||||
"glue",
|
||||
"gpt",
|
||||
"gpt-3",
|
||||
"lexical_analysis",
|
||||
"minilmv2",
|
||||
"mpnet",
|
||||
"msra_ner",
|
||||
"msra_ner",
|
||||
"ofa",
|
||||
"pointer_summarizer",
|
||||
"pp-minilm",
|
||||
"pretrained_models",
|
||||
"question_matching",
|
||||
"rnn",
|
||||
"semantic_indexing",
|
||||
"sentiment_analysis",
|
||||
"simbert",
|
||||
"simbert",
|
||||
"simcse",
|
||||
"skep",
|
||||
"squad",
|
||||
"stacl",
|
||||
"stacl",
|
||||
"tcn",
|
||||
"tinybert",
|
||||
"transformer",
|
||||
"unimo-text",
|
||||
"vae-seq2seq",
|
||||
"word_embedding",
|
||||
]
|
||||
examples_second_list = ["model_interpretation", "semantic_indexing", "lexical_analysis", "word_embedding"]
|
||||
|
||||
model_list = os.listdir("slm/model_zoo")
|
||||
model_list = os.listdir("model_zoo")
|
||||
examples_list = os.listdir("slm/examples/")
|
||||
app_list = os.listdir("applications/")
|
||||
|
||||
# remove model_list README
|
||||
model_list.remove("README.md")
|
||||
examples_list.remove("README.md")
|
||||
model_list.extend(app_list)
|
||||
model_list.extend(examples_second_list)
|
||||
for examples_model_list in examples_list:
|
||||
if examples_model_list not in examples_second_list:
|
||||
examples_model = os.listdir("examples/" + examples_model_list)
|
||||
if "README.md" in examples_model:
|
||||
examples_model.remove("README.md")
|
||||
model_list.extend(examples_model)
|
||||
|
||||
all_examples_dict = set(sorted(model_list))
|
||||
no_test_models = []
|
||||
|
||||
# get model list not in CI/CE
|
||||
for full_model in all_examples_dict:
|
||||
if full_model not in CI_MODEL_LIST:
|
||||
no_test_models.append(full_model)
|
||||
|
||||
# save model list for CI run_ci.sh
|
||||
with io.open("./scripts/regression/model_list.txt", "w", encoding="utf-8") as list:
|
||||
for all_model in all_examples_dict:
|
||||
list.write("{}\n".format(all_model))
|
||||
list.close()
|
||||
return all_examples_dict
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
get_model_list()
|
||||
@@ -0,0 +1,46 @@
|
||||
opencv-python==4.6.0.66
|
||||
opencv-contrib-python==4.6.0.66
|
||||
protobuf==3.20.2
|
||||
regex
|
||||
attrdict==2.0.1
|
||||
easydict==1.10.0
|
||||
PyYAML
|
||||
subword_nmt==0.3.7
|
||||
visualdl
|
||||
pandas
|
||||
pyrouge
|
||||
nltk
|
||||
beautifulsoup4
|
||||
pybind11
|
||||
zstandard
|
||||
pypinyin
|
||||
pycryptodome
|
||||
bce-python-sdk==0.8.74
|
||||
packaging
|
||||
psutil
|
||||
pynvml
|
||||
GPUtil
|
||||
lac
|
||||
yacs
|
||||
coverage
|
||||
h5py
|
||||
pytest
|
||||
pytest-timeout
|
||||
parameterized
|
||||
scikit-learn
|
||||
cma
|
||||
paddleocr
|
||||
simsimd==1.1.2
|
||||
https://paddle-qa.bj.bcebos.com/PaddleSlim/paddleslim-0.0.0.dev0-py3-none-any.whl
|
||||
fast_dataindex
|
||||
emoji
|
||||
ftfy
|
||||
unidecode
|
||||
onnxruntime
|
||||
sacremoses
|
||||
soundfile
|
||||
librosa
|
||||
gradio>=3.45.0
|
||||
tiktoken
|
||||
wget
|
||||
allure-pytest
|
||||
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
####################################
|
||||
set -e
|
||||
export python=$1
|
||||
export paddle=$2
|
||||
export nlp_dir=/workspace/PaddleNLP
|
||||
mkdir -p /workspace/PaddleNLP/model_logs
|
||||
export log_path=/workspace/PaddleNLP/model_logs
|
||||
export P0case_list=()
|
||||
declare -A all_P0case_dic
|
||||
declare -A Build_list
|
||||
target_lists_for_llm=(
|
||||
"paddlenlp/transformers"
|
||||
"paddlenlp/experimental/transformers/"
|
||||
"paddlenlp/data"
|
||||
"paddlenlp/datasets"
|
||||
"paddlenlp/generation"
|
||||
"paddlenlp/peft"
|
||||
"paddlenlp/mergekit"
|
||||
"paddlenlp/quantization"
|
||||
"paddlenlp/trainer"
|
||||
"paddlenlp/trl"
|
||||
"llm"
|
||||
"tests/llm"
|
||||
"csrc"
|
||||
"scripts/regression"
|
||||
".github/workflows/llm.yml"
|
||||
)
|
||||
all_P0case_dic=(["msra_ner"]=15
|
||||
["glue"]=2
|
||||
["bert"]=2
|
||||
["skep"]=10
|
||||
["bigbird"]=2
|
||||
["ernie-1.0"]=2 ["ernie"]=2
|
||||
["ofa"]=2
|
||||
["albert"]=2
|
||||
["lexical_analysis"]=5
|
||||
["transformer"]=5
|
||||
["question_matching"]=5
|
||||
["ernie-csc"]=5
|
||||
["clue"]=5
|
||||
["taskflow"]=5
|
||||
["ernie-3.0"]=5
|
||||
["uie"]=5
|
||||
["ernie-layout"]=5 ["ernie_layout"]=5
|
||||
["ernie_csc"]=5
|
||||
["segment_parallel_utils"]=5
|
||||
["ring_flash_attention"]=5
|
||||
["llm"]=5)
|
||||
####################################
|
||||
|
||||
python -m pip config --user set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
python -m pip config --user set global.trusted-host pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
# Install paddlepaddle-gpu
|
||||
install_paddle(){
|
||||
echo -e "\033[35m ---- Install paddlepaddle-gpu \033[0m"
|
||||
python -m pip install --user -r scripts/regression/requirements_ci.txt
|
||||
python -m pip uninstall paddlepaddle -y
|
||||
python -m pip install --user ${paddle} --no-cache-dir;
|
||||
python -c "import paddle;print('paddle');print(paddle.__version__);print(paddle.version.show())" >> ${log_path}/commit_info.txt
|
||||
python -c 'from visualdl import LogWriter'
|
||||
}
|
||||
# Install paddlenlp func
|
||||
nlp_build (){
|
||||
cd $1
|
||||
rm -rf build/
|
||||
rm -rf paddlenlp.egg-info/
|
||||
rm -rf ppdiffusers.egg-info/
|
||||
rm -rf paddle_pipelines.egg-info/
|
||||
rm -rf dist/
|
||||
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install -r requirements-dev.txt
|
||||
python setup.py bdist_wheel
|
||||
python -m pip install --ignore-installed dist/p****.whl
|
||||
}
|
||||
install_external_ops(){
|
||||
echo -e "\033[31m ---- Install extern_ops \033"
|
||||
export PYTHONPATH=${nlp_dir}:$PYTHONPATH
|
||||
cd ${nlp_dir}/slm/model_zoo/gpt-3/external_ops
|
||||
python setup.py install
|
||||
python -c "import fused_ln;";
|
||||
cd ${nlp_dir}
|
||||
}
|
||||
####################################
|
||||
# get diff case
|
||||
cd ${nlp_dir}
|
||||
get_diff_TO_case(){
|
||||
if [ -z "${AGILE_COMPILE_BRANCH}" ]; then
|
||||
# 定时任务回归测试
|
||||
P0case_list=("llm")
|
||||
Build_list=(["paddlenlp"]="paddlenlp")
|
||||
else
|
||||
for file_name in `git diff --numstat ${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
|
||||
arr_file_name=(${file_name//// })
|
||||
dir1=${arr_file_name[0]}
|
||||
dir2=${arr_file_name[1]}
|
||||
dir3=${arr_file_name[2]}
|
||||
dir4=${arr_file_name[3]}
|
||||
file_item=$dir1/$dir2/$dir3/$dir4
|
||||
ext="${file_name##*.}"
|
||||
echo "file_name:"${file_name}, "dir1:"${dir1}, "dir2:"${dir2},"dir3:"${dir3},".xx:" ${file_name##*.}
|
||||
echo "ext: ${file_name##*.}"
|
||||
if [ ! -f ${file_name} ];then # 针对pr删掉文件
|
||||
continue
|
||||
elif [[ "$ext" == "md" || "$ext" == "rst" || "$file_name" == docs/* ]]; then
|
||||
continue
|
||||
elif [[ "${AGILE_COMPILE_BRANCH}" == "refactor-training-loop" ]];then # 针对特定分支
|
||||
P0case_list[${#P0case_list[*]}]=gpt
|
||||
else
|
||||
# 判断是否命中 target_lists_for_llm 列表-执行llm
|
||||
for ((i=0; i<${#target_lists_for_llm[@]}; i++)); do
|
||||
if [[ "${file_item}" == *"${target_lists_for_llm[i]}"* ]];then
|
||||
P0case_list[${#P0case_list[*]}]=llm
|
||||
fi
|
||||
done
|
||||
# 其他 case 判断
|
||||
if [[ ${dir1} =~ "scripts" ]];then # API 升级
|
||||
if [[ ${dir2} =~ "should_deploy" ]];then # 针对发版mini test
|
||||
P0case_list[${#P0case_list[*]}]=transformer
|
||||
fi
|
||||
elif [[ ${dir1} =~ "paddlenlp" ]];then # API 升级
|
||||
Build_list[${dir1}]="paddlenlp" # 影响编包
|
||||
if [[ ${dir2} =~ "__init__" ]];then # 针对发版mini test
|
||||
P0case_list[${#P0case_list[*]}]=bert
|
||||
elif [[ -n "${all_P0case_dic[$dir2]}" ]]; then
|
||||
P0case_list[${#P0case_list[*]}]=${dir2}
|
||||
elif [[ ${dir2} =~ "transformers" ]];then
|
||||
if [[ -n "${all_P0case_dic[$dir3]}" ]];then
|
||||
P0case_list[${#P0case_list[*]}]=${dir3}
|
||||
fi
|
||||
elif [[ ${dir2} =~ "taskflow" ]];then # ce case
|
||||
P0case_list[${#P0case_list[*]}]=taskflow
|
||||
fi
|
||||
elif [[ "${dir1}" =~ "slm" && "${dir2}" =~ "examples" ]];then # 模型升级
|
||||
if [[ -n "${all_P0case_dic[$dir2]}" ]];then
|
||||
P0case_list[${#P0case_list[*]}]=${dir2}
|
||||
elif [[ -n "${all_P0case_dic[$dir3]}" ]];then
|
||||
P0case_list[${#P0case_list[*]}]=${dir3}
|
||||
fi
|
||||
elif [[ "${dir1}" =~ "slm" && "${dir2}" =~ "model_zoo" ]];then # 模型升级
|
||||
if [[ -n "${all_P0case_dic[$dir2]}" ]];then
|
||||
P0case_list[${#P0case_list[*]}]=${dir2}
|
||||
fi
|
||||
elif [[ ${dir1} =~ "csrc" ]];then # 推理改动
|
||||
Build_list[${dir1}]="paddlenlp_ops" # 影响推理编包
|
||||
elif [[ ${dir1} =~ "requirements" ]];then # 依赖改动
|
||||
Build_list[${dir1}]="paddlenlp" # 影响paddlenlp编包
|
||||
else
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
get_diff_TO_case
|
||||
P0case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${P0case_list[*]}))
|
||||
####################################
|
||||
# build latest paddlenlp/paddlenlp_ops whl and install
|
||||
if [[ ${#Build_list[*]} -ne 0 ]];then
|
||||
install_paddle
|
||||
echo -e "\033[32m start build ${Build_list[*]} whl \033[0m"
|
||||
for build_pkg in ${Build_list[*]};do
|
||||
if [[ ${build_pkg} == "paddlenlp" ]];then
|
||||
echo -e "\033[35m ---- build ${GIT_PR_ID} paddlenlp \033[0m"
|
||||
nlp_build ${nlp_dir}
|
||||
elif [[ ${build_pkg} == "paddlenlp_ops" ]];then
|
||||
echo -e "\033[35m ---- build ${GIT_PR_ID} paddlenlp_ops \033[0m"
|
||||
export http_proxy=${proxy} && export https_proxy=${proxy}
|
||||
cd ${nlp_dir}/csrc
|
||||
bash tools/build_wheel.sh
|
||||
unset http_proxy && unset https_proxy
|
||||
else
|
||||
echo -e "\033[35m ---- build ${GIT_PR_ID} ${build_pkg} \033[0m"
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo -e "\033[32m Don't need build whl \033[0m"
|
||||
fi
|
||||
###################################
|
||||
if [[ ${#P0case_list[*]} -ne 0 ]];then
|
||||
cd ${nlp_dir}
|
||||
# Install paddle
|
||||
if [[ ${#Build_list[*]} -eq 0 ]];then
|
||||
install_paddle
|
||||
else
|
||||
echo "install_paddle done"
|
||||
fi
|
||||
# Install paddlenlp
|
||||
if [ ! -f ./dist/p****.whl ];then
|
||||
echo "install_nlp_develop"
|
||||
python -m pip install --user https://paddlenlp.bj.bcebos.com/wheels/paddlenlp-ci-py3-none-any.whl --no-cache-dir
|
||||
else
|
||||
echo "install_nlp_pr done"
|
||||
fi
|
||||
# install paddlenlp_ops
|
||||
if [ ! -f ./csrc/gpu_dist/p****.whl ];then
|
||||
echo "install_paddlenlp_ops_develop"
|
||||
python -m pip install --user https://paddlenlp.bj.bcebos.com/wheels/paddlenlp_ops-ci-py3-none-any.whl --no-cache-dir
|
||||
else
|
||||
echo "install_paddlenlp_ops_pr done"
|
||||
fi
|
||||
# install fused_ln
|
||||
install_external_ops
|
||||
python -c "from paddlenlp import __version__; print('paddlenlp version:', __version__)" >> ${log_path}/commit_info.txt
|
||||
python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)" >> ${log_path}/commit_info.txt
|
||||
python -m pip list >> ${log_path}/commit_info.txt
|
||||
|
||||
echo -e "\033[35m =======CI Check P0case========= \033[0m"
|
||||
echo -e "\033[35m ---- P0case_list length: ${#P0case_list[*]}, cases: ${P0case_list[*]} \033[0m"
|
||||
set +e
|
||||
echo -e "\033[35m ---- start run P0case \033[0m"
|
||||
case_num=1
|
||||
for p0case in ${P0case_list[*]};do
|
||||
echo -e "\033[35m ---- running P0case $case_num/${#P0case_list[*]}: ${p0case} \033[0m"
|
||||
bash ${nlp_dir}/scripts/regression/ci_case.sh ${p0case} ${cudaid1} ${cudaid2}
|
||||
let case_num++
|
||||
done
|
||||
echo -e "\033[35m ---- end run P0case \033[0m"
|
||||
cd ${nlp_dir}/model_logs
|
||||
FF=`ls *FAIL*|wc -l`
|
||||
EXCODE=0
|
||||
if [ "${FF}" -gt "0" ];then
|
||||
P0case_EXCODE=1
|
||||
EXCODE=2
|
||||
else
|
||||
P0case_EXCODE=0
|
||||
fi
|
||||
if [ $P0case_EXCODE -ne 0 ] ; then
|
||||
echo -e "\033[31m ---- P0case Failed number: ${FF} \033[0m"
|
||||
ls *_FAIL*
|
||||
else
|
||||
echo -e "\033[32m ---- P0case Success \033[0m"
|
||||
fi
|
||||
####################################
|
||||
if [ -n "${AGILE_JOB_BUILD_ID}" ]; then
|
||||
cd ${nlp_dir}
|
||||
echo -e "\033[35m ---- Generate Allure Report \033[0m"
|
||||
unset http_proxy && unset https_proxy
|
||||
cp scripts/regression/gen_allure_report.py ./
|
||||
python gen_allure_report.py > /dev/null
|
||||
echo -e "\033[35m ---- Report: https://xly.bce.baidu.com/ipipe/ipipe-report/report/${AGILE_JOB_BUILD_ID}/report/ \033[0m"
|
||||
fi
|
||||
####################################
|
||||
# run coverage
|
||||
# cd ${nlp_dir}/tests/
|
||||
# bash run_coverage.sh
|
||||
# Coverage_EXCODE=$? || true
|
||||
# mv ./htmlcov ${nlp_dir}/coverage_logs/
|
||||
# if [ $Coverage_EXCODE -ne 0 ] ; then
|
||||
# echo -e "\033[31m ---- Coverage Failed \033[0m"
|
||||
# else
|
||||
# echo -e "\033[32m ---- Coverage Success \033[0m"
|
||||
# fi
|
||||
####################################
|
||||
else
|
||||
echo -e "\033[32m Changed Not CI case, Skips \033[0m"
|
||||
EXCODE=0
|
||||
fi
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
export nlp_dir=/workspace/PaddleNLP/
|
||||
mkdir ${nlp_dir}/logs
|
||||
mkdir ${nlp_dir}/model_logs
|
||||
mkdir ${nlp_dir}/unittest_logs
|
||||
p0case="refactor_training_loop"
|
||||
|
||||
# Install Paddle FleetY
|
||||
install_paddle(){
|
||||
echo -e "\033[35m ---- Install paddlepaddle-gpu \033[0m"
|
||||
python -m pip install paddlepaddle_gpu-0.0.0-cp37-cp37m-linux_x86_64.whl
|
||||
python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
|
||||
}
|
||||
# Install paddlenlp
|
||||
nlp_build (){
|
||||
cd ${nlp_dir}
|
||||
rm -rf build/
|
||||
rm -rf paddlenlp.egg-info/
|
||||
rm -rf ppdiffusers.egg-info/
|
||||
rm -rf paddle_pipelines.egg-info/
|
||||
rm -rf dist/
|
||||
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install rouge
|
||||
python setup.py bdist_wheel
|
||||
python -m pip install dist/p****.whl
|
||||
}
|
||||
install_paddle
|
||||
nlp_build
|
||||
pip list
|
||||
|
||||
# run ci case
|
||||
echo -e "\033[35m ======= Check refactor_training_loop BRANCH ========= \033[0m"
|
||||
set +e
|
||||
echo -e "\033[35m ---- start run case \033[0m"
|
||||
bash ${nlp_dir}/scripts/regression/ci_case.sh ${p0case} ${cudaid1} ${cudaid2}
|
||||
echo -e "\033[35m ---- end run P0case \033[0m"
|
||||
|
||||
# analysis log
|
||||
cd ${nlp_dir}/model_logs
|
||||
FF=`ls *FAIL*|wc -l`
|
||||
EXCODE=0
|
||||
if [ "${FF}" -gt "0" ];then
|
||||
P0case_EXCODE=1
|
||||
EXCODE=2
|
||||
else
|
||||
P0case_EXCODE=0
|
||||
fi
|
||||
if [ $P0case_EXCODE -ne 0 ] ; then
|
||||
echo -e "\033[31m ---- P0case Failed number: ${FF} \033[0m"
|
||||
ls *_FAIL*
|
||||
else
|
||||
echo -e "\033[32m ---- P0case Success \033[0m"
|
||||
fi
|
||||
|
||||
cd ${nlp_dir}/unittest_logs
|
||||
UF=`ls *FAIL*|wc -l`
|
||||
if [ "${UF}" -gt "0" ];then
|
||||
UT_EXCODE=1
|
||||
EXCODE=3
|
||||
else
|
||||
UT_EXCODE=0
|
||||
fi
|
||||
if [ $UT_EXCODE -ne 0 ] ; then
|
||||
echo -e "\033[31m ---- Unittest Failed \033[0m"
|
||||
ls *_FAIL*
|
||||
else
|
||||
echo -e "\033[32m ---- Unittest Success \033[0m"
|
||||
fi
|
||||
exit $EXCODE
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
export Testcase=$3
|
||||
export cudaid2=$2
|
||||
export cudaid1=$1
|
||||
export nlp_dir=${PWD}
|
||||
mkdir ${nlp_dir}/logs
|
||||
mkdir ${nlp_dir}/model_logs
|
||||
mkdir ${nlp_dir}/unittest_logs
|
||||
export log_path=${nlp_dir}/logs
|
||||
####################################
|
||||
# for paddlenlp env
|
||||
python -c 'import sys; print(sys.version_info[:])'
|
||||
python -c 'import nltk; nltk.download("punkt")'
|
||||
set -x
|
||||
python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
|
||||
nlp1_build (){
|
||||
echo -e "\033[35m ---- only install paddlenlp \033[0m"
|
||||
python -m pip install paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html
|
||||
}
|
||||
nlp2_build (){
|
||||
echo -e "\033[35m ---- build and install paddlenlp \033[0m"
|
||||
rm -rf build/
|
||||
rm -rf paddlenlp.egg-info/
|
||||
rm -rf dist/
|
||||
|
||||
python -m pip install -r requirements.txt
|
||||
python setup.py bdist_wheel
|
||||
python -m pip install -U dist/paddlenlp****.whl
|
||||
}
|
||||
nlp2_build
|
||||
python -c 'from visualdl import LogWriter'
|
||||
pip list
|
||||
set +x
|
||||
####################################
|
||||
# run p0case
|
||||
export P0case_list=()
|
||||
export P0case_time=0
|
||||
export all_P0case_time=0
|
||||
declare -A all_P0case_dic
|
||||
get_diff_TO_P0case(){
|
||||
if [[ ${Testcase} =~ "all" ]];then
|
||||
P0case_list=(msra_ner glue bert skep bigbird gpt ernie-1.0 xlnet ofa squad tinybert lexical_analysis seq2seq \
|
||||
word_embedding ernie-ctm distilbert stacl transformer simbert pointer_summarizer question_matching ernie-csc \
|
||||
nptag clue taskflow transformers fast_generation ernie-3.0 fast_transformer fast_gpt llama)
|
||||
elif [[ ${Testcase} =~ "p0" ]];then
|
||||
P0case_list=(glue bert skep gpt ernie-1.0 transformer clue)
|
||||
else
|
||||
P0case_list=${Testcase}
|
||||
fi
|
||||
}
|
||||
get_diff_TO_P0case
|
||||
echo -e "\033[35m =======CI Check P0case========= \033[0m"
|
||||
echo -e "\033[35m ---- P0case_list length: ${#P0case_list[*]}, cases: ${P0case_list[*]} \033[0m"
|
||||
set +e
|
||||
echo -e "\033[35m ---- start run P0case \033[0m"
|
||||
case_num=1
|
||||
for p0case in ${P0case_list[*]};do
|
||||
echo -e "\033[35m ---- running P0case $case_num/${#P0case_list[*]}: ${p0case} \033[0m"
|
||||
bash ${nlp_dir}/scripts/regression/ci_case.sh ${p0case} ${cudaid1} ${cudaid2}
|
||||
let case_num++
|
||||
done
|
||||
echo -e "\033[35m ---- end run P0case \033[0m"
|
||||
cd ${nlp_dir}
|
||||
upload(){
|
||||
if [ -f '/ssd1/paddlenlp/bos/upload.py' ];then
|
||||
cp -r /ssd1/paddlenlp/bos/* ./
|
||||
tar -zcvf model_logs.tar model_logs/
|
||||
mkdir upload && mv model_logs.tar upload
|
||||
python upload.py upload 'paddle-qa/paddlenlp'
|
||||
else
|
||||
echo 'No upload script found'
|
||||
fi
|
||||
}
|
||||
upload
|
||||
cd model_logs/
|
||||
FF=`ls *_FAIL*|wc -l`
|
||||
if [ "${FF}" -gt "0" ];then
|
||||
P0case_EXCODE=1
|
||||
else
|
||||
P0case_EXCODE=0
|
||||
fi
|
||||
if [ $P0case_EXCODE -ne 0 ] ; then
|
||||
cd model_logs/
|
||||
FF=`ls *_FAIL*|wc -l`
|
||||
echo -e "\033[31m ---- P0case failed number: ${FF} \033[0m"
|
||||
ls *_FAIL*
|
||||
exit $P0case_EXCODE
|
||||
else
|
||||
echo -e "\033[32m ---- P0case Success \033[0m"
|
||||
fi
|
||||
@@ -0,0 +1,259 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Test taskflow."""
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from paddlenlp import Taskflow
|
||||
|
||||
|
||||
def test_knowledge_mining():
|
||||
"""
|
||||
test_knowledge_mining
|
||||
"""
|
||||
wordtag = Taskflow("knowledge_mining", model="wordtag", batch_size=2, max_seq_len=128, linking=True)
|
||||
wordtag("《孤女》是2010年九州出版社出版的小说，作者是余兼羽。")
|
||||
|
||||
nptag = Taskflow("knowledge_mining", model="nptag", batch_size=2, max_seq_len=128, linking=True)
|
||||
nptag(["糖醋排骨", "红曲霉菌"])
|
||||
|
||||
|
||||
def test_name_entity_recognition():
|
||||
"""
|
||||
test_name_entity_recognition
|
||||
"""
|
||||
ner = Taskflow("ner", batch_size=2)
|
||||
ner("《长津湖》收尾，北美是最大海外票仓")
|
||||
ner_fast = Taskflow("ner", mode="fast")
|
||||
ner_fast("《长津湖》收尾，北美是最大海外票仓")
|
||||
ner_entity = Taskflow("ner", mode="accurate", entity_only=True)
|
||||
ner_entity("《孤女》是2010年九州出版社出版的小说，作者是余兼羽")
|
||||
|
||||
|
||||
def test_word_segmetation():
|
||||
"""
|
||||
test_word_segmetation
|
||||
"""
|
||||
seg = Taskflow("word_segmentation", batch_size=2)
|
||||
seg(["第十四届全运会在西安举办", "三亚是一个美丽的城市"])
|
||||
seg_fast = Taskflow("word_segmentation", mode="fast")
|
||||
seg_fast(["第十四届全运会在西安举办", "三亚是一个美丽的城市"])
|
||||
seg_acc = Taskflow("word_segmentation", mode="accurate")
|
||||
seg_acc("李伟拿出具有科学性、可操作性的《陕西省高校管理体制改革实施方案》")
|
||||
|
||||
|
||||
def test_pos_tagging():
|
||||
"""
|
||||
test_pos_tagging
|
||||
"""
|
||||
tag = Taskflow("pos_tagging", batch_size=2)
|
||||
tag("第十四届全运会在西安举办")
|
||||
|
||||
|
||||
def test_corrector():
|
||||
"""
|
||||
test_corrector
|
||||
"""
|
||||
corrector = Taskflow("text_correction", batch_size=2)
|
||||
corrector("遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇，这样我们才能朝著成功之路前进。")
|
||||
|
||||
|
||||
@unittest.skip("dependency_parsing is not support for Paddle >= 2.6.1")
|
||||
def test_dependency_parsing():
|
||||
"""
|
||||
test_dependency_parsing
|
||||
"""
|
||||
ddp = Taskflow("dependency_parsing", model="ddparser", batch_size=2, prob=True, use_pos=True)
|
||||
print(ddp("9月9日上午纳达尔在亚瑟·阿什球场击败俄罗斯球员梅德韦杰夫"))
|
||||
print(ddp.from_segments([["9月9日", "上午", "纳达尔", "在", "亚瑟·阿什球场", "击败", "俄罗斯", "球员", "梅德韦杰夫"]]))
|
||||
|
||||
ddp_ernie = Taskflow("dependency_parsing", model="ddparser-ernie-1.0", batch_size=2, prob=True, use_pos=True)
|
||||
print(ddp_ernie("9月9日上午纳达尔在亚瑟·阿什球场击败俄罗斯球员梅德韦杰夫"))
|
||||
|
||||
ddp_ernie_gram = Taskflow(
|
||||
"dependency_parsing", model="ddparser-ernie-gram-zh", batch_size=2, prob=True, use_pos=True
|
||||
)
|
||||
print(ddp_ernie_gram("9月9日上午纳达尔在亚瑟·阿什球场击败俄罗斯球员梅德韦杰夫"))
|
||||
|
||||
|
||||
def test_sentiment_analysis():
|
||||
"""
|
||||
test_sentiment_analysis
|
||||
"""
|
||||
skep = Taskflow("sentiment_analysis", batch_size=2)
|
||||
skep("这个产品用起来真的很流畅，我非常喜欢")
|
||||
|
||||
skep_ernie = Taskflow("sentiment_analysis", model="skep_ernie_1.0_large_ch", batch_size=2)
|
||||
skep_ernie("作为老的四星酒店，房间依然很整洁，相当不错。机场接机服务很好，可以在车上办理入住手续，节省时间。")
|
||||
|
||||
|
||||
def test_text_similarity():
|
||||
"""
|
||||
test_text_similarity
|
||||
"""
|
||||
similarity = Taskflow("text_similarity", batch_size=2)
|
||||
similarity([["世界上什么东西最小", "世界上什么东西最小？"]])
|
||||
|
||||
|
||||
def test_question_answering():
|
||||
"""
|
||||
test_question_answering
|
||||
"""
|
||||
qa = Taskflow("question_answering", batch_size=2)
|
||||
qa("中国的国土面积有多大？")
|
||||
|
||||
|
||||
def test_poetry():
|
||||
"""
|
||||
test_poetry
|
||||
"""
|
||||
poetry = Taskflow("poetry_generation", batch_size=2)
|
||||
poetry("林密不见人")
|
||||
|
||||
|
||||
def test_dialogue():
|
||||
"""
|
||||
test_dialogue
|
||||
"""
|
||||
dialogue = Taskflow("dialogue", batch_size=2, max_seq_len=512)
|
||||
dialogue(["吃饭了吗"])
|
||||
|
||||
|
||||
def test_uie():
|
||||
"""
|
||||
test_uie
|
||||
"""
|
||||
schema_ner = ["时间", "选手", "赛事名称"] # Define the schema for entity extraction
|
||||
ie = Taskflow("information_extraction", schema=schema_ner, model="uie-base", batch_size=2, prob=True, use_pos=True)
|
||||
ie("2月8日上午北京冬奥会自由式滑雪女子大跳台决赛中中国选手谷爱凌以188.25分获得金牌！")
|
||||
|
||||
ie = Taskflow("information_extraction", schema=schema_ner, model="uie-tiny", batch_size=2, prob=True, use_pos=True)
|
||||
schema_re = {"歌曲名称": ["歌手", "所属专辑"]} # Define the schema for relation extraction
|
||||
ie.set_schema(schema_re) # Reset schema
|
||||
ie("《告别了》是孙耀威在专辑爱的故事里面的歌曲")
|
||||
|
||||
ie = Taskflow("information_extraction", schema=schema_ner, prob=True, use_pos=True)
|
||||
schema_ee = {"歌曲名称": ["歌手", "所属专辑"]} # Define the schema for relation extraction
|
||||
ie.set_schema(schema_ee) # Reset schema
|
||||
ie("《告别了》是孙耀威在专辑爱的故事里面的歌曲")
|
||||
|
||||
schema_opinion = {"评价维度": "观点词"} # Define the schema for opinion extraction
|
||||
ie.set_schema(schema_opinion) # Reset schema
|
||||
ie("个人觉得管理太混乱了，票价太高了")
|
||||
|
||||
schema_sa = "情感倾向[正向，负向]" # Define the schema for sentence-level sentiment classification
|
||||
ie.set_schema(schema_sa) # Reset schema
|
||||
ie("这个产品用起来真的很流畅，我非常喜欢")
|
||||
|
||||
schema_bre = ["寺庙", {"丈夫": "妻子"}]
|
||||
ie.set_schema(schema_bre)
|
||||
ie("李治即位后，让身在感业寺的武则天续起头发，重新纳入后宫。")
|
||||
|
||||
schema = {"竞赛名称": ["主办方", "承办方", "已举办次数"]}
|
||||
ie.set_schema(schema)
|
||||
ie("2022语言与智能技术竞赛由中国中文信息学会和中国计算机学会联合主办，百度公司、中国中文信息学会评测工作委员会和中国计算机学会自然语言处理专委会承办，已连续举办4届，成为全球最热门的中文NLP赛事之一。")
|
||||
|
||||
schema = ["Person", "Organization"]
|
||||
ie_en = Taskflow("information_extraction", schema=schema, model="uie-base-en")
|
||||
ie_en("In 1997, Steve was excited to become the CEO of Apple.")
|
||||
|
||||
schema = [{"Person": ["Company", "Position"]}]
|
||||
ie_en.set_schema(schema)
|
||||
ie_en("In 1997, Steve was excited to become the CEO of Apple.")
|
||||
|
||||
schema = [{"Aspect": ["Opinion", "Sentiment classification [negative, positive]"]}]
|
||||
ie_en.set_schema(schema)
|
||||
ie_en("The teacher is very nice.")
|
||||
|
||||
schema = "Sentiment classification [negative, positive]"
|
||||
ie_en.set_schema(schema)
|
||||
ie_en("I am sorry but this is the worst film I have ever seen in my life.")
|
||||
|
||||
|
||||
def test_summarizer():
|
||||
"""
|
||||
test_summarizer
|
||||
"""
|
||||
summarizer = Taskflow("text_summarization")
|
||||
summarizer("2022年，中国房地产进入转型阵痛期，传统“高杠杆、快周转”的模式难以为继，万科甚至直接喊话，中国房地产进入“黑铁时代”")
|
||||
summarizer(
|
||||
[
|
||||
"据悉，2022年教育部将围绕“巩固提高、深化落实、创新突破”三个关键词展开工作。要进一步强化学校教育主阵地作用，继续把落实“双减”作为学校工作的重中之重，\
|
||||
重点从提高作业设计水平、提高课后服务水平、提高课堂教学水平、提高均衡发展水平四个方面持续巩固提高学校“双减”工作水平。",
|
||||
"党参有降血脂，降血压的作用，可以彻底消除血液中的垃圾，从而对冠心病以及心血管疾病的患者都有一定的稳定预防工作作用，因此平时口服党参能远离三高的危害。\
|
||||
另外党参除了益气养血，降低中枢神经作用，调整消化系统功能，健脾补肺的功能。",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_uiex():
|
||||
"""UIE-X"""
|
||||
path = "./cases/"
|
||||
if not os.path.exists(path):
|
||||
os.mkdir(path)
|
||||
os.system(
|
||||
"cd %s && wget %s"
|
||||
% (
|
||||
path,
|
||||
"https://user-images.githubusercontent.com/40840292/203457596-8dbc9241-833d-4b0e-9291-f134a790d0e1.jpeg",
|
||||
)
|
||||
)
|
||||
os.system(
|
||||
"cd %s && wget %s"
|
||||
% (
|
||||
path,
|
||||
"https://user-images.githubusercontent.com/40840292/203457719-84a70241-607e-4bb1-ab4c-3d9beee9e254.jpeg",
|
||||
)
|
||||
)
|
||||
os.system(
|
||||
"cd %s && wget %s"
|
||||
% (
|
||||
path,
|
||||
"https://user-images.githubusercontent.com/40840292/203457817-76fe638a-3277-4619-9066-d1dffd52c5d4.jpg ",
|
||||
)
|
||||
)
|
||||
ie = Taskflow(
|
||||
"information_extraction",
|
||||
schema="",
|
||||
schema_lang="ch",
|
||||
ocr_lang="ch",
|
||||
batch_size=16,
|
||||
model="uie-x-base",
|
||||
layout_analysis=False,
|
||||
position_prob=0.5,
|
||||
precision="fp32",
|
||||
use_fast=True,
|
||||
)
|
||||
schema = ["姓名", "性别", "学校"]
|
||||
ie({"doc": "./cases/203457596-8dbc9241-833d-4b0e-9291-f134a790d0e1.jpeg"})
|
||||
|
||||
schema = ["收发货人", "进口口岸", "进口日期", "申报日期", "提运单号"]
|
||||
ie.set_schema(schema)
|
||||
print(ie({"doc": "./cases/203457719-84a70241-607e-4bb1-ab4c-3d9beee9e254.jpeg"}))
|
||||
|
||||
schema = {"项目名": "单价"}
|
||||
ie.set_schema(schema)
|
||||
print(ie({"doc": "./cases/203457817-76fe638a-3277-4619-9066-d1dffd52c5d4.jpg"}))
|
||||
|
||||
|
||||
def test_codegen():
|
||||
""" """
|
||||
prompt = "def lengthOfLongestSubstring(self, s: str) -> int:"
|
||||
codegen = Taskflow(
|
||||
"code_generation",
|
||||
model="Salesforce/codegen-350M-mono",
|
||||
decode_strategy="greedy_search",
|
||||
repetition_penalty=1.0,
|
||||
)
|
||||
print(codegen(prompt))
|
||||
@@ -0,0 +1,82 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from pkg_resources import parse_version
|
||||
|
||||
|
||||
def read_version_of_remote_package(name: str) -> str:
|
||||
"""get version of remote package,
|
||||
|
||||
adapted from: https://stackoverflow.com/a/58649262/6894382
|
||||
|
||||
Args:
|
||||
name (str): the name of package
|
||||
|
||||
Returns:
|
||||
str: the version of package
|
||||
"""
|
||||
latest_version = str(
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "{}==random".format(name)], capture_output=True, text=True
|
||||
)
|
||||
)
|
||||
latest_version = latest_version[latest_version.find("(from versions:") + 15 :]
|
||||
latest_version = latest_version[: latest_version.find(")")]
|
||||
latest_version = latest_version.replace(" ", "").split(",")[-1]
|
||||
return latest_version
|
||||
|
||||
|
||||
def read_version_of_local_package(version_file_path: str) -> str:
|
||||
"""get version of local package
|
||||
|
||||
Args:
|
||||
version_file_path (str): the path of `VERSION` file
|
||||
|
||||
Returns:
|
||||
str: the version of local package
|
||||
"""
|
||||
with open(version_file_path, "r", encoding="utf-8") as f:
|
||||
version = f.read().strip()
|
||||
return version
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--name", required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
version_file_map = {
|
||||
"ppdiffusers": "ppdiffusers/VERSION",
|
||||
"paddle-pipelines": "pipelines/VERSION",
|
||||
}
|
||||
remote_version = read_version_of_remote_package(args.name)
|
||||
|
||||
if args.name == "paddlenlp":
|
||||
local_version = str(subprocess.check_output(["python", "setup.py", "--version"], text=True))
|
||||
elif args.name in version_file_map:
|
||||
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
local_version_file = os.path.join(PROJECT_ROOT, version_file_map[args.name])
|
||||
local_version = read_version_of_local_package(local_version_file)
|
||||
else:
|
||||
raise ValueError(f"package<{args.name}> not supported")
|
||||
|
||||
should_deploy = str(parse_version(remote_version) < parse_version(local_version)).lower()
|
||||
print(f"should_deploy={should_deploy}")
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -e
|
||||
export paddle=$1
|
||||
export FLAGS_enable_CE=${2-false}
|
||||
export nlp_dir=/workspace/PaddleNLP
|
||||
export log_path=/workspace/PaddleNLP/unittest_logs
|
||||
cd $nlp_dir
|
||||
|
||||
if [ ! -d "unittest_logs" ];then
|
||||
mkdir unittest_logs
|
||||
fi
|
||||
|
||||
install_requirements() {
|
||||
python -m pip config --user set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
python -m pip config --user set global.trusted-host pypi.tuna.tsinghua.edu.cn
|
||||
python -m pip install -r requirements.txt
|
||||
python -m pip install -r requirements-dev.txt
|
||||
python -m pip install -r tests/requirements.txt
|
||||
# python -m pip install -r paddlenlp/experimental/autonlp/requirements.txt
|
||||
python -m pip uninstall paddlepaddle paddlepaddle_gpu -y
|
||||
python -m pip install --no-cache-dir ${paddle}
|
||||
python -c "import paddle;print('paddle');print(paddle.__version__);print(paddle.version.show())" >> ${log_path}/commit_info.txt
|
||||
|
||||
python setup.py bdist_wheel > /dev/null
|
||||
python -m pip install dist/p****.whl
|
||||
python -c "from paddlenlp import __version__; print('paddlenlp version:', __version__)" >> ${log_path}/commit_info.txt
|
||||
python -c "import paddlenlp; print('paddlenlp commit:',paddlenlp.version.commit)" >> ${log_path}/commit_info.txt
|
||||
python -m pip list >> ${log_path}/commit_info.txt
|
||||
}
|
||||
|
||||
set_env() {
|
||||
export NVIDIA_TF32_OVERRIDE=0
|
||||
export FLAGS_cudnn_deterministic=1
|
||||
export HF_ENDPOINT=https://hf-mirror.com
|
||||
export FLAGS_use_cuda_managed_memory=true
|
||||
export running_time=40m
|
||||
|
||||
# for CE
|
||||
if [[ ${FLAGS_enable_CE} == "true" ]];then
|
||||
export CE_TEST_ENV=1
|
||||
export RUN_SLOW_TEST=1
|
||||
export PYTHONPATH=${nlp_dir}:${nlp_dir}/llm:${PYTHONPATH}
|
||||
export running_time=5h
|
||||
fi
|
||||
}
|
||||
|
||||
print_info() {
|
||||
if [ $1 -ne 0 ]; then
|
||||
cat ${log_path}/unittest.log | grep -v "Fail to fscanf: Success" \
|
||||
| grep -v "SKIPPED" | grep -v "warning" > ${log_path}/unittest_FAIL.log
|
||||
tail -n 1 ${log_path}/unittest.log >> ${log_path}/unittest_FAIL.log
|
||||
echo -e "\033[31m ${log_path}/unittest_FAIL \033[0m"
|
||||
cat ${log_path}/unittest_FAIL.log
|
||||
if [ -n "${AGILE_JOB_BUILD_ID}" ]; then
|
||||
cp ${log_path}/unittest_FAIL.log ${PPNLP_HOME}/upload/unittest_FAIL.log.${AGILE_PIPELINE_BUILD_ID}.${AGILE_JOB_BUILD_ID}
|
||||
cd ${PPNLP_HOME} && python upload.py ${PPNLP_HOME}/upload 'paddlenlp/PaddleNLP_CI/PaddleNLP-CI-Unittest-GPU'
|
||||
rm -rf upload/* && cd -
|
||||
fi
|
||||
if [ $1 -eq 124 ]; then
|
||||
echo "\033[32m [failed-timeout] Test case execution was terminated after exceeding the ${running_time} min limit."
|
||||
fi
|
||||
else
|
||||
tail -n 1 ${log_path}/unittest.log
|
||||
echo -e "\033[32m ${log_path}/unittest_SUCCESS \033[0m"
|
||||
fi
|
||||
}
|
||||
|
||||
get_diff_TO_case(){
|
||||
export FLAGS_enable_CI=false
|
||||
if [ -z "${AGILE_COMPILE_BRANCH}" ]; then
|
||||
# 定时任务回归测试
|
||||
export FLAGS_enable_CI=true
|
||||
else
|
||||
for file_name in `git diff --numstat ${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
|
||||
ext="${file_name##*.}"
|
||||
echo "file_name: ${file_name}, ext: ${file_name##*.}"
|
||||
|
||||
if [ ! -f ${file_name} ];then # 针对pr删掉文件
|
||||
continue
|
||||
elif [[ "$ext" == "md" || "$ext" == "rst" || "$file_name" == docs/* ]]; then
|
||||
continue
|
||||
else
|
||||
FLAGS_enable_CI=true
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
get_diff_TO_case
|
||||
set_env
|
||||
if [[ ${FLAGS_enable_CI} == "true" ]] || [[ ${FLAGS_enable_CE} == "true" ]];then
|
||||
install_requirements
|
||||
cd ${nlp_dir}
|
||||
echo ' Testing all unittest cases '
|
||||
export http_proxy=${proxy} && export https_proxy=${proxy}
|
||||
set +e
|
||||
timeout ${running_time} python -m pytest -v -n 8 \
|
||||
--dist loadgroup \
|
||||
--retries 1 --retry-delay 1 \
|
||||
--timeout 200 --durations 20 --alluredir=result \
|
||||
--cov paddlenlp --cov-report xml:coverage.xml > ${log_path}/unittest.log 2>&1
|
||||
exit_code=$?
|
||||
print_info $exit_code unittest
|
||||
|
||||
if [ -n "${AGILE_JOB_BUILD_ID}" ]; then
|
||||
cd ${nlp_dir}
|
||||
echo -e "\033[35m ---- Generate Allure Report \033[0m"
|
||||
unset http_proxy && unset https_proxy
|
||||
cp scripts/regression/gen_allure_report.py ./
|
||||
python gen_allure_report.py > /dev/null
|
||||
echo -e "\033[35m ---- Report: https://xly.bce.baidu.com/ipipe/ipipe-report/report/${AGILE_JOB_BUILD_ID}/report/ \033[0m"
|
||||
else
|
||||
echo "AGILE_JOB_BUILD_ID is empty, skip generate allure report"
|
||||
fi
|
||||
else
|
||||
echo -e "\033[32m Changed Not CI case, Skips \033[0m"
|
||||
exit_code=0
|
||||
fi
|
||||
exit $exit_code
|
||||
Reference in New Issue
Block a user