wehub-resource-sync
661 lines
23 KiB
Python
661 lines
23 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
# port range (21200, 23000) is reserved for dist-ops
|
|
|
|
|
|
# function to process pythonpath env
|
|
# append "${PADDLE_BINARY_DIR}/python" to PYTHONPATH
|
|
def _process_PYTHONPATH(pythonpath_option):
|
|
pythonpath_option += ":${PADDLE_BINARY_DIR}/python"
|
|
return pythonpath_option
|
|
|
|
|
|
def _process_envs(envs):
|
|
"""
|
|
Desc:
|
|
Input a str and output a str with the same function to specify some environment variables.
|
|
Here we can give a special process for some variable if needed.
|
|
Example 1:
|
|
Input: "http_proxy=;PYTHONPATH=.."
|
|
Output: "http_proxy=;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python"
|
|
Example 2:
|
|
Input: "http_proxy=;https_proxy=123.123.123.123:1230"
|
|
Output: "http_proxy=;https_proxy=123.123.123.123:1230"
|
|
"""
|
|
envs = envs.strip()
|
|
|
|
envs_parts = envs.split(";")
|
|
processed_envs = []
|
|
|
|
for p in envs_parts:
|
|
assert (
|
|
" " not in p
|
|
and re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None
|
|
), f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_',
|
|
and the var can not contain space in either env names or values.
|
|
However the var's format is '{p}'."""
|
|
|
|
# if p starts with "PYTHONPATH=", then process python path
|
|
if re.compile("^PYTHONPATH=").search(p):
|
|
p = _process_PYTHONPATH(p)
|
|
|
|
processed_envs.append(p)
|
|
|
|
return ";".join(processed_envs)
|
|
|
|
|
|
def _process_conditions(conditions):
|
|
"""
|
|
Desc:
|
|
Input condition expression in cmake grammar and return a string wrapped by 'AND ()'.
|
|
If the conditions string is empty, return an empty string.
|
|
Example 1:
|
|
Input: "LINUX"
|
|
Output: "AND (LINUX)"
|
|
Example 2:
|
|
Input: ""
|
|
Output: ""
|
|
"""
|
|
if len(conditions.strip()) == 0:
|
|
conditions = []
|
|
else:
|
|
conditions = conditions.strip().split(";")
|
|
return [c.strip() for c in conditions]
|
|
|
|
|
|
def _process_archs(arch):
|
|
"""
|
|
desc:
|
|
Input archs options and warp it with 'WITH_', 'OR' and '()' in cmakelist grammar.
|
|
The case is ignored.
|
|
If the input is empty, return "LOCAL_ALL_ARCH".
|
|
Example 1:
|
|
Input: 'gpu'
|
|
Output: '(WITH_GPU)'
|
|
Example 2:
|
|
Input: 'gpu;ROCM'
|
|
Output: '(WITH_GPU OR WITH_ROCM)'
|
|
"""
|
|
archs = ""
|
|
arch = arch.upper().strip()
|
|
if len(arch) > 0:
|
|
for a in arch.split(";"):
|
|
if '' == a:
|
|
continue
|
|
assert a in ["GPU", "ROCM", "XPU"], (
|
|
f"""Supported arch options are "GPU", "ROCM", and "XPU", but the options is {a}"""
|
|
)
|
|
archs += "WITH_" + a.upper() + " OR "
|
|
arch = "(" + archs[:-4] + ")"
|
|
else:
|
|
arch = "LOCAL_ALL_ARCH"
|
|
return arch
|
|
|
|
|
|
def _process_os(os_):
|
|
"""
|
|
Desc:
|
|
Input os options and output wrapped options with 'OR' and '()'
|
|
If the input is empty, return "LOCAL_ALL_PLAT"
|
|
Example 1:
|
|
Input: "WIN32"
|
|
Output: "(WIN32)"
|
|
Example 2:
|
|
Input: "WIN32;linux"
|
|
Output: "(WIN32 OR LINUX)"
|
|
"""
|
|
os_ = os_.strip()
|
|
if len(os_) > 0:
|
|
os_ = os_.upper()
|
|
for p in os_.split(';'):
|
|
assert p in ["WIN32", "APPLE", "LINUX"], (
|
|
f"""Supported os options are 'WIN32', 'APPLE' and 'LINUX', but the options is {p}"""
|
|
)
|
|
os_ = os_.replace(";", " OR ")
|
|
os_ = "(" + os_ + ")"
|
|
else:
|
|
os_ = "LOCAL_ALL_PLAT"
|
|
return os_
|
|
|
|
|
|
# check whether run_serial is 0, 1 or empty
|
|
def _process_run_serial(run_serial):
|
|
rs = run_serial.strip()
|
|
assert rs in [
|
|
"1",
|
|
"0",
|
|
"",
|
|
], (
|
|
f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}"""
|
|
)
|
|
if rs == "":
|
|
return ""
|
|
return rs
|
|
|
|
|
|
def _file_with_extension(prefix, suffixes):
|
|
"""
|
|
Desc:
|
|
check whether test file exists.
|
|
"""
|
|
for ext in suffixes:
|
|
if os.path.isfile(prefix + ext):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _process_name(name, curdir):
|
|
"""
|
|
Desc:
|
|
check whether name is with a legal format and check whether the test file exists.
|
|
"""
|
|
name = name.strip()
|
|
assert re.compile("^test_[0-9a-zA-Z_]+").search(name), (
|
|
"""If line is not the header of table, the test name must begin with "test_" """
|
|
"""and the following substring must include at least one char of "0-9", "a-z", "A-Z" or "_"."""
|
|
)
|
|
filepath_prefix = os.path.join(curdir, name)
|
|
suffix = [".py", ".sh"]
|
|
assert _file_with_extension(filepath_prefix, suffix), (
|
|
f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'"""
|
|
)
|
|
|
|
return name
|
|
|
|
|
|
def _norm_dirs(dirs):
|
|
# reform all dirs' path as normal absolute path
|
|
# abspath() can automatically normalize the path format
|
|
norm_dirs = []
|
|
for d in dirs:
|
|
d = os.path.abspath(d)
|
|
if d not in norm_dirs:
|
|
norm_dirs.append(d)
|
|
return norm_dirs
|
|
|
|
|
|
def _process_run_type(run_type):
|
|
rt = run_type.strip()
|
|
# completely match one of the strings: 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'HYBRID', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'
|
|
assert re.compile(
|
|
"^(NIGHTLY|EXCLUSIVE|CINN|DIST|HYBRID|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$"
|
|
).search(rt), (
|
|
f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'HYBRID', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'"""
|
|
f"""but the run_type is {rt}"""
|
|
)
|
|
return rt
|
|
|
|
|
|
class DistUTPortManager:
|
|
def __init__(self, ignore_dirs=[]):
|
|
self.dist_ut_port = 21200
|
|
self.assigned_ports = {}
|
|
self.last_test_name = ""
|
|
self.last_test_cmake_file = ""
|
|
self.no_cmake_dirs = []
|
|
self.processed_dirs = set()
|
|
self.ignore_dirs = _norm_dirs(ignore_dirs)
|
|
|
|
def reset_current_port(self, port=None):
|
|
self.dist_ut_port = 21200 if port is None else port
|
|
|
|
def get_current_port(self):
|
|
return self.dist_ut_port
|
|
|
|
def get_set_port(self, test_name, port):
|
|
'''
|
|
Get and set a port for unit test named test_name. If the test has been already holding a port, return the port it holds.
|
|
Else assign the input port as a new port to the test.
|
|
'''
|
|
if test_name not in self.assigned_ports:
|
|
self.assigned_ports[test_name] = port
|
|
self.dist_ut_port = max(
|
|
self.dist_ut_port, self.assigned_ports[test_name]
|
|
)
|
|
return self.assigned_ports[test_name]
|
|
|
|
def process_dist_port_num(self, port_num):
|
|
assert (
|
|
re.compile("^[0-9]+$").search(port_num)
|
|
and int(port_num) > 0
|
|
or port_num.strip() == ""
|
|
), (
|
|
f"""port_num must be format as a positive integer or empty, but this port_num is '{port_num}'"""
|
|
)
|
|
port_num = port_num.strip()
|
|
if len(port_num) == 0:
|
|
return 0
|
|
port = self.dist_ut_port
|
|
assert port < 23000, "dist port is exhausted"
|
|
self.dist_ut_port += int(port_num)
|
|
return port
|
|
|
|
def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name):
|
|
'''
|
|
Desc:
|
|
Find all signed ut ports in cmake_file and update the ASSIGNED_PORTS
|
|
and keep the DIST_UT_PORT max of all assigned ports
|
|
'''
|
|
with open(cmake_file_name) as cmake_file:
|
|
# match lines including 'PADDLE_DIST_UT_PORT=' followed by a number
|
|
port_reg = re.compile("PADDLE_DIST_UT_PORT=[0-9]+")
|
|
lines = cmake_file.readlines()
|
|
for idx, line in enumerate(lines):
|
|
matched = port_reg.search(line)
|
|
if matched is None:
|
|
continue
|
|
p = matched.span()
|
|
port = int(line[p[0] : p[1]].split("=")[-1])
|
|
|
|
# find the test name which the port belongs to
|
|
for k in range(idx, 0, -1):
|
|
if lines[k].strip() == "START_BASH":
|
|
break
|
|
name = lines[k - 1].strip()
|
|
|
|
# match right tests name format, the name must start with 'test_' followed by at least one char of
|
|
# '0-9'. 'a-z'. 'A-Z' or '_'
|
|
assert re.compile(
|
|
"^test_[0-9a-zA-Z_]+"
|
|
).search(
|
|
name
|
|
), f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong
|
|
at line {k - 1}, in file {cmake_file_name}
|
|
'''
|
|
self.get_set_port(name, port)
|
|
|
|
# get the test_name which latest assigned port belongs to
|
|
if self.assigned_ports[name] == self.dist_ut_port:
|
|
self.last_test_name = name
|
|
self.last_test_cmake_file = cmake_file_name
|
|
|
|
def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0):
|
|
'''
|
|
Desc:
|
|
get all assigned dist ports to keep port of unmodified test fixed.
|
|
'''
|
|
if current_work_dir in self.processed_dirs:
|
|
return
|
|
|
|
# if root(depth==0)
|
|
if depth == 0:
|
|
self.processed_dirs.clear()
|
|
|
|
self.processed_dirs.add(current_work_dir)
|
|
contents = os.listdir(current_work_dir)
|
|
cmake_file = os.path.join(current_work_dir, "CMakeLists.txt")
|
|
csv = cmake_file.replace("CMakeLists.txt", 'testslist.csv')
|
|
|
|
if os.path.isfile(csv) or os.path.isfile(cmake_file):
|
|
if current_work_dir not in self.ignore_dirs:
|
|
if os.path.isfile(cmake_file) and os.path.isfile(csv):
|
|
self._init_dist_ut_ports_from_cmakefile(cmake_file)
|
|
elif not os.path.isfile(cmake_file):
|
|
# put the directory which has csv but no cmake into NO_CMAKE_DIR_WARNING
|
|
self.no_cmake_dirs.append(current_work_dir)
|
|
|
|
# recursively process the subdirectories
|
|
for c in contents:
|
|
c_path = os.path.join(current_work_dir, c)
|
|
if os.path.isdir(c_path):
|
|
self.parse_assigned_dist_ut_ports(c_path, depth + 1)
|
|
|
|
if depth == 0:
|
|
# After all directories are scanned and processed
|
|
# 1. Get the num_port of last added test and set DIST_UT_PORT+=num_port
|
|
# to guarantee the DIST_UT_PORT is not assigned
|
|
# 2. Summary all the directories which include csv but no cmake and show an error
|
|
# if such a directory exists
|
|
|
|
# step 1
|
|
if (
|
|
len(self.last_test_name) > 0
|
|
and len(self.last_test_cmake_file) > 0
|
|
):
|
|
with open(
|
|
self.last_test_cmake_file.replace(
|
|
"CMakeLists.txt", "testslist.csv"
|
|
)
|
|
) as csv_file:
|
|
found = False
|
|
for line in csv_file:
|
|
(
|
|
name,
|
|
_,
|
|
_,
|
|
_,
|
|
_,
|
|
launcher,
|
|
num_port,
|
|
_,
|
|
_,
|
|
_,
|
|
) = line.strip().split(",")
|
|
if name == self.last_test_name:
|
|
found = True
|
|
break
|
|
assert found, (
|
|
f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'"
|
|
)
|
|
if launcher[-2:] == ".sh":
|
|
self.process_dist_port_num(num_port)
|
|
|
|
# step 2
|
|
err_msg = """==================[No Old CMakeLists.txt Error]==================================
|
|
Following directories has no CmakeLists.txt files:
|
|
"""
|
|
for c in self.no_cmake_dirs:
|
|
err_msg += " " + c + "\n"
|
|
err_msg += """
|
|
This may cause the dist ports different with the old version.
|
|
If the directories are newly created or there is no CMakeLists.txt before, or ignore this error, you
|
|
must specify the directories using the args option --ignore-cmake-dirs/-i.
|
|
If you want to keep the dist ports of old tests unchanged, please ensure the old
|
|
version CMakeLists.txt file existing before using the gen_ut_cmakelists tool to
|
|
generate new CmakeLists.txt files.
|
|
====================================================================================
|
|
"""
|
|
assert len(self.no_cmake_dirs) == 0, err_msg
|
|
|
|
|
|
class CMakeGenerator:
|
|
def __init__(self, current_dirs, only_check, ignore_dirs):
|
|
self.processed_dirs = set()
|
|
self.port_manager = DistUTPortManager(ignore_dirs)
|
|
self.current_dirs = _norm_dirs(current_dirs)
|
|
self.modified_or_created_files = []
|
|
self._only_check = only_check
|
|
|
|
def prepare_dist_ut_port(self):
|
|
for c in self._find_root_dirs():
|
|
self.port_manager.parse_assigned_dist_ut_ports(c, depth=0)
|
|
|
|
def parse_csvs(self):
|
|
'''
|
|
parse csv files, return the lists of created or modified files
|
|
'''
|
|
self.modified_or_created_files = []
|
|
for c in self.current_dirs:
|
|
self._gen_cmakelists(c)
|
|
return self.modified_or_created_files
|
|
|
|
def _find_root_dirs(self):
|
|
root_dirs = []
|
|
# for each current directory, find its highest ancient directory (at least itself)
|
|
# which includes CMakeLists.txt or testslist.csv.txt in the file system tree
|
|
for c in self.current_dirs:
|
|
while True:
|
|
ppath = os.path.dirname(c)
|
|
if ppath == c:
|
|
break
|
|
cmake = os.path.join(ppath, "CMakeLists.txt")
|
|
csv = os.path.join(ppath, "testslist.csv.txt")
|
|
if not (os.path.isfile(cmake) or os.path.isfile(csv)):
|
|
break
|
|
c = ppath
|
|
if c not in root_dirs:
|
|
root_dirs.append(c)
|
|
return root_dirs
|
|
|
|
def _parse_line(self, line, curdir):
|
|
"""
|
|
Desc:
|
|
Input a line in csv file and output a string in cmake grammar, adding the specified test and setting its properties.
|
|
Example:
|
|
Input: "test_allreduce,linux,gpu;rocm,120,DIST,test_runner.py,20071,1,PYTHONPATH=..;http_proxy=;https_proxy=,"
|
|
Output:
|
|
"if((WITH_GPU OR WITH_ROCM) AND (LINUX) )
|
|
py_test_modules(
|
|
test_allreduce
|
|
MODULES
|
|
test_allreduce
|
|
ENVS
|
|
"PADDLE_DIST_UT_PORT=20071;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python;http_proxy=;https_proxy=")
|
|
set_tests_properties(test_allreduce PROPERTIES TIMEOUT "120" RUN_SERIAL 1)
|
|
endif()"
|
|
"""
|
|
|
|
(
|
|
name,
|
|
os_,
|
|
archs,
|
|
timeout,
|
|
run_type,
|
|
launcher,
|
|
num_port,
|
|
run_serial,
|
|
envs,
|
|
conditions,
|
|
) = line.strip().split(",")
|
|
|
|
# name == "name" means the line being parsed is the header of the table
|
|
# we should skip this line and return empty here.
|
|
if name == "name":
|
|
return ""
|
|
name = _process_name(name, curdir)
|
|
|
|
envs = _process_envs(envs)
|
|
conditions = _process_conditions(conditions)
|
|
archs = _process_archs(archs)
|
|
os_ = _process_os(os_)
|
|
run_serial = _process_run_serial(run_serial)
|
|
|
|
cmd = ""
|
|
|
|
for c in conditions:
|
|
cmd += f"if ({c})\n"
|
|
|
|
time_out_str = (
|
|
f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else ''
|
|
)
|
|
|
|
if launcher[-3:] == ".sh":
|
|
run_type = _process_run_type(run_type)
|
|
dist_ut_port = self.port_manager.process_dist_port_num(num_port)
|
|
dist_ut_port = self.port_manager.get_set_port(name, dist_ut_port)
|
|
cmd += f'''if({archs} AND {os_})
|
|
bash_test_modules(
|
|
{name}
|
|
START_BASH
|
|
{launcher}
|
|
{time_out_str}
|
|
LABELS
|
|
"RUN_TYPE={run_type}"
|
|
ENVS
|
|
"PADDLE_DIST_UT_PORT={dist_ut_port};{envs}")%s
|
|
endif()
|
|
'''
|
|
run_type_str = ""
|
|
else:
|
|
try:
|
|
run_type = _process_run_type(run_type)
|
|
except Exception as e:
|
|
assert run_type.strip() == "", (
|
|
f"{e}\nIf use test_runner.py, the run_type can be ''"
|
|
)
|
|
cmd += f'''if({archs} AND {os_})
|
|
py_test_modules(
|
|
{name}
|
|
MODULES
|
|
{name}
|
|
ENVS
|
|
"{envs}")%s
|
|
endif()
|
|
'''
|
|
run_type_str = (
|
|
"" if len(run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"'
|
|
)
|
|
run_serial_str = (
|
|
f' RUN_SERIAL {run_serial}' if len(run_serial) > 0 else ''
|
|
)
|
|
if (
|
|
len(time_out_str) > 0
|
|
or len(run_serial_str) > 0
|
|
or len(run_type_str) > 0
|
|
):
|
|
set_properties = f'''
|
|
set_tests_properties({name} PROPERTIES{time_out_str}{run_serial_str}{run_type_str})'''
|
|
else:
|
|
set_properties = ""
|
|
cmd = cmd % set_properties
|
|
for _ in conditions:
|
|
cmd += "endif()\n"
|
|
return cmd
|
|
|
|
def _gen_cmakelists(self, current_work_dir, depth=0):
|
|
if depth == 0:
|
|
self.processed_dirs.clear()
|
|
if current_work_dir == "":
|
|
current_work_dir = "."
|
|
|
|
contents = os.listdir(current_work_dir)
|
|
contents.sort()
|
|
sub_dirs = []
|
|
for c in contents:
|
|
c_path = os.path.join(current_work_dir, c)
|
|
if c_path in self.processed_dirs:
|
|
return
|
|
if not os.path.isdir(c_path):
|
|
continue
|
|
self.processed_dirs.add(c_path)
|
|
if os.path.isfile(
|
|
os.path.join(current_work_dir, c, "testslist.csv")
|
|
) or os.path.isfile(
|
|
os.path.join(current_work_dir, c, "CMakeLists.txt")
|
|
):
|
|
self._gen_cmakelists(
|
|
os.path.join(current_work_dir, c), depth + 1
|
|
)
|
|
sub_dirs.append(c)
|
|
|
|
if not os.path.isfile(os.path.join(current_work_dir, "testslist.csv")):
|
|
return
|
|
cmds = """# This file is generated by ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py.
|
|
# Please don't modify this file manually.
|
|
# If you need to change unittests in this file, please modify testslist.csv in the current directory
|
|
# and then run the command `python3 ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py -f ${CURRENT_DIRECTORY}/testslist.csv`
|
|
set(LOCAL_ALL_ARCH ON)
|
|
set(LOCAL_ALL_PLAT ON)\n"""
|
|
with open(f"{current_work_dir}/testslist.csv") as csv_file:
|
|
for i, line in enumerate(csv_file.readlines()):
|
|
try:
|
|
cmds += self._parse_line(line, current_work_dir)
|
|
except Exception as e:
|
|
print("===============PARSE LINE ERRORS OCCUR==========")
|
|
print(e)
|
|
print(f"[ERROR FILE]: {current_work_dir}/testslist.csv")
|
|
print(f"[ERROR LINE {i + 1}]: {line.strip()}")
|
|
sys.exit(1)
|
|
|
|
for sub in sub_dirs:
|
|
cmds += f"add_subdirectory({sub})\n"
|
|
|
|
# check whether the generated file are the same with the existing file, ignoring the blank chars
|
|
# if they are same, skip the waiting process
|
|
if os.path.isfile(f"{current_work_dir}/CMakeLists.txt"):
|
|
with open(
|
|
f"{current_work_dir}/CMakeLists.txt", "r"
|
|
) as old_cmake_file:
|
|
char_seq = old_cmake_file.read().split()
|
|
else:
|
|
char_seq = []
|
|
char_seq = "".join(char_seq)
|
|
|
|
if char_seq != "".join(cmds.split()):
|
|
assert (
|
|
f"{current_work_dir}/CMakeLists.txt"
|
|
not in self.modified_or_created_files
|
|
), (
|
|
f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error"
|
|
)
|
|
self.modified_or_created_files.append(
|
|
f"{current_work_dir}/CMakeLists.txt"
|
|
)
|
|
if not self._only_check:
|
|
with open(
|
|
f"{current_work_dir}/CMakeLists.txt", "w"
|
|
) as cmake_file:
|
|
print(cmds, end="", file=cmake_file)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--files",
|
|
"-f",
|
|
type=str,
|
|
required=False,
|
|
default=[],
|
|
nargs="+",
|
|
help="Input a list of files named testslist.csv and output files named CMakeLists.txt in the same directories as the csv files respectively",
|
|
)
|
|
parser.add_argument(
|
|
"--dirpaths",
|
|
"-d",
|
|
type=str,
|
|
required=False,
|
|
default=[],
|
|
nargs="+",
|
|
help="Input a list of dir paths including files named testslist.csv and output CMakeLists.txt in these directories respectively",
|
|
)
|
|
parser.add_argument(
|
|
"--ignore-cmake-dirs",
|
|
'-i',
|
|
type=str,
|
|
required=False,
|
|
default=[],
|
|
nargs='*',
|
|
help="To keep dist ports the same with old version cmake, old CMakeLists.txt files are needed to parse dist_ports. If a directories are newly created and there is no CMakeLists.txt file, the directory path must be specified by this option. The dirs are not recursive.",
|
|
)
|
|
parser.add_argument(
|
|
"--only-check-changed",
|
|
'-o',
|
|
type=lambda x: x.lower() not in ["false", "0", "off"],
|
|
required=False,
|
|
default=False,
|
|
help="Only check whether the CMake files should be rewritten, do not write it even if it should be write",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
assert not (len(args.files) == 0 and len(args.dirpaths) == 0), (
|
|
"You must provide at least one file or dirpath"
|
|
)
|
|
current_work_dirs = []
|
|
if len(args.files) >= 1:
|
|
for p in args.files:
|
|
assert os.path.basename(p) == "testslist.csv", (
|
|
"you must input file named testslist.csv"
|
|
)
|
|
current_work_dirs = current_work_dirs + [
|
|
os.path.dirname(file) for file in args.files
|
|
]
|
|
if len(args.dirpaths) >= 1:
|
|
current_work_dirs = current_work_dirs + list(args.dirpaths)
|
|
|
|
cmake_generator = CMakeGenerator(
|
|
current_work_dirs, args.only_check_changed, args.ignore_cmake_dirs
|
|
)
|
|
cmake_generator.prepare_dist_ut_port()
|
|
created = cmake_generator.parse_csvs()
|
|
|
|
# summary the modified files
|
|
for f in created:
|
|
print("modified/new:", f)
|