wehub-resource-sync
e768098d0e
Flake8 Lint / flake8 (push) Waiting to run
Spell check CI / Spell_Check (push) Waiting to run
tools_continuous_delivery / Private PyPI non-main branch release (push) Has been skipped
tools_continuous_delivery / Private PyPI main branch release (push) Failing after 2m42s
Publish Promptflow Doc / Build (push) Has been cancelled
Publish Promptflow Doc / Deploy (push) Has been cancelled
129 lines
4.4 KiB
Python
129 lines
4.4 KiB
Python
import argparse
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from utils import Color, run_command, print_red
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=Color.RED + "Test Coverage for Promptflow!" + Color.END + "\n")
|
|
|
|
parser.add_argument("-p", required=True, nargs="+", help="The paths to calculate code coverage")
|
|
parser.add_argument("-t", required=True, nargs="+", help="The path to the tests")
|
|
parser.add_argument("-l", required=True, help="Location to run tests in")
|
|
parser.add_argument(
|
|
"-m",
|
|
required=True,
|
|
help="Pytest marker to identify the tests to run",
|
|
default="all",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
required=False,
|
|
help="Pytest output file name",
|
|
default="test-results.xml",
|
|
)
|
|
parser.add_argument("-n", help="Pytest number of process to run the tests", default="auto")
|
|
parser.add_argument(
|
|
"--model-name",
|
|
help="The model file name to run the tests",
|
|
type=str,
|
|
default="",
|
|
)
|
|
parser.add_argument("--timeout", help="Timeout for individual tests (seconds)", type=str, default="")
|
|
parser.add_argument(
|
|
"--coverage-config",
|
|
help="The path of code coverage config file",
|
|
type=str,
|
|
default="",
|
|
)
|
|
parser.add_argument(
|
|
"--disable-cov-branch",
|
|
action="store_true",
|
|
help="Whether to enable branch coverage calculation",
|
|
)
|
|
parser.add_argument(
|
|
"--ignore-glob",
|
|
help="The path of ignored test file",
|
|
type=str,
|
|
default="",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
print("Working directory: " + str(os.getcwd()))
|
|
print("Args.p: " + str(args.p))
|
|
print("Args.t: " + str(args.t))
|
|
print("Args.l: " + str(args.l))
|
|
print("Args.m: " + str(args.m))
|
|
print("Args.n: " + str(args.n))
|
|
print("Args.o: " + str(args.o))
|
|
print("Args.model-name: " + str(args.model_name))
|
|
print("Args.timeout: " + str(args.timeout))
|
|
print("Args.coverage-config: " + str(args.coverage_config))
|
|
print("Args.ignore-glob: " + str(args.ignore_glob))
|
|
print("Args.disable-cov-branch: " + str(args.disable_cov_branch))
|
|
|
|
test_paths_list = [str(Path(path).absolute()) for path in args.t]
|
|
|
|
# display a list of all Python packages installed in the current Python environment
|
|
run_command(["pip", "list"])
|
|
run_command(["pip", "show", "promptflow", "promptflow-sdk"])
|
|
|
|
pytest_command = ["pytest", f"--junitxml={args.o}"]
|
|
pytest_command += test_paths_list
|
|
if args.coverage_config:
|
|
if args.p:
|
|
cov_path_list = [f"--cov={path}" for path in args.p]
|
|
pytest_command += cov_path_list
|
|
if not args.disable_cov_branch:
|
|
pytest_command += ["--cov-branch"]
|
|
pytest_command += [ # noqa: W503
|
|
"--cov-report=term",
|
|
"--cov-report=html",
|
|
"--cov-report=xml",
|
|
]
|
|
pytest_command = pytest_command + [f"--cov-config={args.coverage_config}"]
|
|
if args.ignore_glob:
|
|
pytest_command = pytest_command + [f"--ignore-glob={args.ignore_glob}"]
|
|
pytest_command += [
|
|
"-n",
|
|
args.n,
|
|
"--dist",
|
|
"loadfile",
|
|
"--log-level=info",
|
|
"--log-format=%(asctime)s %(levelname)s %(message)s",
|
|
"--log-date-format=[%Y-%m-%d %H:%M:%S]",
|
|
"--durations=5",
|
|
"-ra",
|
|
"-vv",
|
|
]
|
|
|
|
if args.timeout:
|
|
pytest_command = pytest_command + [
|
|
"--timeout",
|
|
args.timeout,
|
|
"--timeout_method",
|
|
"thread",
|
|
]
|
|
|
|
if args.m != "all":
|
|
pytest_command = pytest_command + ["-m", args.m]
|
|
|
|
if args.model_name:
|
|
pytest_command = pytest_command + ["--model-name", args.model_name]
|
|
|
|
# pytest --junit-xml=test-results.xml --cov=azure.ai.ml --cov-report=html --cov-report=xml -ra ./tests/*/unittests/
|
|
error_code, _ = run_command(pytest_command, throw_on_retcode=False)
|
|
# https://docs.pytest.org/en/7.1.x/reference/exit-codes.html
|
|
if error_code == 1:
|
|
print_red("Tests were collected and run but some of the tests failed.")
|
|
elif error_code == 2:
|
|
print_red("Test execution was interrupted by the user.")
|
|
elif error_code == 3:
|
|
print_red("Internal error happened while executing tests.")
|
|
elif error_code == 4:
|
|
print_red("pytest command line usage error.")
|
|
elif error_code == 5:
|
|
print_red("No tests were collected.")
|
|
sys.exit(error_code)
|