paddlepaddle--paddle/tools/sampcd_processor.py

# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
please make sure to run in the tools path
usage: python sampcd_processor.py --mode {cpu or gpu}
    {cpu or gpu}: running in cpu version or gpu version

for example, you can run cpu version testing like this:

    python sampcd_processor.py --mode cpu

"""

from __future__ import annotations

import collections
import functools
import multiprocessing
import os
import platform
import queue
import re
import threading
import time
import typing

import xdoctest
from sampcd_processor_utils import (
    TEST_TIMEOUT,
    DocTester,
    TestResult,
    log_exit,
    logger,
    parse_args,
    run_doctest,
)

XDOCTEST_CONFIG = {
    "global_exec": r"\n".join(
        [
            "import paddle",
            "paddle.device.set_device('cpu')",
            "paddle.set_default_dtype('float32')",
            "paddle.disable_static()",
        ]
    ),
    "default_runtime_state": {"IGNORE_WHITESPACE": True},
}


def _patch_global_state(debug, verbose):
    # patch xdoctest global_state
    from xdoctest import global_state

    _debug_xdoctest = debug and verbose > 2
    global_state.DEBUG = _debug_xdoctest
    global_state.DEBUG_PARSER = global_state.DEBUG_PARSER and _debug_xdoctest
    global_state.DEBUG_CORE = global_state.DEBUG_CORE and _debug_xdoctest
    global_state.DEBUG_RUNNER = global_state.DEBUG_RUNNER and _debug_xdoctest
    global_state.DEBUG_DOCTEST = global_state.DEBUG_DOCTEST and _debug_xdoctest


def _patch_tensor_place():
    from xdoctest import checker

    pattern_tensor = re.compile(
        r"""
        (Tensor\(.*?place=)     # Tensor start
        (.*?)                   # Place=(XXX)
        (\,.*?\))
        """,
        re.VERBOSE | re.DOTALL,
    )

    _check_output = checker.check_output

    def check_output(got, want, runstate=None):
        if not want:  # nocover
            return True

        return _check_output(
            got=pattern_tensor.sub(r'\1Place(cpu)\3', got),
            want=pattern_tensor.sub(r'\1Place(cpu)\3', want),
            runstate=runstate,
        )

    checker.check_output = check_output


def _patch_float_precision(digits):
    from xdoctest import checker

    pattern_number = re.compile(
        r"""
        (?:
            (?:(?<=[\s*\[\(\'\"\:])|^)                  # number starts
            (?:                                         # int/float or complex-real
                (?:
                    [+-]?
                    (?:
                        (?: \d*\.\d+) | (?: \d+\.?)     # int/float
                    )
                )
                (?:[Ee][+-]?\d+)?
            )
            (?:                                         # complex-imag
                (?:
                    (?:
                        [+-]?
                        (?:
                            (?: \d*\.\d+) | (?: \d+\.?)
                        )
                    )
                    (?:[Ee][+-]?\d+)?
                )
            (?:[Jj])
            )?
        )
        """,
        re.VERBOSE | re.DOTALL,
    )

    _check_output = checker.check_output

    def _sub_number(match_obj, digits):
        match_str = match_obj.group()

        if 'j' in match_str or 'J' in match_str:
            try:
                match_num = complex(match_str)
            except ValueError:
                return match_str

            return (
                str(
                    complex(
                        round(match_num.real, digits),
                        round(match_num.imag, digits),
                    )
                )
                .strip('(')
                .strip(')')
            )
        else:
            try:
                return str(round(float(match_str), digits))
            except ValueError:
                return match_str

    sub_number = functools.partial(_sub_number, digits=digits)

    def check_output(got, want, runstate=None):
        if not want:  # nocover
            return True

        return _check_output(
            got=pattern_number.sub(sub_number, got),
            want=pattern_number.sub(sub_number, want),
            runstate=runstate,
        )

    checker.check_output = check_output


class Directive:
    """Base class of global directives just for `xdoctest`."""

    pattern: typing.Pattern

    def parse_directive(self, docstring: str) -> tuple[str, typing.Any]:
        pass


class TimeoutDirective(Directive):
    pattern = re.compile(
        r"""
        (?:
            (?:
                \s*\>{3}\s*\#\s*x?doctest\:\s*
            )
            (?P<op>[\+\-])
            (?:
                TIMEOUT
            )
            \(
                (?P<time>\d+)
            \)
            (?:
                \s*?
            )
        )
        """,
        re.VERBOSE | re.DOTALL,
    )

    def __init__(self, timeout):
        self._timeout = timeout

    def parse_directive(self, docstring):
        match_obj = self.pattern.search(docstring)
        if match_obj is not None:
            op_time = match_obj.group('time')
            match_start = match_obj.start()
            match_end = match_obj.end()

            return (
                (docstring[:match_start] + '\n' + docstring[match_end:]),
                float(op_time),
            )

        return docstring, float(self._timeout)


class SingleProcessDirective(Directive):
    pattern = re.compile(
        r"""
        (?:
            (?:
                \s*\>{3}\s*\#\s*x?doctest\:\s*
            )
            (?P<op>[\+\-])
            (?:
                SOLO
            )
            (?:
                (?P<reason>.*?)
            )
            \s
        )
        """,
        re.VERBOSE | re.DOTALL,
    )

    def parse_directive(self, docstring):
        match_obj = self.pattern.search(docstring)
        if match_obj is not None:
            op_reason = match_obj.group('reason')
            match_start = match_obj.start()
            match_end = match_obj.end()

            return (
                (docstring[:match_start] + '\n' + docstring[match_end:]),
                op_reason,
            )

        return docstring, None


class BadStatement:
    msg: str = ''

    def check(self, docstring: str) -> bool:
        """Return `True` for bad statement detected."""
        raise NotImplementedError


class Fluid(BadStatement):
    msg = 'Please do NOT use `fluid` api.'

    _pattern = re.compile(
        r"""
        (\>{3}|\.{3})
        (?P<comment>.*)
        import
        .*
        (\bfluid\b)
        """,
        re.VERBOSE,
    )

    def check(self, docstring):
        for match_obj in self._pattern.finditer(docstring):
            comment = match_obj.group('comment').strip()
            if not comment.startswith('#'):
                return True

        return False


class SkipNoReason(BadStatement):
    msg = 'Please add sample code skip reason.'

    _pattern = re.compile(
        r"""
        \#
        \s*
        (x?doctest:)
        \s*
        [+]SKIP
        (?P<reason>.*)
        """,
        re.VERBOSE,
    )

    def check(self, docstring):
        for match_obj in self._pattern.finditer(docstring):
            reason = (
                match_obj.group('reason').strip().strip('(').strip(')').strip()
            )
            if not reason:
                return True

        return False


class DeprecatedRequired(BadStatement):
    msg = 'Please use `# doctest: +REQUIRES({})` instead of `# {} {}`.'

    _pattern = re.compile(
        r"""
        \#
        \s*
        (?P<directive>require[sd]?\s*:)
        (?P<env>.+)
        """,
        re.VERBOSE,
    )

    def check(self, docstring):
        for match_obj in self._pattern.finditer(docstring):
            dep_directive = match_obj.group('directive').strip()
            dep_env = match_obj.group('env').strip()

            if dep_env:
                env = 'env:' + ', env:'.join(
                    [e.strip().upper() for e in dep_env.split(',') if e.strip()]
                )
                self.msg = self.__class__.msg.format(
                    env, dep_directive, dep_env
                )
                return True

        return False


class Xdoctester(DocTester):
    """A Xdoctest doctester."""

    directives: dict[str, tuple[type[Directive], ...]] = {
        'timeout': (TimeoutDirective, TEST_TIMEOUT),
        'solo': (SingleProcessDirective,),
    }

    bad_statements: dict[str, tuple[type[BadStatement], ...]] = {
        'fluid': (Fluid,),
        'skip': (SkipNoReason,),
        'require': (DeprecatedRequired,),
    }

    def __init__(
        self,
        debug=False,
        style='freeform',
        target='codeblock',
        mode='native',
        verbose=2,
        patch_global_state=True,
        patch_tensor_place=True,
        patch_float_precision=5,
        use_multiprocessing=True,
        **config,
    ):
        self.debug = debug

        self.style = style
        self.target = target
        self.mode = mode
        self.verbose = verbose
        self.config = XDOCTEST_CONFIG | (config or {})
        self._test_capacity = set()

        self._patch_global_state = patch_global_state
        self._patch_tensor_place = patch_tensor_place
        self._patch_float_precision = patch_float_precision
        self._use_multiprocessing = use_multiprocessing

        # patch xdoctest before `xdoctest.core.parse_docstr_examples`
        self._patch_xdoctest()

        self.docstring_parser = functools.partial(
            xdoctest.core.parse_docstr_examples, style=self.style
        )

        self.directive_pattern = re.compile(
            r"""
            (?<=(\#\s))     # positive lookbehind, directive begins
            (doctest)       # directive prefix, which should be replaced
            (?=(:\s*.*\n))  # positive lookahead, directive content
            """,
            re.VERBOSE,
        )

        self.directive_prefix = 'xdoctest'

    def _patch_xdoctest(self):
        if self._patch_global_state:
            _patch_global_state(self.debug, self.verbose)

        if self._patch_tensor_place:
            _patch_tensor_place()

        if self._patch_float_precision is not None:
            _patch_float_precision(self._patch_float_precision)

    def _parse_directive(
        self, docstring: str
    ) -> tuple[str, dict[str, Directive]]:
        directives = {}
        for name, directive_cls in self.directives.items():
            docstring, direct = directive_cls[0](
                *directive_cls[1:]
            ).parse_directive(docstring)
            directives[name] = direct

        return docstring, directives

    def convert_directive(self, docstring: str) -> str:
        """Replace directive prefix with xdoctest"""
        return self.directive_pattern.sub(self.directive_prefix, docstring)

    def prepare(self, test_capacity: set):
        """Set environs for xdoctest directive.
        The keys in environs, which also used in `# xdoctest: +REQUIRES(env:XX)`, should be UPPER case.

        If `test_capacity = {"cpu"}`, then we set:

            - `os.environ["CPU"] = "True"`

        which makes this SKIPPED:

            - # xdoctest: +REQUIRES(env:GPU)

        If `test_capacity = {"cpu", "gpu"}`, then we set:

            - `os.environ["CPU"] = "True"`
            - `os.environ["GPU"] = "True"`

        which makes this SUCCESS:

            - # xdoctest: +REQUIRES(env:GPU)
        """
        logger.info("Set xdoctest environ ...")
        for capacity in test_capacity:
            key = capacity.upper()
            os.environ[key] = "True"
            logger.info("Environ: %s , set to True.", key)

        logger.info("API check using Xdoctest prepared!-- Example Code")
        logger.info("running under python %s", platform.python_version())
        logger.info("running under xdoctest %s", xdoctest.__version__)

        self._test_capacity = test_capacity

    def _check_bad_statements(self, docstring: str) -> set[BadStatement]:
        bad_results = set()
        for _, statement_cls in self.bad_statements.items():
            bad_statement = statement_cls[0](*statement_cls[1:])
            if bad_statement.check(docstring):
                bad_results.add(bad_statement)

        return bad_results

    def run(self, api_name: str, docstring: str) -> list[TestResult]:
        """Run the xdoctest with a docstring."""
        # check bad statements
        bad_results = self._check_bad_statements(docstring)
        if bad_results:
            for bad_statement in bad_results:
                logger.warning("%s >>> %s", api_name, bad_statement.msg)

            return [
                TestResult(
                    name=api_name,
                    badstatement=True,
                )
            ]

        # parse global directive
        docstring, directives = self._parse_directive(docstring)

        # extract xdoctest examples
        examples_to_test, examples_nocode = self._extract_examples(
            api_name, docstring, **directives
        )

        # run xdoctest
        try:
            result = self._execute_xdoctest(
                examples_to_test, examples_nocode, **directives
            )
        except queue.Empty:
            result = [
                TestResult(
                    name=api_name,
                    timeout=True,
                    time=directives.get('timeout', TEST_TIMEOUT),
                )
            ]

        return result

    def _extract_examples(self, api_name, docstring, **directives):
        """Extract code block examples from docstring."""
        examples_to_test = {}
        examples_nocode = {}
        for example_idx, example in enumerate(
            self.docstring_parser(docstr=docstring, callname=api_name)
        ):
            example.mode = self.mode
            example.config.update(self.config)
            example_key = f"{api_name}_{example_idx}"

            # check whether there are some parts parsed by xdoctest
            if not example._parts:
                examples_nocode[example_key] = example
                continue

            examples_to_test[example_key] = example

        if not examples_nocode and not examples_to_test:
            examples_nocode[api_name] = api_name

        return examples_to_test, examples_nocode

    def _execute_xdoctest(
        self, examples_to_test, examples_nocode, **directives
    ):
        # if use solo(single process), execute without multiprocessing/thread
        if directives.get('solo') is not None:
            return self._execute(examples_to_test, examples_nocode)

        if self._use_multiprocessing:
            _ctx = multiprocessing.get_context('spawn')
            result_queue = _ctx.Queue()
            exec_processor = functools.partial(_ctx.Process, daemon=True)
        else:
            result_queue = queue.Queue()
            exec_processor = functools.partial(threading.Thread, daemon=True)

        processor = exec_processor(
            target=self._execute_with_queue,
            args=(
                result_queue,
                examples_to_test,
                examples_nocode,
            ),
        )

        processor.start()
        result = result_queue.get(
            timeout=directives.get('timeout', TEST_TIMEOUT)
        )
        processor.join()

        return result

    def _execute(self, examples_to_test, examples_nocode):
        """Run xdoctest for each example"""
        # patch xdoctest first in each process
        self._patch_xdoctest()

        # run the xdoctest
        test_results = []
        for _, example in examples_to_test.items():
            start_time = time.time()
            result = example.run(verbose=self.verbose, on_error='return')
            end_time = time.time()

            test_results.append(
                TestResult(
                    name=str(example),
                    passed=result['passed'],
                    skipped=result['skipped'],
                    failed=result['failed'],
                    test_msg=str(result['exc_info']),
                    time=end_time - start_time,
                )
            )

        for _, example in examples_nocode.items():
            test_results.append(TestResult(name=str(example), nocode=True))

        return test_results

    def _execute_with_queue(self, queue, examples_to_test, examples_nocode):
        queue.put(self._execute(examples_to_test, examples_nocode))

    def print_summary(self, test_results, whl_error=None):
        summary = collections.defaultdict(list)
        is_fail = False

        logger.warning("----------------Check results--------------------")
        logger.warning(">>> Sample code test capacity: %s", self._test_capacity)

        if whl_error is not None and whl_error:
            logger.warning("%s is not in whl.", whl_error)
            logger.warning("")
            logger.warning("Please check the whl package and API_PR.spec!")
            logger.warning(
                "You can follow these steps in order to generate API.spec:"
            )
            logger.warning("1. cd ${paddle_path}, compile paddle;")
            logger.warning(
                "2. pip install build/python/dist/(build whl package);"
            )
            logger.warning(
                "3. run 'python tools/print_signatures.py paddle > paddle/fluid/API.spec'."
            )
            for test_result in test_results:
                if test_result.failed:
                    logger.error(
                        "In addition, mistakes found in sample codes: %s",
                        test_result.name,
                    )
            log_exit(1)

        else:
            for test_result in test_results:
                summary[test_result.state].append(test_result)
                if test_result.state.is_fail:
                    is_fail = True

            summary = sorted(summary.items(), key=lambda x: x[0].order)

            for result_cls, result_list in summary:
                logging_msg = result_cls.msg(
                    len(result_list), self._test_capacity
                )
                result_cls.logger(logging_msg)
                result_cls.logger('\n'.join([str(r) for r in result_list]))

            if is_fail:
                logger.warning(
                    ">>> Mistakes found in sample codes in env: %s!",
                    self._test_capacity,
                )
                logger.warning(">>> Please recheck the sample codes.")
                log_exit(1)

        logger.warning(
            ">>> Sample code check is successful in env: %s!",
            self._test_capacity,
        )
        logger.warning("----------------End of the Check--------------------")


if __name__ == '__main__':
    args = parse_args()
    run_doctest(args, doctester=Xdoctester(debug=args.debug))