chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,562 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Natural Language Toolkit: BLEU Score
|
||||
#
|
||||
# Copyright (C) 2001-2020 NLTK Project
|
||||
# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
|
||||
# Contributors: Björn Mattsson, Dmitrijs Milajevs, Liling Tan
|
||||
# URL: <http://nltk.org/>
|
||||
# For license information, see LICENSE.TXT
|
||||
|
||||
"""BLEU score implementation."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
from fractions import Fraction
|
||||
import warnings
|
||||
from collections import Counter
|
||||
|
||||
from codebleu.utils import ngrams
|
||||
import pdb
|
||||
|
||||
|
||||
def sentence_bleu(
|
||||
references,
|
||||
hypothesis,
|
||||
weights=(0.25, 0.25, 0.25, 0.25),
|
||||
smoothing_function=None,
|
||||
auto_reweigh=False,
|
||||
):
|
||||
"""
|
||||
Calculate BLEU score (Bilingual Evaluation Understudy) from
|
||||
Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002.
|
||||
"BLEU: a method for automatic evaluation of machine translation."
|
||||
In Proceedings of ACL. http://www.aclweb.org/anthology/P02-1040.pdf
|
||||
>>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
|
||||
... 'ensures', 'that', 'the', 'military', 'always',
|
||||
... 'obeys', 'the', 'commands', 'of', 'the', 'party']
|
||||
>>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops',
|
||||
... 'forever', 'hearing', 'the', 'activity', 'guidebook',
|
||||
... 'that', 'party', 'direct']
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
|
||||
... 'ensures', 'that', 'the', 'military', 'will', 'forever',
|
||||
... 'heed', 'Party', 'commands']
|
||||
>>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',
|
||||
... 'guarantees', 'the', 'military', 'forces', 'always',
|
||||
... 'being', 'under', 'the', 'command', 'of', 'the',
|
||||
... 'Party']
|
||||
>>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
|
||||
... 'army', 'always', 'to', 'heed', 'the', 'directions',
|
||||
... 'of', 'the', 'party']
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS
|
||||
0.5045...
|
||||
If there is no ngrams overlap for any order of n-grams, BLEU returns the
|
||||
value 0. This is because the precision for the order of n-grams without
|
||||
overlap is 0, and the geometric mean in the final BLEU score computation
|
||||
multiplies the 0 with the precision of other n-grams. This results in 0
|
||||
(independently of the precision of the othe n-gram orders). The following
|
||||
example has zero 3-gram and 4-gram overlaps:
|
||||
>>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS
|
||||
0.0
|
||||
To avoid this harsh behaviour when no ngram overlaps are found a smoothing
|
||||
function can be used.
|
||||
>>> chencherry = SmoothingFunction()
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis2,
|
||||
... smoothing_function=chencherry.method1) # doctest: +ELLIPSIS
|
||||
0.0370...
|
||||
The default BLEU calculates a score for up to 4-grams using uniform
|
||||
weights (this is called BLEU-4). To evaluate your translations with
|
||||
higher/lower order ngrams, use customized weights. E.g. when accounting
|
||||
for up to 5-grams with uniform weights (this is called BLEU-5) use:
|
||||
>>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.)
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS
|
||||
0.3920...
|
||||
:param references: reference sentences
|
||||
:type references: list(list(str))
|
||||
:param hypothesis: a hypothesis sentence
|
||||
:type hypothesis: list(str)
|
||||
:param weights: weights for unigrams, bigrams, trigrams and so on
|
||||
:type weights: list(float)
|
||||
:param smoothing_function:
|
||||
:type smoothing_function: SmoothingFunction
|
||||
:param auto_reweigh: Option to re-normalize the weights uniformly.
|
||||
:type auto_reweigh: bool
|
||||
:return: The sentence-level BLEU score.
|
||||
:rtype: float
|
||||
"""
|
||||
return corpus_bleu(
|
||||
[references], [hypothesis], weights, smoothing_function, auto_reweigh
|
||||
)
|
||||
|
||||
|
||||
def corpus_bleu(
|
||||
list_of_references,
|
||||
hypotheses,
|
||||
weights=(0.25, 0.25, 0.25, 0.25),
|
||||
smoothing_function=None,
|
||||
auto_reweigh=False,
|
||||
):
|
||||
"""
|
||||
Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all
|
||||
hypotheses and their respective references.
|
||||
|
||||
Instead of averaging the sentence level BLEU scores (i.e. macro-average
|
||||
precision), the original BLEU metric (Papineni et al. 2002) accounts for
|
||||
the micro-average precision (i.e. summing the numerators and denominators
|
||||
for each hypothesis-reference(s) pairs before the division).
|
||||
|
||||
Args:
|
||||
list_of_references (list of list of list of str):
|
||||
A corpus of lists of reference sentences, with respect to hypotheses.
|
||||
hypotheses (list of list of str):
|
||||
A list of hypothesis sentences.
|
||||
weights (tuple of float):
|
||||
Weights for unigrams, bigrams, trigrams, and so on.
|
||||
smoothing_function:
|
||||
Smoothing function for n-grams.
|
||||
auto_reweigh (bool):
|
||||
Option to re-normalize the weights uniformly.
|
||||
|
||||
Returns:
|
||||
float:
|
||||
The corpus-level BLEU score.
|
||||
|
||||
Examples:
|
||||
>>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
|
||||
... 'ensures', 'that', 'the', 'military', 'always',
|
||||
... 'obeys', 'the', 'commands', 'of', 'the', 'party']
|
||||
>>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
|
||||
... 'ensures', 'that', 'the', 'military', 'will', 'forever',
|
||||
... 'heed', 'Party', 'commands']
|
||||
>>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which',
|
||||
... 'guarantees', 'the', 'military', 'forces', 'always',
|
||||
... 'being', 'under', 'the', 'command', 'of', 'the', 'Party']
|
||||
>>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
|
||||
... 'army', 'always', 'to', 'heed', 'the', 'directions',
|
||||
... 'of', 'the', 'party']
|
||||
>>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',
|
||||
... 'interested', 'in', 'world', 'history']
|
||||
>>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',
|
||||
... 'because', 'he', 'read', 'the', 'book']
|
||||
>>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
|
||||
>>> hypotheses = [hyp1, hyp2]
|
||||
>>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS
|
||||
0.5920...
|
||||
"""
|
||||
# Before proceeding to compute BLEU, perform sanity checks.
|
||||
|
||||
p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches.
|
||||
p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref.
|
||||
hyp_lengths, ref_lengths = 0, 0
|
||||
|
||||
assert len(list_of_references) == len(hypotheses), (
|
||||
"The number of hypotheses and their reference(s) should be the " "same "
|
||||
)
|
||||
|
||||
# Iterate through each hypothesis and their corresponding references.
|
||||
for references, hypothesis in zip(list_of_references, hypotheses):
|
||||
# For each order of ngram, calculate the numerator and
|
||||
# denominator for the corpus-level modified precision.
|
||||
for i, _ in enumerate(weights, start=1):
|
||||
p_i = modified_precision(references, hypothesis, i)
|
||||
p_numerators[i] += p_i.numerator
|
||||
p_denominators[i] += p_i.denominator
|
||||
|
||||
# Calculate the hypothesis length and the closest reference length.
|
||||
# Adds them to the corpus-level hypothesis and reference counts.
|
||||
hyp_len = len(hypothesis)
|
||||
hyp_lengths += hyp_len
|
||||
ref_lengths += closest_ref_length(references, hyp_len)
|
||||
|
||||
# Calculate corpus-level brevity penalty.
|
||||
bp = brevity_penalty(ref_lengths, hyp_lengths)
|
||||
|
||||
# Uniformly re-weighting based on maximum hypothesis lengths if largest
|
||||
# order of n-grams < 4 and weights is set at default.
|
||||
if auto_reweigh:
|
||||
if hyp_lengths < 4 and weights == (0.25, 0.25, 0.25, 0.25):
|
||||
weights = (1 / hyp_lengths,) * hyp_lengths
|
||||
|
||||
# Collects the various precision values for the different ngram orders.
|
||||
p_n = [
|
||||
Fraction(p_numerators[i], p_denominators[i], _normalize=False)
|
||||
for i, _ in enumerate(weights, start=1)
|
||||
]
|
||||
|
||||
# Returns 0 if there's no matching n-grams
|
||||
# We only need to check for p_numerators[1] == 0, since if there's
|
||||
# no unigrams, there won't be any higher order ngrams.
|
||||
if p_numerators[1] == 0:
|
||||
return 0
|
||||
|
||||
# If there's no smoothing, set use method0 from SmoothinFunction class.
|
||||
if not smoothing_function:
|
||||
smoothing_function = SmoothingFunction().method1
|
||||
# Smoothen the modified precision.
|
||||
# Note: smoothing_function() may convert values into floats;
|
||||
# it tries to retain the Fraction object as much as the
|
||||
# smoothing method allows.
|
||||
p_n = smoothing_function(
|
||||
p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths
|
||||
)
|
||||
s = (w_i * math.log(p_i) for w_i, p_i in zip(weights, p_n))
|
||||
s = bp * math.exp(math.fsum(s))
|
||||
return s
|
||||
|
||||
|
||||
def modified_precision(references, hypothesis, n):
|
||||
"""
|
||||
Calculate modified ngram precision.
|
||||
|
||||
The normal precision method can sometimes result in incorrect translations with high precision.
|
||||
For instance, when a word in the reference repeats multiple times, it might yield an artificially high precision.
|
||||
|
||||
This function computes the Fraction object that includes the numerator and denominator required to calculate
|
||||
the corpus-level precision. To calculate the modified precision for a single hypothesis-reference pair,
|
||||
you can cast the Fraction object into a float.
|
||||
|
||||
Example 1: Demonstrates the impact of repeating words in the reference.
|
||||
>>> reference1 = 'the cat is on the mat'.split()
|
||||
>>> reference2 = 'there is a cat on the mat'.split()
|
||||
>>> hypothesis1 = 'the the the the the the the'.split()
|
||||
>>> references = [reference1, reference2]
|
||||
>>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS
|
||||
0.2857...
|
||||
|
||||
Example 2: Reference words considered exhausted after a match with the hypothesis.
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', ...]
|
||||
>>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', ...]
|
||||
>>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', ...]
|
||||
>>> hypothesis = 'of the'.split()
|
||||
>>> references = [reference1, reference2, reference3]
|
||||
>>> float(modified_precision(references, hypothesis, n=1))
|
||||
1.0
|
||||
>>> float(modified_precision(references, hypothesis, n=2))
|
||||
1.0
|
||||
|
||||
Example 3: Comparison of BLEU precision for different hypotheses and references.
|
||||
>>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', ...]
|
||||
>>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops', ...]
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', ...]
|
||||
>>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which', ...]
|
||||
>>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the', ...]
|
||||
>>> references = [reference1, reference2, reference3]
|
||||
>>> float(modified_precision(references, hypothesis1, n=1)) # doctest: +ELLIPSIS
|
||||
0.9444...
|
||||
>>> float(modified_precision(references, hypothesis2, n=1)) # doctest: +ELLIPSIS
|
||||
0.5714...
|
||||
>>> float(modified_precision(references, hypothesis1, n=2)) # doctest: +ELLIPSIS
|
||||
0.5882...
|
||||
>>> float(modified_precision(references, hypothesis2, n=2)) # doctest: +ELLIPSIS
|
||||
0.07692...
|
||||
|
||||
Args:
|
||||
references (list of list of str):
|
||||
A list of reference translations.
|
||||
hypothesis (list of str):
|
||||
A hypothesis translation.
|
||||
n (int):
|
||||
The ngram order.
|
||||
|
||||
Returns:
|
||||
Fraction:
|
||||
BLEU's modified precision for the nth order ngram.
|
||||
"""
|
||||
# Extracts all ngrams in hypothesis
|
||||
# Set an empty Counter if hypothesis is empty.
|
||||
|
||||
counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter()
|
||||
# Extract a union of references' counts.
|
||||
# max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references])
|
||||
max_counts = {}
|
||||
for reference in references:
|
||||
reference_counts = (
|
||||
Counter(ngrams(reference, n)) if len(reference) >= n else Counter()
|
||||
)
|
||||
for ngram in counts:
|
||||
max_counts[ngram] = max(max_counts.get(ngram, 0), reference_counts[ngram])
|
||||
|
||||
# Assigns the intersection between hypothesis and references' counts.
|
||||
clipped_counts = {
|
||||
ngram: min(count, max_counts[ngram]) for ngram, count in counts.items()
|
||||
}
|
||||
|
||||
numerator = sum(clipped_counts.values())
|
||||
# Ensures that denominator is minimum 1 to avoid ZeroDivisionError.
|
||||
# Usually this happens when the ngram order is > len(reference).
|
||||
denominator = max(1, sum(counts.values()))
|
||||
|
||||
return Fraction(numerator, denominator, _normalize=False)
|
||||
|
||||
|
||||
def closest_ref_length(references, hyp_len):
|
||||
"""
|
||||
Find the length of the reference that is closest to the hypothesis length.
|
||||
|
||||
This function calculates the reference length that is closest to the length
|
||||
of the hypothesis. The closest reference length is used in the brevity penalty
|
||||
formula as described in the paper by Papineni et al. (2002).
|
||||
|
||||
Args:
|
||||
references (list of list of str): A list of reference translations.
|
||||
hyp_len (int): The length of the hypothesis.
|
||||
|
||||
Returns:
|
||||
int: The length of the reference that is closest to the hypothesis length.
|
||||
"""
|
||||
ref_lens = (len(reference) for reference in references)
|
||||
closest_ref_len = min(
|
||||
ref_lens, key=lambda ref_len: (abs(ref_len - hyp_len), ref_len)
|
||||
)
|
||||
return closest_ref_len
|
||||
|
||||
|
||||
def brevity_penalty(closest_ref_len, hyp_len):
|
||||
"""
|
||||
Calculate the brevity penalty.
|
||||
|
||||
The brevity penalty modifies the overall BLEU score according to the length of hypotheses and references.
|
||||
|
||||
Example 1: Hypothesis length matches closest reference length.
|
||||
>>> reference1 = list('aaaaaaaaaaaa') # i.e. ['a'] * 12
|
||||
>>> reference2 = list('aaaaaaaaaaaaaaa') # i.e. ['a'] * 15
|
||||
>>> reference3 = list('aaaaaaaaaaaaaaaaa') # i.e. ['a'] * 17
|
||||
>>> hypothesis = list('aaaaaaaaaaaa') # i.e. ['a'] * 12
|
||||
>>> references = [reference1, reference2, reference3]
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(hyp_len, closest_ref_len)
|
||||
1.0
|
||||
|
||||
Example 2: Hypothesis length is shorter than closest reference length.
|
||||
>>> references = [['a'] * 28, ['a'] * 28]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(hyp_len, closest_ref_len)
|
||||
0.2635971381157267
|
||||
|
||||
Example 3: Hypothesis length is shorter than the closest reference length.
|
||||
>>> references = [['a'] * 13, ['a'] * 2]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(hyp_len, closest_ref_len)
|
||||
0.920045281565605
|
||||
|
||||
Example 4: Hypothesis length is longer than the closest reference length.
|
||||
>>> references = [['a'] * 13, ['a'] * 11]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(hyp_len, closest_ref_len)
|
||||
1.0
|
||||
|
||||
Args:
|
||||
hyp_len (int):
|
||||
The length of the hypothesis for a single sentence OR the sum of all the hypotheses' lengths for a corpus.
|
||||
closest_ref_len (int):
|
||||
The length of the closest reference for a single hypothesis OR the sum of all the closest references for every hypothesis.
|
||||
|
||||
Returns:
|
||||
float:
|
||||
BLEU's brevity penalty.
|
||||
"""
|
||||
if hyp_len > closest_ref_len:
|
||||
return 1
|
||||
# If hypothesis is empty, brevity penalty = 0 should result in BLEU = 0.0
|
||||
elif hyp_len == 0:
|
||||
return 0
|
||||
else:
|
||||
return math.exp(1 - closest_ref_len / hyp_len)
|
||||
|
||||
|
||||
class SmoothingFunction:
|
||||
"""
|
||||
This is an implementation of the smoothing techniques
|
||||
for segment-level BLEU scores that was presented in
|
||||
Boxing Chen and Collin Cherry (2014) A Systematic Comparison of
|
||||
Smoothing Techniques for Sentence-Level BLEU. In WMT14.
|
||||
http://acl2014.org/acl2014/W14-33/pdf/W14-3346.pdf
|
||||
"""
|
||||
|
||||
def __init__(self, epsilon=0.1, alpha=5, k=5):
|
||||
"""
|
||||
This will initialize the parameters required for the various smoothing
|
||||
techniques, the default values are set to the numbers used in the
|
||||
experiments from Chen and Cherry (2014).
|
||||
>>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures',
|
||||
... 'that', 'the', 'military', 'always', 'obeys', 'the',
|
||||
... 'commands', 'of', 'the', 'party']
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', 'ensures',
|
||||
... 'that', 'the', 'military', 'will', 'forever', 'heed',
|
||||
... 'Party', 'commands']
|
||||
>>> chencherry = SmoothingFunction()
|
||||
>>> print(sentence_bleu([reference1], hypothesis1)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method0)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS
|
||||
0.4489...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method5)) # doctest: +ELLIPSIS
|
||||
0.4905...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method6)) # doctest: +ELLIPSIS
|
||||
0.4135...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method7)) # doctest: +ELLIPSIS
|
||||
0.4905...
|
||||
:param epsilon: the epsilon value use in method 1
|
||||
:type epsilon: float
|
||||
:param alpha: the alpha value use in method 6
|
||||
:type alpha: int
|
||||
:param k: the k value use in method 4
|
||||
:type k: int
|
||||
"""
|
||||
self.epsilon = epsilon
|
||||
self.alpha = alpha
|
||||
self.k = k
|
||||
|
||||
def method0(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
No smoothing.
|
||||
"""
|
||||
p_n_new = []
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i.numerator != 0:
|
||||
p_n_new.append(p_i)
|
||||
else:
|
||||
_msg = str(
|
||||
"\nThe hypothesis contains 0 counts of {}-gram overlaps.\n"
|
||||
"Therefore the BLEU score evaluates to 0, independently of\n"
|
||||
"how many N-gram overlaps of lower order it contains.\n"
|
||||
"Consider using lower n-gram order or use "
|
||||
"SmoothingFunction()"
|
||||
).format(i + 1)
|
||||
warnings.warn(_msg)
|
||||
# When numerator==0 where denonminator==0 or !=0, the result
|
||||
# for the precision score should be equal to 0 or undefined.
|
||||
# Due to BLEU geometric mean computation in logarithm space,
|
||||
# we we need to take the return sys.float_info.min such that
|
||||
# math.log(sys.float_info.min) returns a 0 precision score.
|
||||
p_n_new.append(sys.float_info.min)
|
||||
return p_n_new
|
||||
|
||||
def method1(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 1: Add *epsilon* counts to precision with 0 counts.
|
||||
"""
|
||||
return [
|
||||
(p_i.numerator + self.epsilon) / p_i.denominator
|
||||
if p_i.numerator == 0
|
||||
else p_i
|
||||
for p_i in p_n
|
||||
]
|
||||
|
||||
def method2(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 2: Add 1 to both numerator and denominator from
|
||||
Chin-Yew Lin and Franz Josef Och (2004) Automatic evaluation of
|
||||
machine translation quality using longest common subsequence and
|
||||
skip-bigram statistics. In ACL04.
|
||||
"""
|
||||
return [
|
||||
Fraction(p_i.numerator + 1, p_i.denominator + 1, _normalize=False)
|
||||
for p_i in p_n
|
||||
]
|
||||
|
||||
def method3(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 3: NIST geometric sequence smoothing
|
||||
The smoothing is computed by taking 1 / ( 2^k ), instead of 0, for each
|
||||
precision score whose matching n-gram count is null.
|
||||
k is 1 for the first 'n' value for which the n-gram match count is null/
|
||||
For example, if the text contains:
|
||||
- one 2-gram match
|
||||
- and (consequently) two 1-gram matches
|
||||
the n-gram count for each individual precision score would be:
|
||||
- n=1 => prec_count = 2 (two unigrams)
|
||||
- n=2 => prec_count = 1 (one bigram)
|
||||
- n=3 => prec_count = 1/2 (no trigram, taking 'smoothed' value of 1 / ( 2^k ), with k=1)
|
||||
- n=4 => prec_count = 1/4 (no fourgram, taking 'smoothed' value of 1 / ( 2^k ), with k=2)
|
||||
"""
|
||||
incvnt = 1 # From the mteval-v13a.pl, it's referred to as k.
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i.numerator == 0:
|
||||
p_n[i] = 1 / (2 ** incvnt * p_i.denominator)
|
||||
incvnt += 1
|
||||
return p_n
|
||||
|
||||
def method4(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 4:
|
||||
Shorter translations may have inflated precision values due to having
|
||||
smaller denominators; therefore, we give them proportionally
|
||||
smaller smoothed counts. Instead of scaling to 1/(2^k), Chen and Cherry
|
||||
suggests dividing by 1/ln(len(T)), where T is the length of the translation.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i.numerator == 0 and hyp_len != 0:
|
||||
incvnt = i + 1 * self.k / math.log(
|
||||
hyp_len
|
||||
) # Note that this K is different from the K from NIST.
|
||||
p_n[i] = incvnt / p_i.denominator
|
||||
return p_n
|
||||
|
||||
def method5(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 5:
|
||||
The matched counts for similar values of n should be similar. To a
|
||||
calculate the n-gram matched count, it averages the n−1, n and n+1 gram
|
||||
matched counts.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
m = {}
|
||||
# Requires an precision value for an addition ngram order.
|
||||
p_n_plus1 = p_n + [modified_precision(references, hypothesis, 5)]
|
||||
m[-1] = p_n[0] + 1
|
||||
for i, p_i in enumerate(p_n):
|
||||
p_n[i] = (m[i - 1] + p_i + p_n_plus1[i + 1]) / 3
|
||||
m[i] = p_n[i]
|
||||
return p_n
|
||||
|
||||
def method6(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 6:
|
||||
Interpolates the maximum likelihood estimate of the precision *p_n* with
|
||||
a prior estimate *pi0*. The prior is estimated by assuming that the ratio
|
||||
between pn and pn−1 will be the same as that between pn−1 and pn−2; from
|
||||
Gao and He (2013) Training MRF-Based Phrase Translation Models using
|
||||
Gradient Ascent. In NAACL.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
# This smoothing only works when p_1 and p_2 is non-zero.
|
||||
# Raise an error with an appropriate message when the input is too short
|
||||
# to use this smoothing technique.
|
||||
assert p_n[2], "This smoothing method requires non-zero precision for bigrams."
|
||||
for i, p_i in enumerate(p_n):
|
||||
if i in [0, 1]: # Skips the first 2 orders of ngrams.
|
||||
continue
|
||||
else:
|
||||
pi0 = 0 if p_n[i - 2] == 0 else p_n[i - 1] ** 2 / p_n[i - 2]
|
||||
# No. of ngrams in translation that matches the reference.
|
||||
m = p_i.numerator
|
||||
# No. of ngrams in translation.
|
||||
l = sum(1 for _ in ngrams(hypothesis, i + 1))
|
||||
# Calculates the interpolated precision.
|
||||
p_n[i] = (m + self.alpha * pi0) / (l + self.alpha)
|
||||
return p_n
|
||||
|
||||
def method7(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 7:
|
||||
Interpolates methods 4 and 5.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
p_n = self.method4(p_n, references, hypothesis, hyp_len)
|
||||
p_n = self.method5(p_n, references, hypothesis, hyp_len)
|
||||
return p_n
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
from codebleu.parser import DFG_python
|
||||
from codebleu.parser import (remove_comments_and_docstrings,
|
||||
tree_to_token_index,
|
||||
index_to_code_token,
|
||||
tree_to_variable_index)
|
||||
from tree_sitter import Language, Parser
|
||||
import pdb
|
||||
|
||||
dfg_function={
|
||||
'python':DFG_python
|
||||
}
|
||||
|
||||
def calc_dataflow_match(references, candidate, lang):
|
||||
"""
|
||||
Calculate the dataflow match score for a candidate code against references.
|
||||
|
||||
Args:
|
||||
references (list): A list of reference code samples.
|
||||
candidate (str): The candidate code to be evaluated.
|
||||
lang (str): The programming language of the code samples.
|
||||
|
||||
Returns:
|
||||
float: The dataflow match score.
|
||||
"""
|
||||
return corpus_dataflow_match([references], [candidate], lang)
|
||||
|
||||
def corpus_dataflow_match(references, candidates, lang):
|
||||
"""
|
||||
Calculate the corpus-level dataflow match score for candidates against references.
|
||||
|
||||
Args:
|
||||
references (list): A list of lists of reference code samples.
|
||||
candidates (list): A list of candidate code samples.
|
||||
lang (str): The programming language of the code samples.
|
||||
|
||||
Returns:
|
||||
float: The corpus-level dataflow match score.
|
||||
"""
|
||||
LANGUAGE = Language('codebleu/parser/my-languages.so', lang)
|
||||
parser = Parser()
|
||||
parser.set_language(LANGUAGE)
|
||||
parser = [parser,dfg_function[lang]]
|
||||
match_count = 0
|
||||
total_count = 0
|
||||
|
||||
for i in range(len(candidates)):
|
||||
references_sample = references[i]
|
||||
candidate = candidates[i]
|
||||
for reference in references_sample:
|
||||
try:
|
||||
candidate=remove_comments_and_docstrings(candidate,'java')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
reference=remove_comments_and_docstrings(reference,'java')
|
||||
except:
|
||||
pass
|
||||
|
||||
cand_dfg = get_data_flow(candidate, parser)
|
||||
ref_dfg = get_data_flow(reference, parser)
|
||||
|
||||
normalized_cand_dfg = normalize_dataflow(cand_dfg)
|
||||
normalized_ref_dfg = normalize_dataflow(ref_dfg)
|
||||
|
||||
if len(normalized_ref_dfg) > 0:
|
||||
total_count += len(normalized_ref_dfg)
|
||||
for dataflow in normalized_ref_dfg:
|
||||
if dataflow in normalized_cand_dfg:
|
||||
match_count += 1
|
||||
normalized_cand_dfg.remove(dataflow)
|
||||
|
||||
if total_count == 0:
|
||||
return 0
|
||||
|
||||
score = match_count / total_count
|
||||
return score
|
||||
|
||||
def get_data_flow(code, parser):
|
||||
"""
|
||||
Extract the dataflow graph (DFG) from the given code using the parser.
|
||||
|
||||
Args:
|
||||
code (str): The code from which to extract the DFG.
|
||||
parser (list): A list containing the language parser and DFG function.
|
||||
|
||||
Returns:
|
||||
list: The extracted dataflow graph.
|
||||
"""
|
||||
try:
|
||||
tree = parser[0].parse(bytes(code,'utf8'))
|
||||
root_node = tree.root_node
|
||||
tokens_index=tree_to_token_index(root_node)
|
||||
code=code.split('\n')
|
||||
code_tokens=[index_to_code_token(x,code) for x in tokens_index]
|
||||
index_to_code={}
|
||||
for idx,(index,code) in enumerate(zip(tokens_index,code_tokens)):
|
||||
index_to_code[index]=(idx,code)
|
||||
try:
|
||||
DFG,_=parser[1](root_node,index_to_code,{})
|
||||
except:
|
||||
DFG=[]
|
||||
DFG=sorted(DFG,key=lambda x:x[1])
|
||||
indexs=set()
|
||||
for d in DFG:
|
||||
if len(d[-1])!=0:
|
||||
indexs.add(d[1])
|
||||
for x in d[-1]:
|
||||
indexs.add(x)
|
||||
new_DFG=[]
|
||||
for d in DFG:
|
||||
if d[1] in indexs:
|
||||
new_DFG.append(d)
|
||||
codes=code_tokens
|
||||
dfg=new_DFG
|
||||
except:
|
||||
codes=code.split()
|
||||
dfg=[]
|
||||
#merge nodes
|
||||
dic={}
|
||||
for d in dfg:
|
||||
if d[1] not in dic:
|
||||
dic[d[1]]=d
|
||||
else:
|
||||
dic[d[1]]=(d[0],d[1],d[2],list(set(dic[d[1]][3]+d[3])),list(set(dic[d[1]][4]+d[4])))
|
||||
DFG=[]
|
||||
for d in dic:
|
||||
DFG.append(dic[d])
|
||||
dfg=DFG
|
||||
return dfg
|
||||
|
||||
def normalize_dataflow_item(dataflow_item):
|
||||
"""
|
||||
Normalize a single dataflow item.
|
||||
|
||||
Args:
|
||||
dataflow_item (tuple): A dataflow item tuple.
|
||||
|
||||
Returns:
|
||||
tuple: The normalized dataflow item.
|
||||
"""
|
||||
var_name = dataflow_item[0]
|
||||
var_pos = dataflow_item[1]
|
||||
relationship = dataflow_item[2]
|
||||
par_vars_name_list = dataflow_item[3]
|
||||
par_vars_pos_list = dataflow_item[4]
|
||||
|
||||
var_names = list(set(par_vars_name_list+[var_name]))
|
||||
norm_names = {}
|
||||
for i in range(len(var_names)):
|
||||
norm_names[var_names[i]] = 'var_'+str(i)
|
||||
|
||||
norm_var_name = norm_names[var_name]
|
||||
relationship = dataflow_item[2]
|
||||
norm_par_vars_name_list = [norm_names[x] for x in par_vars_name_list]
|
||||
|
||||
return (norm_var_name, relationship, norm_par_vars_name_list)
|
||||
|
||||
def normalize_dataflow(dataflow):
|
||||
"""
|
||||
Normalize a list of dataflow items.
|
||||
|
||||
Args:
|
||||
dataflow (list): A list of dataflow items.
|
||||
|
||||
Returns:
|
||||
list: The normalized list of dataflow items.
|
||||
"""
|
||||
var_dict = {}
|
||||
i = 0
|
||||
normalized_dataflow = []
|
||||
for item in dataflow:
|
||||
var_name = item[0]
|
||||
relationship = item[2]
|
||||
par_vars_name_list = item[3]
|
||||
for name in par_vars_name_list:
|
||||
if name not in var_dict:
|
||||
var_dict[name] = 'var_'+str(i)
|
||||
i += 1
|
||||
if var_name not in var_dict:
|
||||
var_dict[var_name] = 'var_'+str(i)
|
||||
i+= 1
|
||||
normalized_dataflow.append((var_dict[var_name], relationship, [var_dict[x] for x in par_vars_name_list]))
|
||||
return normalized_dataflow
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
abstract
|
||||
as
|
||||
base
|
||||
bool
|
||||
break
|
||||
byte
|
||||
case
|
||||
catch
|
||||
char
|
||||
checked
|
||||
class
|
||||
const
|
||||
continue
|
||||
decimal
|
||||
default
|
||||
delegate
|
||||
do
|
||||
double
|
||||
else
|
||||
enum
|
||||
event
|
||||
explicit
|
||||
extern
|
||||
false
|
||||
finally
|
||||
fixed
|
||||
float
|
||||
for
|
||||
foreach
|
||||
goto
|
||||
if
|
||||
implicit
|
||||
in
|
||||
int
|
||||
interface
|
||||
internal
|
||||
is
|
||||
lock
|
||||
long
|
||||
namespace
|
||||
new
|
||||
null
|
||||
object
|
||||
operator
|
||||
out
|
||||
override
|
||||
params
|
||||
private
|
||||
protected
|
||||
public
|
||||
readonly
|
||||
ref
|
||||
return
|
||||
sbyte
|
||||
sealed
|
||||
short
|
||||
sizeof
|
||||
stackalloc
|
||||
static
|
||||
string
|
||||
struct
|
||||
switch
|
||||
this
|
||||
throw
|
||||
true
|
||||
try
|
||||
typeof
|
||||
uint
|
||||
ulong
|
||||
unchecked
|
||||
unsafe
|
||||
ushort
|
||||
using
|
||||
virtual
|
||||
void
|
||||
volatile
|
||||
while
|
||||
add
|
||||
alias
|
||||
ascending
|
||||
async
|
||||
await
|
||||
by
|
||||
descending
|
||||
dynamic
|
||||
equals
|
||||
from
|
||||
get
|
||||
global
|
||||
group
|
||||
into
|
||||
join
|
||||
let
|
||||
nameof
|
||||
notnull
|
||||
on
|
||||
orderby
|
||||
partial
|
||||
remove
|
||||
select
|
||||
set
|
||||
unmanaged
|
||||
value
|
||||
var
|
||||
when
|
||||
where
|
||||
yield
|
||||
@@ -0,0 +1,50 @@
|
||||
abstract
|
||||
assert
|
||||
boolean
|
||||
break
|
||||
byte
|
||||
case
|
||||
catch
|
||||
char
|
||||
class
|
||||
const
|
||||
continue
|
||||
default
|
||||
do
|
||||
double
|
||||
else
|
||||
enum
|
||||
extends
|
||||
final
|
||||
finally
|
||||
float
|
||||
for
|
||||
goto
|
||||
if
|
||||
implements
|
||||
import
|
||||
instanceof
|
||||
int
|
||||
interface
|
||||
long
|
||||
native
|
||||
new
|
||||
package
|
||||
private
|
||||
protected
|
||||
public
|
||||
return
|
||||
short
|
||||
static
|
||||
strictfp
|
||||
super
|
||||
switch
|
||||
synchronized
|
||||
this
|
||||
throw
|
||||
throws
|
||||
transient
|
||||
try
|
||||
void
|
||||
volatile
|
||||
while
|
||||
@@ -0,0 +1,35 @@
|
||||
False
|
||||
None
|
||||
True
|
||||
and
|
||||
as
|
||||
assert
|
||||
async
|
||||
await
|
||||
break
|
||||
class
|
||||
continue
|
||||
def
|
||||
del
|
||||
elif
|
||||
else
|
||||
except
|
||||
finally
|
||||
for
|
||||
from
|
||||
global
|
||||
if
|
||||
import
|
||||
in
|
||||
is
|
||||
lambda
|
||||
nonlocal
|
||||
not
|
||||
or
|
||||
pass
|
||||
raise
|
||||
return
|
||||
try
|
||||
while
|
||||
with
|
||||
yield
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
from .utils import (remove_comments_and_docstrings,
|
||||
tree_to_token_index,
|
||||
index_to_code_token,
|
||||
tree_to_variable_index)
|
||||
from .DFG import DFG_python,DFG_java,DFG_ruby,DFG_go,DFG_php,DFG_javascript,DFG_csharp
|
||||
@@ -0,0 +1,14 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
from tree_sitter import Language, Parser
|
||||
|
||||
Language.build_library(
|
||||
# Store the library in the `build` directory
|
||||
'my-languages.so',
|
||||
|
||||
# Include one or more languages
|
||||
[ 'tree-sitter-python',
|
||||
]
|
||||
)
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
git clone https://github.com/tree-sitter/tree-sitter-python
|
||||
pip install tree_sitter
|
||||
python build.py
|
||||
Binary file not shown.
@@ -0,0 +1,2 @@
|
||||
/src/** linguist-vendored
|
||||
/examples/* linguist-vendored
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
name: Build/test
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- "**"
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
os: [macos-latest, ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 16
|
||||
- run: npm install
|
||||
- run: npm test
|
||||
test_windows:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 16
|
||||
- run: npm install
|
||||
- run: npm run-script test-windows
|
||||
@@ -0,0 +1,7 @@
|
||||
Cargo.lock
|
||||
package-lock.json
|
||||
node_modules
|
||||
build
|
||||
*.log
|
||||
/examples/*/
|
||||
/target/
|
||||
@@ -0,0 +1,6 @@
|
||||
corpus
|
||||
examples
|
||||
build
|
||||
script
|
||||
target
|
||||
bindings/rust
|
||||
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "tree-sitter-python"
|
||||
description = "Python grammar for the tree-sitter parsing library"
|
||||
version = "0.20.2"
|
||||
authors = [
|
||||
"Max Brunsfeld <maxbrunsfeld@gmail.com>",
|
||||
"Douglas Creager <dcreager@dcreager.net>",
|
||||
]
|
||||
license = "MIT"
|
||||
readme = "bindings/rust/README.md"
|
||||
keywords = ["incremental", "parsing", "python"]
|
||||
categories = ["parsing", "text-editors"]
|
||||
repository = "https://github.com/tree-sitter/tree-sitter-python"
|
||||
edition = "2018"
|
||||
|
||||
build = "bindings/rust/build.rs"
|
||||
include = [
|
||||
"bindings/rust/*",
|
||||
"grammar.js",
|
||||
"queries/*",
|
||||
"src/*",
|
||||
]
|
||||
|
||||
[lib]
|
||||
path = "bindings/rust/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
tree-sitter = ">= 0.19, < 0.21"
|
||||
|
||||
[build-dependencies]
|
||||
cc = "1.0"
|
||||
@@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016 Max Brunsfeld
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,13 @@
|
||||
tree-sitter-python
|
||||
==================
|
||||
|
||||
[](https://github.com/tree-sitter/tree-sitter-python/actions/workflows/ci.yml)
|
||||
|
||||
Python grammar for [tree-sitter][].
|
||||
|
||||
[tree-sitter]: https://github.com/tree-sitter/tree-sitter
|
||||
|
||||
#### References
|
||||
|
||||
* [Python 2 Grammar](https://docs.python.org/2/reference/grammar.html)
|
||||
* [Python 3 Grammar](https://docs.python.org/3/reference/grammar.html)
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"target_name": "tree_sitter_python_binding",
|
||||
"include_dirs": [
|
||||
"<!(node -e \"require('nan')\")",
|
||||
"src"
|
||||
],
|
||||
"sources": [
|
||||
"src/parser.c",
|
||||
"bindings/node/binding.cc",
|
||||
"src/scanner.cc"
|
||||
],
|
||||
"cflags_c": [
|
||||
"-std=c99",
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
#include "tree_sitter/parser.h"
|
||||
#include <node.h>
|
||||
#include "nan.h"
|
||||
|
||||
using namespace v8;
|
||||
|
||||
extern "C" TSLanguage * tree_sitter_python();
|
||||
|
||||
namespace {
|
||||
|
||||
NAN_METHOD(New) {}
|
||||
|
||||
void Init(Local<Object> exports, Local<Object> module) {
|
||||
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
|
||||
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
|
||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
||||
|
||||
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
|
||||
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
|
||||
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_python());
|
||||
|
||||
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("python").ToLocalChecked());
|
||||
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
|
||||
}
|
||||
|
||||
NODE_MODULE(tree_sitter_python_binding, Init)
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,19 @@
|
||||
try {
|
||||
module.exports = require("../../build/Release/tree_sitter_python_binding");
|
||||
} catch (error1) {
|
||||
if (error1.code !== 'MODULE_NOT_FOUND') {
|
||||
throw error1;
|
||||
}
|
||||
try {
|
||||
module.exports = require("../../build/Debug/tree_sitter_python_binding");
|
||||
} catch (error2) {
|
||||
if (error2.code !== 'MODULE_NOT_FOUND') {
|
||||
throw error2;
|
||||
}
|
||||
throw error1
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
module.exports.nodeTypeInfo = require("../../src/node-types.json");
|
||||
} catch (_) {}
|
||||
@@ -0,0 +1,36 @@
|
||||
# tree-sitter-python
|
||||
|
||||
This crate provides a Python grammar for the [tree-sitter][] parsing library.
|
||||
To use this crate, add it to the `[dependencies]` section of your `Cargo.toml`
|
||||
file. (Note that you will probably also need to depend on the
|
||||
[`tree-sitter`][tree-sitter crate] crate to use the parsed result in any useful
|
||||
way.)
|
||||
|
||||
``` toml
|
||||
[dependencies]
|
||||
tree-sitter = "0.17"
|
||||
tree-sitter-python = "0.17"
|
||||
```
|
||||
|
||||
Typically, you will use the [language][language func] function to add this
|
||||
grammar to a tree-sitter [Parser][], and then use the parser to parse some code:
|
||||
|
||||
``` rust
|
||||
let code = r#"
|
||||
def double(x):
|
||||
return x * 2
|
||||
"#;
|
||||
let mut parser = Parser::new();
|
||||
parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
|
||||
let parsed = parser.parse(code, None);
|
||||
```
|
||||
|
||||
If you have any questions, please reach out to us in the [tree-sitter
|
||||
discussions] page.
|
||||
|
||||
[Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
|
||||
[language func]: https://docs.rs/tree-sitter-python/*/tree_sitter_python/fn.language.html
|
||||
[Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
|
||||
[tree-sitter]: https://tree-sitter.github.io/
|
||||
[tree-sitter crate]: https://crates.io/crates/tree-sitter
|
||||
[tree-sitter discussions]: https://github.com/tree-sitter/tree-sitter/discussions
|
||||
@@ -0,0 +1,28 @@
|
||||
use std::path::Path;
|
||||
extern crate cc;
|
||||
|
||||
fn main() {
|
||||
let src_dir = Path::new("src");
|
||||
|
||||
let mut c_config = cc::Build::new();
|
||||
c_config.include(&src_dir);
|
||||
c_config
|
||||
.flag_if_supported("-Wno-unused-parameter")
|
||||
.flag_if_supported("-Wno-unused-but-set-variable")
|
||||
.flag_if_supported("-Wno-trigraphs");
|
||||
let parser_path = src_dir.join("parser.c");
|
||||
c_config.file(&parser_path);
|
||||
println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
|
||||
c_config.compile("parser");
|
||||
|
||||
let mut cpp_config = cc::Build::new();
|
||||
cpp_config.cpp(true);
|
||||
cpp_config.include(&src_dir);
|
||||
cpp_config
|
||||
.flag_if_supported("-Wno-unused-parameter")
|
||||
.flag_if_supported("-Wno-unused-but-set-variable");
|
||||
let scanner_path = src_dir.join("scanner.cc");
|
||||
cpp_config.file(&scanner_path);
|
||||
println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
|
||||
cpp_config.compile("scanner");
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// -*- coding: utf-8 -*-
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Copyright © 2020, tree-sitter-python authors.
|
||||
// See the LICENSE file in this repo for license details.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
//! This crate provides a Python grammar for the [tree-sitter][] parsing library.
|
||||
//!
|
||||
//! Typically, you will use the [language][language func] function to add this grammar to a
|
||||
//! tree-sitter [Parser][], and then use the parser to parse some code:
|
||||
//!
|
||||
//! ```
|
||||
//! use tree_sitter::Parser;
|
||||
//!
|
||||
//! let code = r#"
|
||||
//! def double(x):
|
||||
//! return x * 2
|
||||
//! "#;
|
||||
//! let mut parser = Parser::new();
|
||||
//! parser.set_language(tree_sitter_python::language()).expect("Error loading Python grammar");
|
||||
//! let parsed = parser.parse(code, None);
|
||||
//! # let parsed = parsed.unwrap();
|
||||
//! # let root = parsed.root_node();
|
||||
//! # assert!(!root.has_error());
|
||||
//! ```
|
||||
//!
|
||||
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
|
||||
//! [language func]: fn.language.html
|
||||
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
|
||||
//! [tree-sitter]: https://tree-sitter.github.io/
|
||||
|
||||
use tree_sitter::Language;
|
||||
|
||||
extern "C" {
|
||||
fn tree_sitter_python() -> Language;
|
||||
}
|
||||
|
||||
/// Returns the tree-sitter [Language][] for this grammar.
|
||||
///
|
||||
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
|
||||
pub fn language() -> Language {
|
||||
unsafe { tree_sitter_python() }
|
||||
}
|
||||
|
||||
/// The source of the Python tree-sitter grammar description.
|
||||
pub const GRAMMAR: &'static str = include_str!("../../grammar.js");
|
||||
|
||||
/// The syntax highlighting query for this language.
|
||||
pub const HIGHLIGHT_QUERY: &'static str = include_str!("../../queries/highlights.scm");
|
||||
|
||||
/// The content of the [`node-types.json`][] file for this grammar.
|
||||
///
|
||||
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
|
||||
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
|
||||
|
||||
/// The symbol tagging query for this language.
|
||||
pub const TAGGING_QUERY: &'static str = include_str!("../../queries/tags.scm");
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn can_load_grammar() {
|
||||
let mut parser = tree_sitter::Parser::new();
|
||||
parser
|
||||
.set_language(super::language())
|
||||
.expect("Error loading Python grammar");
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
|
||||
class Foo:
|
||||
def bar():
|
||||
print "hi"
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
print a
|
||||
|
||||
if b:
|
||||
if c:
|
||||
d
|
||||
e
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
def main():
|
||||
print "hello"
|
||||
# 1 tab = 8 spaces in Python 2
|
||||
return
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
def hi():
|
||||
|
||||
|
||||
|
||||
print "hi"
|
||||
|
||||
|
||||
def bye():
|
||||
print "bye"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Vendored
+973
@@ -0,0 +1,973 @@
|
||||
# Python test set -- part 1, grammar.
|
||||
# This just tests whether the parser accepts them all.
|
||||
|
||||
# NOTE: When you run this test as a script from the command line, you
|
||||
# get warnings about certain hex/oct constants. Since those are
|
||||
# issued by the parser, you can't suppress them by adding a
|
||||
# filterwarnings() call to this module. Therefore, to shut up the
|
||||
# regression test, the filterwarnings() call has been added to
|
||||
# regrtest.py.
|
||||
|
||||
from test.test_support import run_unittest, check_syntax_error
|
||||
import unittest
|
||||
import sys
|
||||
# testing import *
|
||||
from sys import *
|
||||
|
||||
class TokenTests(unittest.TestCase):
|
||||
|
||||
def testBackslash(self):
|
||||
# Backslash means line continuation:
|
||||
x = 1 \
|
||||
+ 1
|
||||
self.assertEqual(x, 2, 'backslash for line continuation')
|
||||
|
||||
# Backslash does not means continuation in comments :\
|
||||
x = 0
|
||||
self.assertEqual(x, 0, 'backslash ending comment')
|
||||
|
||||
def testPlainIntegers(self):
|
||||
self.assertEqual(0xff, 255)
|
||||
self.assertEqual(0377, 255)
|
||||
self.assertEqual(2147483647, 017777777777)
|
||||
# "0x" is not a valid literal
|
||||
self.assertRaises(SyntaxError, eval, "0x")
|
||||
from sys import maxint
|
||||
if maxint == 2147483647:
|
||||
self.assertEqual(-2147483647-1, -020000000000)
|
||||
# XXX -2147483648
|
||||
self.assert_(037777777777 > 0)
|
||||
self.assert_(0xffffffff > 0)
|
||||
for s in '2147483648', '040000000000', '0x100000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
elif maxint == 9223372036854775807:
|
||||
self.assertEqual(-9223372036854775807-1, -01000000000000000000000)
|
||||
self.assert_(01777777777777777777777 > 0)
|
||||
self.assert_(0xffffffffffffffff > 0)
|
||||
for s in '9223372036854775808', '02000000000000000000000','0x10000000000000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
else:
|
||||
self.fail('Weird maxint value %r' % maxint)
|
||||
|
||||
def testLongIntegers(self):
|
||||
x = 0L
|
||||
x = 0l
|
||||
x = 0xffffffffffffffffL
|
||||
x = 0xffffffffffffffffl
|
||||
x = 077777777777777777L
|
||||
x = 077777777777777777l
|
||||
x = 123456789012345678901234567890L
|
||||
x = 123456789012345678901234567890l
|
||||
|
||||
def testFloats(self):
|
||||
x = 3.14
|
||||
x = 314.
|
||||
x = 0.314
|
||||
# XXX x = 000.314
|
||||
x = .314
|
||||
x = 3e14
|
||||
x = 3E14
|
||||
x = 3e-14
|
||||
x = 3e+14
|
||||
x = 3.e14
|
||||
x = .3e14
|
||||
x = 3.1e4
|
||||
|
||||
class GrammarTests(unittest.TestCase):
|
||||
|
||||
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
||||
# XXX can't test in a script -- this rule is only used when interactive
|
||||
|
||||
# file_input: (NEWLINE | stmt)* ENDMARKER
|
||||
# Being tested as this very moment this very module
|
||||
|
||||
# expr_input: testlist NEWLINE
|
||||
# XXX Hard to test -- used only in calls to input()
|
||||
|
||||
def testEvalInput(self):
|
||||
# testlist ENDMARKER
|
||||
x = eval('1, 0 or 1')
|
||||
|
||||
def testFuncdef(self):
|
||||
### 'def' NAME parameters ':' suite
|
||||
### parameters: '(' [varargslist] ')'
|
||||
### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
|
||||
### | ('**'|'*' '*') NAME)
|
||||
### | fpdef ['=' test] (',' fpdef ['=' test])* [',']
|
||||
### fpdef: NAME | '(' fplist ')'
|
||||
### fplist: fpdef (',' fpdef)* [',']
|
||||
### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
|
||||
### argument: [test '='] test # Really [keyword '='] test
|
||||
def f1(): pass
|
||||
f1()
|
||||
f1(*())
|
||||
f1(*(), **{})
|
||||
def f2(one_argument): pass
|
||||
def f3(two, arguments): pass
|
||||
def f4(two, (compound, (argument, list))): pass
|
||||
def f5((compound, first), two): pass
|
||||
self.assertEqual(f2.func_code.co_varnames, ('one_argument',))
|
||||
self.assertEqual(f3.func_code.co_varnames, ('two', 'arguments'))
|
||||
if sys.platform.startswith('java'):
|
||||
self.assertEqual(f4.func_code.co_varnames,
|
||||
('two', '(compound, (argument, list))', 'compound', 'argument',
|
||||
'list',))
|
||||
self.assertEqual(f5.func_code.co_varnames,
|
||||
('(compound, first)', 'two', 'compound', 'first'))
|
||||
else:
|
||||
self.assertEqual(f4.func_code.co_varnames,
|
||||
('two', '.1', 'compound', 'argument', 'list'))
|
||||
self.assertEqual(f5.func_code.co_varnames,
|
||||
('.0', 'two', 'compound', 'first'))
|
||||
def a1(one_arg,): pass
|
||||
def a2(two, args,): pass
|
||||
def v0(*rest): pass
|
||||
def v1(a, *rest): pass
|
||||
def v2(a, b, *rest): pass
|
||||
def v3(a, (b, c), *rest): return a, b, c, rest
|
||||
|
||||
f1()
|
||||
f2(1)
|
||||
f2(1,)
|
||||
f3(1, 2)
|
||||
f3(1, 2,)
|
||||
f4(1, (2, (3, 4)))
|
||||
v0()
|
||||
v0(1)
|
||||
v0(1,)
|
||||
v0(1,2)
|
||||
v0(1,2,3,4,5,6,7,8,9,0)
|
||||
v1(1)
|
||||
v1(1,)
|
||||
v1(1,2)
|
||||
v1(1,2,3)
|
||||
v1(1,2,3,4,5,6,7,8,9,0)
|
||||
v2(1,2)
|
||||
v2(1,2,3)
|
||||
v2(1,2,3,4)
|
||||
v2(1,2,3,4,5,6,7,8,9,0)
|
||||
v3(1,(2,3))
|
||||
v3(1,(2,3),4)
|
||||
v3(1,(2,3),4,5,6,7,8,9,0)
|
||||
|
||||
# ceval unpacks the formal arguments into the first argcount names;
|
||||
# thus, the names nested inside tuples must appear after these names.
|
||||
if sys.platform.startswith('java'):
|
||||
self.assertEqual(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
|
||||
else:
|
||||
self.assertEqual(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
|
||||
self.assertEqual(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
|
||||
def d01(a=1): pass
|
||||
d01()
|
||||
d01(1)
|
||||
d01(*(1,))
|
||||
d01(**{'a':2})
|
||||
def d11(a, b=1): pass
|
||||
d11(1)
|
||||
d11(1, 2)
|
||||
d11(1, **{'b':2})
|
||||
def d21(a, b, c=1): pass
|
||||
d21(1, 2)
|
||||
d21(1, 2, 3)
|
||||
d21(*(1, 2, 3))
|
||||
d21(1, *(2, 3))
|
||||
d21(1, 2, *(3,))
|
||||
d21(1, 2, **{'c':3})
|
||||
def d02(a=1, b=2): pass
|
||||
d02()
|
||||
d02(1)
|
||||
d02(1, 2)
|
||||
d02(*(1, 2))
|
||||
d02(1, *(2,))
|
||||
d02(1, **{'b':2})
|
||||
d02(**{'a': 1, 'b': 2})
|
||||
def d12(a, b=1, c=2): pass
|
||||
d12(1)
|
||||
d12(1, 2)
|
||||
d12(1, 2, 3)
|
||||
def d22(a, b, c=1, d=2): pass
|
||||
d22(1, 2)
|
||||
d22(1, 2, 3)
|
||||
d22(1, 2, 3, 4)
|
||||
def d01v(a=1, *rest): pass
|
||||
d01v()
|
||||
d01v(1)
|
||||
d01v(1, 2)
|
||||
d01v(*(1, 2, 3, 4))
|
||||
d01v(*(1,))
|
||||
d01v(**{'a':2})
|
||||
def d11v(a, b=1, *rest): pass
|
||||
d11v(1)
|
||||
d11v(1, 2)
|
||||
d11v(1, 2, 3)
|
||||
def d21v(a, b, c=1, *rest): pass
|
||||
d21v(1, 2)
|
||||
d21v(1, 2, 3)
|
||||
d21v(1, 2, 3, 4)
|
||||
d21v(*(1, 2, 3, 4))
|
||||
d21v(1, 2, **{'c': 3})
|
||||
def d02v(a=1, b=2, *rest): pass
|
||||
d02v()
|
||||
d02v(1)
|
||||
d02v(1, 2)
|
||||
d02v(1, 2, 3)
|
||||
d02v(1, *(2, 3, 4))
|
||||
d02v(**{'a': 1, 'b': 2})
|
||||
def d12v(a, b=1, c=2, *rest): pass
|
||||
d12v(1)
|
||||
d12v(1, 2)
|
||||
d12v(1, 2, 3)
|
||||
d12v(1, 2, 3, 4)
|
||||
d12v(*(1, 2, 3, 4))
|
||||
d12v(1, 2, *(3, 4, 5))
|
||||
d12v(1, *(2,), **{'c': 3})
|
||||
def d22v(a, b, c=1, d=2, *rest): pass
|
||||
d22v(1, 2)
|
||||
d22v(1, 2, 3)
|
||||
d22v(1, 2, 3, 4)
|
||||
d22v(1, 2, 3, 4, 5)
|
||||
d22v(*(1, 2, 3, 4))
|
||||
d22v(1, 2, *(3, 4, 5))
|
||||
d22v(1, *(2, 3), **{'d': 4})
|
||||
def d31v((x)): pass
|
||||
d31v(1)
|
||||
def d32v((x,)): pass
|
||||
d32v((1,))
|
||||
|
||||
# keyword arguments after *arglist
|
||||
def f(*args, **kwargs):
|
||||
return args, kwargs
|
||||
self.assertEqual(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
|
||||
{'x':2, 'y':5}))
|
||||
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
|
||||
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
|
||||
|
||||
# Check ast errors in *args and *kwargs
|
||||
check_syntax_error(self, "f(*g(1=2))")
|
||||
check_syntax_error(self, "f(**g(1=2))")
|
||||
|
||||
def testLambdef(self):
|
||||
### lambdef: 'lambda' [varargslist] ':' test
|
||||
l1 = lambda : 0
|
||||
self.assertEqual(l1(), 0)
|
||||
l2 = lambda : a[d] # XXX just testing the expression
|
||||
l3 = lambda : [2 < x for x in [-1, 3, 0L]]
|
||||
self.assertEqual(l3(), [0, 1, 0])
|
||||
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
|
||||
self.assertEqual(l4(), 1)
|
||||
l5 = lambda x, y, z=2: x + y + z
|
||||
self.assertEqual(l5(1, 2), 5)
|
||||
self.assertEqual(l5(1, 2, 3), 6)
|
||||
check_syntax_error(self, "lambda x: x = 2")
|
||||
check_syntax_error(self, "lambda (None,): None")
|
||||
|
||||
### stmt: simple_stmt | compound_stmt
|
||||
# Tested below
|
||||
|
||||
def testSimpleStmt(self):
|
||||
### simple_stmt: small_stmt (';' small_stmt)* [';']
|
||||
x = 1; pass; del x
|
||||
def foo():
|
||||
# verify statements that end with semi-colons
|
||||
x = 1; pass; del x;
|
||||
foo()
|
||||
|
||||
### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
|
||||
# Tested below
|
||||
|
||||
def testExprStmt(self):
|
||||
# (exprlist '=')* exprlist
|
||||
1
|
||||
1, 2, 3
|
||||
x = 1
|
||||
x = 1, 2, 3
|
||||
x = y = z = 1, 2, 3
|
||||
x, y, z = 1, 2, 3
|
||||
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
|
||||
|
||||
check_syntax_error(self, "x + 1 = 1")
|
||||
check_syntax_error(self, "a + 1 = b + 2")
|
||||
|
||||
def testPrintStmt(self):
|
||||
# 'print' (test ',')* [test]
|
||||
import StringIO
|
||||
|
||||
# Can't test printing to real stdout without comparing output
|
||||
# which is not available in unittest.
|
||||
save_stdout = sys.stdout
|
||||
sys.stdout = StringIO.StringIO()
|
||||
|
||||
print 1, 2, 3
|
||||
print 1, 2, 3,
|
||||
print
|
||||
print 0 or 1, 0 or 1,
|
||||
print 0 or 1
|
||||
|
||||
# 'print' '>>' test ','
|
||||
print >> sys.stdout, 1, 2, 3
|
||||
print >> sys.stdout, 1, 2, 3,
|
||||
print >> sys.stdout
|
||||
print >> sys.stdout, 0 or 1, 0 or 1,
|
||||
print >> sys.stdout, 0 or 1
|
||||
|
||||
# test printing to an instance
|
||||
class Gulp:
|
||||
def write(self, msg): pass
|
||||
|
||||
gulp = Gulp()
|
||||
print >> gulp, 1, 2, 3
|
||||
print >> gulp, 1, 2, 3,
|
||||
print >> gulp
|
||||
print >> gulp, 0 or 1, 0 or 1,
|
||||
print >> gulp, 0 or 1
|
||||
|
||||
# test print >> None
|
||||
def driver():
|
||||
oldstdout = sys.stdout
|
||||
sys.stdout = Gulp()
|
||||
try:
|
||||
tellme(Gulp())
|
||||
tellme()
|
||||
finally:
|
||||
sys.stdout = oldstdout
|
||||
|
||||
# we should see this once
|
||||
def tellme(file=sys.stdout):
|
||||
print >> file, 'hello world'
|
||||
|
||||
driver()
|
||||
|
||||
# we should not see this at all
|
||||
def tellme(file=None):
|
||||
print >> file, 'goodbye universe'
|
||||
|
||||
driver()
|
||||
|
||||
self.assertEqual(sys.stdout.getvalue(), '''\
|
||||
1 2 3
|
||||
1 2 3
|
||||
1 1 1
|
||||
1 2 3
|
||||
1 2 3
|
||||
1 1 1
|
||||
hello world
|
||||
''')
|
||||
sys.stdout = save_stdout
|
||||
|
||||
# syntax errors
|
||||
check_syntax_error(self, 'print ,')
|
||||
check_syntax_error(self, 'print >> x,')
|
||||
|
||||
def testDelStmt(self):
|
||||
# 'del' exprlist
|
||||
abc = [1,2,3]
|
||||
x, y, z = abc
|
||||
xyz = x, y, z
|
||||
|
||||
del abc
|
||||
del x, y, (z, xyz)
|
||||
|
||||
def testPassStmt(self):
|
||||
# 'pass'
|
||||
pass
|
||||
|
||||
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
|
||||
# Tested below
|
||||
|
||||
def testBreakStmt(self):
|
||||
# 'break'
|
||||
while 1: break
|
||||
|
||||
def testContinueStmt(self):
|
||||
# 'continue'
|
||||
i = 1
|
||||
while i: i = 0; continue
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "ok"
|
||||
try:
|
||||
continue
|
||||
msg = "continue failed to continue inside try"
|
||||
except:
|
||||
msg = "continue inside try called except block"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "finally block not called"
|
||||
try:
|
||||
continue
|
||||
finally:
|
||||
msg = "ok"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
def test_break_continue_loop(self):
|
||||
# This test warrants an explanation. It is a test specifically for SF bugs
|
||||
# #463359 and #462937. The bug is that a 'break' statement executed or
|
||||
# exception raised inside a try/except inside a loop, *after* a continue
|
||||
# statement has been executed in that loop, will cause the wrong number of
|
||||
# arguments to be popped off the stack and the instruction pointer reset to
|
||||
# a very small number (usually 0.) Because of this, the following test
|
||||
# *must* written as a function, and the tracking vars *must* be function
|
||||
# arguments with default values. Otherwise, the test will loop and loop.
|
||||
|
||||
def test_inner(extra_burning_oil = 1, count=0):
|
||||
big_hippo = 2
|
||||
while big_hippo:
|
||||
count += 1
|
||||
try:
|
||||
if extra_burning_oil and big_hippo == 1:
|
||||
extra_burning_oil -= 1
|
||||
break
|
||||
big_hippo -= 1
|
||||
continue
|
||||
except:
|
||||
raise
|
||||
if count > 2 or big_hippo <> 1:
|
||||
self.fail("continue then break in try/except in loop broken!")
|
||||
test_inner()
|
||||
|
||||
def testReturn(self):
|
||||
# 'return' [testlist]
|
||||
def g1(): return
|
||||
def g2(): return 1
|
||||
g1()
|
||||
x = g2()
|
||||
check_syntax_error(self, "class foo:return 1")
|
||||
|
||||
def testYield(self):
|
||||
check_syntax_error(self, "class foo:yield 1")
|
||||
|
||||
def testRaise(self):
|
||||
# 'raise' test [',' test]
|
||||
try: raise RuntimeError, 'just testing'
|
||||
except RuntimeError: pass
|
||||
try: raise KeyboardInterrupt
|
||||
except KeyboardInterrupt: pass
|
||||
|
||||
def testImport(self):
|
||||
# 'import' dotted_as_names
|
||||
import sys
|
||||
import time, sys
|
||||
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
|
||||
from time import time
|
||||
from time import (time)
|
||||
# not testable inside a function, but already done at top of the module
|
||||
# from sys import *
|
||||
from sys import path, argv
|
||||
from sys import (path, argv)
|
||||
from sys import (path, argv,)
|
||||
|
||||
def testGlobal(self):
|
||||
# 'global' NAME (',' NAME)*
|
||||
global a
|
||||
global a, b
|
||||
global one, two, three, four, five, six, seven, eight, nine, ten
|
||||
|
||||
def testExec(self):
|
||||
# 'exec' expr ['in' expr [',' expr]]
|
||||
z = None
|
||||
del z
|
||||
exec 'z=1+1\n'
|
||||
if z != 2: self.fail('exec \'z=1+1\'\\n')
|
||||
del z
|
||||
exec 'z=1+1'
|
||||
if z != 2: self.fail('exec \'z=1+1\'')
|
||||
z = None
|
||||
del z
|
||||
import types
|
||||
if hasattr(types, "UnicodeType"):
|
||||
exec r"""if 1:
|
||||
exec u'z=1+1\n'
|
||||
if z != 2: self.fail('exec u\'z=1+1\'\\n')
|
||||
del z
|
||||
exec u'z=1+1'
|
||||
if z != 2: self.fail('exec u\'z=1+1\'')"""
|
||||
g = {}
|
||||
exec 'z = 1' in g
|
||||
if g.has_key('__builtins__'): del g['__builtins__']
|
||||
if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
|
||||
g = {}
|
||||
l = {}
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", "global statement", module="<string>")
|
||||
exec 'global a; a = 1; b = 2' in g, l
|
||||
if g.has_key('__builtins__'): del g['__builtins__']
|
||||
if l.has_key('__builtins__'): del l['__builtins__']
|
||||
if (g, l) != ({'a':1}, {'b':2}):
|
||||
self.fail('exec ... in g (%s), l (%s)' %(g,l))
|
||||
|
||||
def testAssert(self):
|
||||
# assert_stmt: 'assert' test [',' test]
|
||||
assert 1
|
||||
assert 1, 1
|
||||
assert lambda x:x
|
||||
assert 1, lambda x:x+1
|
||||
try:
|
||||
assert 0, "msg"
|
||||
except AssertionError, e:
|
||||
self.assertEqual(e.args[0], "msg")
|
||||
else:
|
||||
if __debug__:
|
||||
self.fail("AssertionError not raised by assert 0")
|
||||
|
||||
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
|
||||
# Tested below
|
||||
|
||||
def testIf(self):
|
||||
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||
if 1: pass
|
||||
if 1: pass
|
||||
else: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
else: pass
|
||||
|
||||
def testWhile(self):
|
||||
# 'while' test ':' suite ['else' ':' suite]
|
||||
while 0: pass
|
||||
while 0: pass
|
||||
else: pass
|
||||
|
||||
# Issue1920: "while 0" is optimized away,
|
||||
# ensure that the "else" clause is still present.
|
||||
x = 0
|
||||
while 0:
|
||||
x = 1
|
||||
else:
|
||||
x = 2
|
||||
self.assertEqual(x, 2)
|
||||
|
||||
def testFor(self):
|
||||
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
|
||||
for i in 1, 2, 3: pass
|
||||
for i, j, k in (): pass
|
||||
else: pass
|
||||
class Squares:
|
||||
def __init__(self, max):
|
||||
self.max = max
|
||||
self.sofar = []
|
||||
def __len__(self): return len(self.sofar)
|
||||
def __getitem__(self, i):
|
||||
if not 0 <= i < self.max: raise IndexError
|
||||
n = len(self.sofar)
|
||||
while n <= i:
|
||||
self.sofar.append(n*n)
|
||||
n = n+1
|
||||
return self.sofar[i]
|
||||
n = 0
|
||||
for x in Squares(10): n = n+x
|
||||
if n != 285:
|
||||
self.fail('for over growing sequence')
|
||||
|
||||
result = []
|
||||
for x, in [(1,), (2,), (3,)]:
|
||||
result.append(x)
|
||||
self.assertEqual(result, [1, 2, 3])
|
||||
|
||||
def testTry(self):
|
||||
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
|
||||
### | 'try' ':' suite 'finally' ':' suite
|
||||
### except_clause: 'except' [expr [('as' | ',') expr]]
|
||||
try:
|
||||
1/0
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
try: 1/0
|
||||
except EOFError: pass
|
||||
except TypeError as msg: pass
|
||||
except RuntimeError, msg: pass
|
||||
except: pass
|
||||
else: pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError): pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError), msg: pass
|
||||
try: pass
|
||||
finally: pass
|
||||
|
||||
def testSuite(self):
|
||||
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
|
||||
if 1: pass
|
||||
if 1:
|
||||
pass
|
||||
if 1:
|
||||
#
|
||||
#
|
||||
#
|
||||
pass
|
||||
pass
|
||||
#
|
||||
pass
|
||||
#
|
||||
|
||||
def testTest(self):
|
||||
### and_test ('or' and_test)*
|
||||
### and_test: not_test ('and' not_test)*
|
||||
### not_test: 'not' not_test | comparison
|
||||
if not 1: pass
|
||||
if 1 and 1: pass
|
||||
if 1 or 1: pass
|
||||
if not not not 1: pass
|
||||
if not 1 and 1 and 1: pass
|
||||
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
|
||||
|
||||
def testComparison(self):
|
||||
### comparison: expr (comp_op expr)*
|
||||
### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||||
if 1: pass
|
||||
x = (1 == 1)
|
||||
if 1 == 1: pass
|
||||
if 1 != 1: pass
|
||||
if 1 <> 1: pass
|
||||
if 1 < 1: pass
|
||||
if 1 > 1: pass
|
||||
if 1 <= 1: pass
|
||||
if 1 >= 1: pass
|
||||
if 1 is 1: pass
|
||||
if 1 is not 1: pass
|
||||
if 1 in (): pass
|
||||
if 1 not in (): pass
|
||||
if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
|
||||
|
||||
def testBinaryMaskOps(self):
|
||||
x = 1 & 1
|
||||
x = 1 ^ 1
|
||||
x = 1 | 1
|
||||
|
||||
def testShiftOps(self):
|
||||
x = 1 << 1
|
||||
x = 1 >> 1
|
||||
x = 1 << 1 >> 1
|
||||
|
||||
def testAdditiveOps(self):
|
||||
x = 1
|
||||
x = 1 + 1
|
||||
x = 1 - 1 - 1
|
||||
x = 1 - 1 + 1 - 1 + 1
|
||||
|
||||
def testMultiplicativeOps(self):
|
||||
x = 1 * 1
|
||||
x = 1 / 1
|
||||
x = 1 % 1
|
||||
x = 1 / 1 * 1 % 1
|
||||
|
||||
def testUnaryOps(self):
|
||||
x = +1
|
||||
x = -1
|
||||
x = ~1
|
||||
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
|
||||
x = -1*1/1 + 1*1 - ---1*1
|
||||
|
||||
def testSelectors(self):
|
||||
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
|
||||
### subscript: expr | [expr] ':' [expr]
|
||||
|
||||
import sys, time
|
||||
c = sys.path[0]
|
||||
x = time.time()
|
||||
x = sys.modules['time'].time()
|
||||
a = '01234'
|
||||
c = a[0]
|
||||
c = a[-1]
|
||||
s = a[0:5]
|
||||
s = a[:5]
|
||||
s = a[0:]
|
||||
s = a[:]
|
||||
s = a[-5:]
|
||||
s = a[:-1]
|
||||
s = a[-4:-3]
|
||||
# A rough test of SF bug 1333982. http://python.org/sf/1333982
|
||||
# The testing here is fairly incomplete.
|
||||
# Test cases should include: commas with 1 and 2 colons
|
||||
d = {}
|
||||
d[1] = 1
|
||||
d[1,] = 2
|
||||
d[1,2] = 3
|
||||
d[1,2,3] = 4
|
||||
L = list(d)
|
||||
L.sort()
|
||||
self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
|
||||
|
||||
def testAtoms(self):
|
||||
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
|
||||
### dictmaker: test ':' test (',' test ':' test)* [',']
|
||||
|
||||
x = (1)
|
||||
x = (1 or 2 or 3)
|
||||
x = (1 or 2 or 3, 2, 3)
|
||||
|
||||
x = []
|
||||
x = [1]
|
||||
x = [1 or 2 or 3]
|
||||
x = [1 or 2 or 3, 2, 3]
|
||||
x = []
|
||||
|
||||
x = {}
|
||||
x = {'one': 1}
|
||||
x = {'one': 1,}
|
||||
x = {'one' or 'two': 1 or 2}
|
||||
x = {'one': 1, 'two': 2}
|
||||
x = {'one': 1, 'two': 2,}
|
||||
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
|
||||
|
||||
x = `x`
|
||||
x = `1 or 2 or 3`
|
||||
self.assertEqual(`1,2`, '(1, 2)')
|
||||
|
||||
x = x
|
||||
x = 'x'
|
||||
x = 123
|
||||
|
||||
### exprlist: expr (',' expr)* [',']
|
||||
### testlist: test (',' test)* [',']
|
||||
# These have been exercised enough above
|
||||
|
||||
def testClassdef(self):
|
||||
# 'class' NAME ['(' [testlist] ')'] ':' suite
|
||||
class B: pass
|
||||
class B2(): pass
|
||||
class C1(B): pass
|
||||
class C2(B): pass
|
||||
class D(C1, C2, B): pass
|
||||
class C:
|
||||
def meth1(self): pass
|
||||
def meth2(self, arg): pass
|
||||
def meth3(self, a1, a2): pass
|
||||
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
# decorators: decorator+
|
||||
# decorated: decorators (classdef | funcdef)
|
||||
def class_decorator(x):
|
||||
x.decorated = True
|
||||
return x
|
||||
@class_decorator
|
||||
class G:
|
||||
pass
|
||||
self.assertEqual(G.decorated, True)
|
||||
|
||||
def testListcomps(self):
|
||||
# list comprehension tests
|
||||
nums = [1, 2, 3, 4, 5]
|
||||
strs = ["Apple", "Banana", "Coconut"]
|
||||
spcs = [" Apple", " Banana ", "Coco nut "]
|
||||
|
||||
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
|
||||
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
|
||||
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
|
||||
self.assertEqual([(i, s) for i in nums for s in strs],
|
||||
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
|
||||
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
|
||||
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
|
||||
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
|
||||
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
|
||||
|
||||
def test_in_func(l):
|
||||
return [None < x < 3 for x in l if x > 2]
|
||||
|
||||
self.assertEqual(test_in_func(nums), [False, False, False])
|
||||
|
||||
def test_nested_front():
|
||||
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
|
||||
[[1, 2], [3, 4], [5, 6]])
|
||||
|
||||
test_nested_front()
|
||||
|
||||
check_syntax_error(self, "[i, s for i in nums for s in strs]")
|
||||
check_syntax_error(self, "[x if y]")
|
||||
|
||||
suppliers = [
|
||||
(1, "Boeing"),
|
||||
(2, "Ford"),
|
||||
(3, "Macdonalds")
|
||||
]
|
||||
|
||||
parts = [
|
||||
(10, "Airliner"),
|
||||
(20, "Engine"),
|
||||
(30, "Cheeseburger")
|
||||
]
|
||||
|
||||
suppart = [
|
||||
(1, 10), (1, 20), (2, 20), (3, 30)
|
||||
]
|
||||
|
||||
x = [
|
||||
(sname, pname)
|
||||
for (sno, sname) in suppliers
|
||||
for (pno, pname) in parts
|
||||
for (sp_sno, sp_pno) in suppart
|
||||
if sno == sp_sno and pno == sp_pno
|
||||
]
|
||||
|
||||
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
|
||||
('Macdonalds', 'Cheeseburger')])
|
||||
|
||||
def testGenexps(self):
|
||||
# generator expression tests
|
||||
g = ([x for x in range(10)] for x in range(1))
|
||||
self.assertEqual(g.next(), [x for x in range(10)])
|
||||
try:
|
||||
g.next()
|
||||
self.fail('should produce StopIteration exception')
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
a = 1
|
||||
try:
|
||||
g = (a for d in a)
|
||||
g.next()
|
||||
self.fail('should produce TypeError')
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
|
||||
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
|
||||
|
||||
a = [x for x in range(10)]
|
||||
b = (x for x in (y for y in a))
|
||||
self.assertEqual(sum(b), sum([x for x in range(10)]))
|
||||
|
||||
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
|
||||
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
|
||||
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
|
||||
check_syntax_error(self, "foo(x for x in range(10), 100)")
|
||||
check_syntax_error(self, "foo(100, x for x in range(10))")
|
||||
|
||||
def testComprehensionSpecials(self):
|
||||
# test for outmost iterable precomputation
|
||||
x = 10; g = (i for i in range(x)); x = 5
|
||||
self.assertEqual(len(list(g)), 10)
|
||||
|
||||
# This should hold, since we're only precomputing outmost iterable.
|
||||
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
|
||||
x = 5; t = True;
|
||||
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
|
||||
|
||||
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
|
||||
# even though it's silly. Make sure it works (ifelse broke this.)
|
||||
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
|
||||
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
|
||||
|
||||
# verify unpacking single element tuples in listcomp/genexp.
|
||||
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
|
||||
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
|
||||
|
||||
def test_with_statement(self):
|
||||
class manager(object):
|
||||
def __enter__(self):
|
||||
return (1, 2)
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
with manager():
|
||||
pass
|
||||
with manager() as x:
|
||||
pass
|
||||
with manager() as (x, y):
|
||||
pass
|
||||
with manager(), manager():
|
||||
pass
|
||||
with manager() as x, manager() as y:
|
||||
pass
|
||||
with manager() as x, manager():
|
||||
pass
|
||||
|
||||
def testIfElseExpr(self):
|
||||
# Test ifelse expressions in various cases
|
||||
def _checkeval(msg, ret):
|
||||
"helper to check that evaluation of expressions is done correctly"
|
||||
print x
|
||||
return ret
|
||||
|
||||
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
|
||||
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
|
||||
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
|
||||
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
|
||||
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
|
||||
self.assertEqual((5 and 6 if 0 else 1), 1)
|
||||
self.assertEqual(((5 and 6) if 0 else 1), 1)
|
||||
self.assertEqual((5 and (6 if 1 else 1)), 6)
|
||||
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
|
||||
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
|
||||
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
|
||||
self.assertEqual((not 5 if 1 else 1), False)
|
||||
self.assertEqual((not 5 if 0 else 1), 1)
|
||||
self.assertEqual((6 + 1 if 1 else 2), 7)
|
||||
self.assertEqual((6 - 1 if 1 else 2), 5)
|
||||
self.assertEqual((6 * 2 if 1 else 4), 12)
|
||||
self.assertEqual((6 / 2 if 1 else 3), 3)
|
||||
self.assertEqual((6 < 4 if 0 else 2), 2)
|
||||
|
||||
def testStringLiterals(self):
|
||||
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
|
||||
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
|
||||
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
|
||||
x = "doesn't \"shrink\" does it"
|
||||
y = 'doesn\'t "shrink" does it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = "does \"shrink\" doesn't it"
|
||||
y = 'does "shrink" doesn\'t it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = """
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
"""
|
||||
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
|
||||
self.assertEqual(x, y)
|
||||
y = '''
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
'''
|
||||
self.assertEqual(x, y)
|
||||
y = "\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the 'lazy' dog.\n\
|
||||
"
|
||||
self.assertEqual(x, y)
|
||||
y = '\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the \'lazy\' dog.\n\
|
||||
'
|
||||
self.assertEqual(x, y)
|
||||
|
||||
|
||||
|
||||
def test_main():
|
||||
run_unittest(TokenTests, GrammarTests)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_main()
|
||||
+975
@@ -0,0 +1,975 @@
|
||||
# Python test set -- part 1, grammar.
|
||||
# This just tests whether the parser accepts them all.
|
||||
|
||||
# NOTE: When you run this test as a script from the command line, you
|
||||
# get warnings about certain hex/oct constants. Since those are
|
||||
# issued by the parser, you can't suppress them by adding a
|
||||
# filterwarnings() call to this module. Therefore, to shut up the
|
||||
# regression test, the filterwarnings() call has been added to
|
||||
# regrtest.py.
|
||||
|
||||
from test.test_support import run_unittest, check_syntax_error
|
||||
import unittest
|
||||
import sys
|
||||
# testing import *
|
||||
from sys import *
|
||||
|
||||
class TokenTests(unittest.TestCase):
|
||||
|
||||
def testBackslash(self):
|
||||
# Backslash means line continuation:
|
||||
x = 1 \
|
||||
+ 1
|
||||
self.assertEqual(x, 2, 'backslash for line continuation')
|
||||
|
||||
# Backslash does not means continuation in comments :\
|
||||
x = 0
|
||||
self.assertEqual(x, 0, 'backslash ending comment')
|
||||
|
||||
def testPlainIntegers(self):
|
||||
self.assertEqual(0xff, 255)
|
||||
self.assertEqual(0377, 255)
|
||||
self.assertEqual(2147483647, 017777777777)
|
||||
# "0x" is not a valid literal
|
||||
self.assertRaises(SyntaxError, eval, "0x")
|
||||
from sys import maxint
|
||||
if maxint == 2147483647:
|
||||
self.assertEqual(-2147483647-1, -020000000000)
|
||||
# XXX -2147483648
|
||||
self.assert_(037777777777 > 0)
|
||||
self.assert_(0xffffffff > 0)
|
||||
for s in '2147483648', '040000000000', '0x100000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
elif maxint == 9223372036854775807:
|
||||
self.assertEqual(-9223372036854775807-1, -01000000000000000000000)
|
||||
self.assert_(01777777777777777777777 > 0)
|
||||
self.assert_(0xffffffffffffffff > 0)
|
||||
for s in '9223372036854775808', '02000000000000000000000', \
|
||||
'0x10000000000000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
else:
|
||||
self.fail('Weird maxint value %r' % maxint)
|
||||
|
||||
def testLongIntegers(self):
|
||||
x = 0L
|
||||
x = 0l
|
||||
x = 0xffffffffffffffffL
|
||||
x = 0xffffffffffffffffl
|
||||
x = 077777777777777777L
|
||||
x = 077777777777777777l
|
||||
x = 123456789012345678901234567890L
|
||||
x = 123456789012345678901234567890l
|
||||
|
||||
def testFloats(self):
|
||||
x = 3.14
|
||||
x = 314.
|
||||
x = 0.314
|
||||
# XXX x = 000.314
|
||||
x = .314
|
||||
x = 3e14
|
||||
x = 3E14
|
||||
x = 3e-14
|
||||
x = 3e+14
|
||||
x = 3.e14
|
||||
x = .3e14
|
||||
x = 3.1e4
|
||||
|
||||
class GrammarTests(unittest.TestCase):
|
||||
|
||||
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
||||
# XXX can't test in a script -- this rule is only used when interactive
|
||||
|
||||
# file_input: (NEWLINE | stmt)* ENDMARKER
|
||||
# Being tested as this very moment this very module
|
||||
|
||||
# expr_input: testlist NEWLINE
|
||||
# XXX Hard to test -- used only in calls to input()
|
||||
|
||||
def testEvalInput(self):
|
||||
# testlist ENDMARKER
|
||||
x = eval('1, 0 or 1')
|
||||
|
||||
def testFuncdef(self):
|
||||
### 'def' NAME parameters ':' suite
|
||||
### parameters: '(' [varargslist] ')'
|
||||
### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
|
||||
### | ('**'|'*' '*') NAME)
|
||||
### | fpdef ['=' test] (',' fpdef ['=' test])* [',']
|
||||
### fpdef: NAME | '(' fplist ')'
|
||||
### fplist: fpdef (',' fpdef)* [',']
|
||||
### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
|
||||
### argument: [test '='] test # Really [keyword '='] test
|
||||
def f1(): pass
|
||||
f1()
|
||||
f1(*())
|
||||
f1(*(), **{})
|
||||
def f2(one_argument): pass
|
||||
def f3(two, arguments): pass
|
||||
def f4(two, (compound, (argument, list))): pass
|
||||
def f5((compound, first), two): pass
|
||||
self.assertEqual(f2.func_code.co_varnames, ('one_argument',))
|
||||
self.assertEqual(f3.func_code.co_varnames, ('two', 'arguments'))
|
||||
if sys.platform.startswith('java'):
|
||||
self.assertEqual(f4.func_code.co_varnames,
|
||||
('two', '(compound, (argument, list))', 'compound', 'argument',
|
||||
'list',))
|
||||
self.assertEqual(f5.func_code.co_varnames,
|
||||
('(compound, first)', 'two', 'compound', 'first'))
|
||||
else:
|
||||
self.assertEqual(f4.func_code.co_varnames,
|
||||
('two', '.1', 'compound', 'argument', 'list'))
|
||||
self.assertEqual(f5.func_code.co_varnames,
|
||||
('.0', 'two', 'compound', 'first'))
|
||||
def a1(one_arg,): pass
|
||||
def a2(two, args,): pass
|
||||
def v0(*rest): pass
|
||||
def v1(a, *rest): pass
|
||||
def v2(a, b, *rest): pass
|
||||
def v3(a, (b, c), *rest): return a, b, c, rest
|
||||
|
||||
f1()
|
||||
f2(1)
|
||||
f2(1,)
|
||||
f3(1, 2)
|
||||
f3(1, 2,)
|
||||
f4(1, (2, (3, 4)))
|
||||
v0()
|
||||
v0(1)
|
||||
v0(1,)
|
||||
v0(1,2)
|
||||
v0(1,2,3,4,5,6,7,8,9,0)
|
||||
v1(1)
|
||||
v1(1,)
|
||||
v1(1,2)
|
||||
v1(1,2,3)
|
||||
v1(1,2,3,4,5,6,7,8,9,0)
|
||||
v2(1,2)
|
||||
v2(1,2,3)
|
||||
v2(1,2,3,4)
|
||||
v2(1,2,3,4,5,6,7,8,9,0)
|
||||
v3(1,(2,3))
|
||||
v3(1,(2,3),4)
|
||||
v3(1,(2,3),4,5,6,7,8,9,0)
|
||||
|
||||
# ceval unpacks the formal arguments into the first argcount names;
|
||||
# thus, the names nested inside tuples must appear after these names.
|
||||
if sys.platform.startswith('java'):
|
||||
self.assertEqual(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
|
||||
else:
|
||||
self.assertEqual(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
|
||||
self.assertEqual(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
|
||||
def d01(a=1): pass
|
||||
d01()
|
||||
d01(1)
|
||||
d01(*(1,))
|
||||
d01(**{'a':2})
|
||||
def d11(a, b=1): pass
|
||||
d11(1)
|
||||
d11(1, 2)
|
||||
d11(1, **{'b':2})
|
||||
def d21(a, b, c=1): pass
|
||||
d21(1, 2)
|
||||
d21(1, 2, 3)
|
||||
d21(*(1, 2, 3))
|
||||
d21(1, *(2, 3))
|
||||
d21(1, 2, *(3,))
|
||||
d21(1, 2, **{'c':3})
|
||||
def d02(a=1, b=2): pass
|
||||
d02()
|
||||
d02(1)
|
||||
d02(1, 2)
|
||||
d02(*(1, 2))
|
||||
d02(1, *(2,))
|
||||
d02(1, **{'b':2})
|
||||
d02(**{'a': 1, 'b': 2})
|
||||
def d12(a, b=1, c=2): pass
|
||||
d12(1)
|
||||
d12(1, 2)
|
||||
d12(1, 2, 3)
|
||||
def d22(a, b, c=1, d=2): pass
|
||||
d22(1, 2)
|
||||
d22(1, 2, 3)
|
||||
d22(1, 2, 3, 4)
|
||||
def d01v(a=1, *rest): pass
|
||||
d01v()
|
||||
d01v(1)
|
||||
d01v(1, 2)
|
||||
d01v(*(1, 2, 3, 4))
|
||||
d01v(*(1,))
|
||||
d01v(**{'a':2})
|
||||
def d11v(a, b=1, *rest): pass
|
||||
d11v(1)
|
||||
d11v(1, 2)
|
||||
d11v(1, 2, 3)
|
||||
def d21v(a, b, c=1, *rest): pass
|
||||
d21v(1, 2)
|
||||
d21v(1, 2, 3)
|
||||
d21v(1, 2, 3, 4)
|
||||
d21v(*(1, 2, 3, 4))
|
||||
d21v(1, 2, **{'c': 3})
|
||||
def d02v(a=1, b=2, *rest): pass
|
||||
d02v()
|
||||
d02v(1)
|
||||
d02v(1, 2)
|
||||
d02v(1, 2, 3)
|
||||
d02v(1, *(2, 3, 4))
|
||||
d02v(**{'a': 1, 'b': 2})
|
||||
def d12v(a, b=1, c=2, *rest): pass
|
||||
d12v(1)
|
||||
d12v(1, 2)
|
||||
d12v(1, 2, 3)
|
||||
d12v(1, 2, 3, 4)
|
||||
d12v(*(1, 2, 3, 4))
|
||||
d12v(1, 2, *(3, 4, 5))
|
||||
d12v(1, *(2,), **{'c': 3})
|
||||
def d22v(a, b, c=1, d=2, *rest): pass
|
||||
d22v(1, 2)
|
||||
d22v(1, 2, 3)
|
||||
d22v(1, 2, 3, 4)
|
||||
d22v(1, 2, 3, 4, 5)
|
||||
d22v(*(1, 2, 3, 4))
|
||||
d22v(1, 2, *(3, 4, 5))
|
||||
d22v(1, *(2, 3), **{'d': 4})
|
||||
def d31v((x)): pass
|
||||
d31v(1)
|
||||
def d32v((x,)): pass
|
||||
d32v((1,))
|
||||
|
||||
# keyword arguments after *arglist
|
||||
def f(*args, **kwargs):
|
||||
return args, kwargs
|
||||
self.assertEqual(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
|
||||
{'x':2, 'y':5}))
|
||||
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
|
||||
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
|
||||
|
||||
# Check ast errors in *args and *kwargs
|
||||
check_syntax_error(self, "f(*g(1=2))")
|
||||
check_syntax_error(self, "f(**g(1=2))")
|
||||
|
||||
def testLambdef(self):
|
||||
### lambdef: 'lambda' [varargslist] ':' test
|
||||
l1 = lambda : 0
|
||||
self.assertEqual(l1(), 0)
|
||||
l2 = lambda : a[d] # XXX just testing the expression
|
||||
l3 = lambda : [2 < x for x in [-1, 3, 0L]]
|
||||
self.assertEqual(l3(), [0, 1, 0])
|
||||
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
|
||||
self.assertEqual(l4(), 1)
|
||||
l5 = lambda x, y, z=2: x + y + z
|
||||
self.assertEqual(l5(1, 2), 5)
|
||||
self.assertEqual(l5(1, 2, 3), 6)
|
||||
check_syntax_error(self, "lambda x: x = 2")
|
||||
check_syntax_error(self, "lambda (None,): None")
|
||||
|
||||
### stmt: simple_stmt | compound_stmt
|
||||
# Tested below
|
||||
|
||||
def testSimpleStmt(self):
|
||||
### simple_stmt: small_stmt (';' small_stmt)* [';']
|
||||
x = 1; pass; del x
|
||||
def foo():
|
||||
# verify statements that end with semi-colons
|
||||
x = 1; pass; del x;
|
||||
foo()
|
||||
|
||||
### small_stmt: expr_stmt | print_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
|
||||
# Tested below
|
||||
|
||||
def testExprStmt(self):
|
||||
# (exprlist '=')* exprlist
|
||||
1
|
||||
1, 2, 3
|
||||
x = 1
|
||||
x = 1, 2, 3
|
||||
x = y = z = 1, 2, 3
|
||||
x, y, z = 1, 2, 3
|
||||
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
|
||||
|
||||
check_syntax_error(self, "x + 1 = 1")
|
||||
check_syntax_error(self, "a + 1 = b + 2")
|
||||
|
||||
def testPrintStmt(self):
|
||||
# 'print' (test ',')* [test]
|
||||
import StringIO
|
||||
|
||||
# Can't test printing to real stdout without comparing output
|
||||
# which is not available in unittest.
|
||||
save_stdout = sys.stdout
|
||||
sys.stdout = StringIO.StringIO()
|
||||
|
||||
print 1, 2, 3
|
||||
print 1, 2, 3,
|
||||
print
|
||||
print 0 or 1, 0 or 1,
|
||||
print 0 or 1
|
||||
|
||||
# 'print' '>>' test ','
|
||||
print >> sys.stdout, 1, 2, 3
|
||||
print >> sys.stdout, 1, 2, 3,
|
||||
print >> sys.stdout
|
||||
print >> sys.stdout, 0 or 1, 0 or 1,
|
||||
print >> sys.stdout, 0 or 1
|
||||
|
||||
# test printing to an instance
|
||||
class Gulp:
|
||||
def write(self, msg): pass
|
||||
|
||||
gulp = Gulp()
|
||||
print >> gulp, 1, 2, 3
|
||||
print >> gulp, 1, 2, 3,
|
||||
print >> gulp
|
||||
print >> gulp, 0 or 1, 0 or 1,
|
||||
print >> gulp, 0 or 1
|
||||
|
||||
# test print >> None
|
||||
def driver():
|
||||
oldstdout = sys.stdout
|
||||
sys.stdout = Gulp()
|
||||
try:
|
||||
tellme(Gulp())
|
||||
tellme()
|
||||
finally:
|
||||
sys.stdout = oldstdout
|
||||
|
||||
# we should see this once
|
||||
def tellme(file=sys.stdout):
|
||||
print >> file, 'hello world'
|
||||
|
||||
driver()
|
||||
|
||||
# we should not see this at all
|
||||
def tellme(file=None):
|
||||
print >> file, 'goodbye universe'
|
||||
|
||||
driver()
|
||||
|
||||
self.assertEqual(sys.stdout.getvalue(), '''\
|
||||
1 2 3
|
||||
1 2 3
|
||||
1 1 1
|
||||
1 2 3
|
||||
1 2 3
|
||||
1 1 1
|
||||
hello world
|
||||
''')
|
||||
sys.stdout = save_stdout
|
||||
|
||||
# syntax errors
|
||||
check_syntax_error(self, 'print ,')
|
||||
check_syntax_error(self, 'print >> x,')
|
||||
|
||||
def testDelStmt(self):
|
||||
# 'del' exprlist
|
||||
abc = [1,2,3]
|
||||
x, y, z = abc
|
||||
xyz = x, y, z
|
||||
|
||||
del abc
|
||||
del x, y, (z, xyz)
|
||||
|
||||
def testPassStmt(self):
|
||||
# 'pass'
|
||||
pass
|
||||
|
||||
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
|
||||
# Tested below
|
||||
|
||||
def testBreakStmt(self):
|
||||
# 'break'
|
||||
while 1: break
|
||||
|
||||
def testContinueStmt(self):
|
||||
# 'continue'
|
||||
i = 1
|
||||
while i: i = 0; continue
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "ok"
|
||||
try:
|
||||
continue
|
||||
msg = "continue failed to continue inside try"
|
||||
except:
|
||||
msg = "continue inside try called except block"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "finally block not called"
|
||||
try:
|
||||
continue
|
||||
finally:
|
||||
msg = "ok"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
def test_break_continue_loop(self):
|
||||
# This test warrants an explanation. It is a test specifically for SF bugs
|
||||
# #463359 and #462937. The bug is that a 'break' statement executed or
|
||||
# exception raised inside a try/except inside a loop, *after* a continue
|
||||
# statement has been executed in that loop, will cause the wrong number of
|
||||
# arguments to be popped off the stack and the instruction pointer reset to
|
||||
# a very small number (usually 0.) Because of this, the following test
|
||||
# *must* written as a function, and the tracking vars *must* be function
|
||||
# arguments with default values. Otherwise, the test will loop and loop.
|
||||
|
||||
def test_inner(extra_burning_oil = 1, count=0):
|
||||
big_hippo = 2
|
||||
while big_hippo:
|
||||
count += 1
|
||||
try:
|
||||
if extra_burning_oil and big_hippo == 1:
|
||||
extra_burning_oil -= 1
|
||||
break
|
||||
big_hippo -= 1
|
||||
continue
|
||||
except:
|
||||
raise
|
||||
if count > 2 or big_hippo <> 1:
|
||||
self.fail("continue then break in try/except in loop broken!")
|
||||
test_inner()
|
||||
|
||||
def testReturn(self):
|
||||
# 'return' [testlist]
|
||||
def g1(): return
|
||||
def g2(): return 1
|
||||
g1()
|
||||
x = g2()
|
||||
check_syntax_error(self, "class foo:return 1")
|
||||
|
||||
def testYield(self):
|
||||
check_syntax_error(self, "class foo:yield 1")
|
||||
|
||||
def testRaise(self):
|
||||
# 'raise' test [',' test]
|
||||
try: raise RuntimeError, 'just testing'
|
||||
except RuntimeError: pass
|
||||
try: raise KeyboardInterrupt
|
||||
except KeyboardInterrupt: pass
|
||||
|
||||
def testImport(self):
|
||||
# 'import' dotted_as_names
|
||||
import sys
|
||||
import time, sys
|
||||
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
|
||||
from time import time
|
||||
from time import (time)
|
||||
# not testable inside a function, but already done at top of the module
|
||||
# from sys import *
|
||||
from sys import path, argv
|
||||
from sys import (path, argv)
|
||||
from sys import (path, argv,)
|
||||
|
||||
def testGlobal(self):
|
||||
# 'global' NAME (',' NAME)*
|
||||
global a
|
||||
global a, b
|
||||
global one, two, three, four, five, six, seven, eight, nine, ten
|
||||
|
||||
def testExec(self):
|
||||
# 'exec' expr ['in' expr [',' expr]]
|
||||
z = None
|
||||
del z
|
||||
exec 'z=1+1\n'
|
||||
if z != 2: self.fail('exec \'z=1+1\'\\n')
|
||||
del z
|
||||
exec 'z=1+1'
|
||||
if z != 2: self.fail('exec \'z=1+1\'')
|
||||
z = None
|
||||
del z
|
||||
import types
|
||||
if hasattr(types, "UnicodeType"):
|
||||
exec r"""if 1:
|
||||
exec u'z=1+1\n'
|
||||
if z != 2: self.fail('exec u\'z=1+1\'\\n')
|
||||
del z
|
||||
exec u'z=1+1'
|
||||
if z != 2: self.fail('exec u\'z=1+1\'')"""
|
||||
g = {}
|
||||
exec 'z = 1' in g
|
||||
if g.has_key('__builtins__'): del g['__builtins__']
|
||||
if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
|
||||
g = {}
|
||||
l = {}
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", "global statement", module="<string>")
|
||||
exec 'global a; a = 1; b = 2' in g, l
|
||||
if g.has_key('__builtins__'): del g['__builtins__']
|
||||
if l.has_key('__builtins__'): del l['__builtins__']
|
||||
if (g, l) != ({'a':1}, {'b':2}):
|
||||
self.fail('exec ... in g (%s), l (%s)' %(g,l))
|
||||
|
||||
def testAssert(self):
|
||||
# assert_stmt: 'assert' test [',' test]
|
||||
assert 1
|
||||
assert 1, 1
|
||||
assert lambda x:x
|
||||
assert 1, lambda x:x+1
|
||||
try:
|
||||
assert 0, "msg"
|
||||
except AssertionError, e:
|
||||
self.assertEqual(e.args[0], "msg")
|
||||
else:
|
||||
if __debug__:
|
||||
self.fail("AssertionError not raised by assert 0")
|
||||
|
||||
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
|
||||
# Tested below
|
||||
|
||||
def testIf(self):
|
||||
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||
if 1: pass
|
||||
if 1: pass
|
||||
else: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
else: pass
|
||||
|
||||
def testWhile(self):
|
||||
# 'while' test ':' suite ['else' ':' suite]
|
||||
while 0: pass
|
||||
while 0: pass
|
||||
else: pass
|
||||
|
||||
# Issue1920: "while 0" is optimized away,
|
||||
# ensure that the "else" clause is still present.
|
||||
x = 0
|
||||
while 0:
|
||||
x = 1
|
||||
else:
|
||||
x = 2
|
||||
self.assertEqual(x, 2)
|
||||
|
||||
def testFor(self):
|
||||
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
|
||||
for i in 1, 2, 3: pass
|
||||
for i, j, k in (): pass
|
||||
else: pass
|
||||
class Squares:
|
||||
def __init__(self, max):
|
||||
self.max = max
|
||||
self.sofar = []
|
||||
def __len__(self): return len(self.sofar)
|
||||
def __getitem__(self, i):
|
||||
if not 0 <= i < self.max: raise IndexError
|
||||
n = len(self.sofar)
|
||||
while n <= i:
|
||||
self.sofar.append(n*n)
|
||||
n = n+1
|
||||
return self.sofar[i]
|
||||
n = 0
|
||||
for x in Squares(10): n = n+x
|
||||
if n != 285:
|
||||
self.fail('for over growing sequence')
|
||||
|
||||
result = []
|
||||
for x, in [(1,), (2,), (3,)]:
|
||||
result.append(x)
|
||||
self.assertEqual(result, [1, 2, 3])
|
||||
|
||||
def testTry(self):
|
||||
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
|
||||
### | 'try' ':' suite 'finally' ':' suite
|
||||
### except_clause: 'except' [expr [('as' | ',') expr]]
|
||||
try:
|
||||
1/0
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
try: 1/0
|
||||
except EOFError: pass
|
||||
except TypeError as msg: pass
|
||||
except RuntimeError, msg: pass
|
||||
except: pass
|
||||
else: pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError): pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError), msg: pass
|
||||
try: pass
|
||||
finally: pass
|
||||
|
||||
def testSuite(self):
|
||||
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
|
||||
if 1: pass
|
||||
if 1:
|
||||
pass
|
||||
if 1:
|
||||
#
|
||||
#
|
||||
#
|
||||
pass
|
||||
pass
|
||||
#
|
||||
pass
|
||||
#
|
||||
|
||||
def testTest(self):
|
||||
### and_test ('or' and_test)*
|
||||
### and_test: not_test ('and' not_test)*
|
||||
### not_test: 'not' not_test | comparison
|
||||
if not 1: pass
|
||||
if 1 and 1: pass
|
||||
if 1 or 1: pass
|
||||
if not not not 1: pass
|
||||
if not 1 and 1 and 1: pass
|
||||
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
|
||||
|
||||
def testComparison(self):
|
||||
### comparison: expr (comp_op expr)*
|
||||
### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||||
if 1: pass
|
||||
x = (1 == 1)
|
||||
if 1 == 1: pass
|
||||
if 1 != 1: pass
|
||||
if 1 <> 1: pass
|
||||
if 1 < 1: pass
|
||||
if 1 > 1: pass
|
||||
if 1 <= 1: pass
|
||||
if 1 >= 1: pass
|
||||
if 1 is 1: pass
|
||||
if 1 is not 1: pass
|
||||
if 1 in (): pass
|
||||
if 1 not in (): pass
|
||||
if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
|
||||
|
||||
def testBinaryMaskOps(self):
|
||||
x = 1 & 1
|
||||
x = 1 ^ 1
|
||||
x = 1 | 1
|
||||
|
||||
def testShiftOps(self):
|
||||
x = 1 << 1
|
||||
x = 1 >> 1
|
||||
x = 1 << 1 >> 1
|
||||
|
||||
def testAdditiveOps(self):
|
||||
x = 1
|
||||
x = 1 + 1
|
||||
x = 1 - 1 - 1
|
||||
x = 1 - 1 + 1 - 1 + 1
|
||||
|
||||
def testMultiplicativeOps(self):
|
||||
x = 1 * 1
|
||||
x = 1 / 1
|
||||
x = 1 % 1
|
||||
x = 1 / 1 * 1 % 1
|
||||
|
||||
def testUnaryOps(self):
|
||||
x = +1
|
||||
x = -1
|
||||
x = ~1
|
||||
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
|
||||
x = -1*1/1 + 1*1 - ---1*1
|
||||
|
||||
def testSelectors(self):
|
||||
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
|
||||
### subscript: expr | [expr] ':' [expr]
|
||||
|
||||
import sys, time
|
||||
c = sys.path[0]
|
||||
x = time.time()
|
||||
x = sys.modules['time'].time()
|
||||
a = '01234'
|
||||
c = a[0]
|
||||
c = a[-1]
|
||||
s = a[0:5]
|
||||
s = a[:5]
|
||||
s = a[0:]
|
||||
s = a[:]
|
||||
s = a[-5:]
|
||||
s = a[:-1]
|
||||
s = a[-4:-3]
|
||||
# A rough test of SF bug 1333982. http://python.org/sf/1333982
|
||||
# The testing here is fairly incomplete.
|
||||
# Test cases should include: commas with 1 and 2 colons
|
||||
d = {}
|
||||
d[1] = 1
|
||||
d[1,] = 2
|
||||
d[1,2] = 3
|
||||
d[1,2,3] = 4
|
||||
L = list(d)
|
||||
L.sort()
|
||||
self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
|
||||
|
||||
def testAtoms(self):
|
||||
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
|
||||
### dictmaker: test ':' test (',' test ':' test)* [',']
|
||||
|
||||
x = (1)
|
||||
x = (1 or 2 or 3)
|
||||
x = (1 or 2 or 3, 2, 3)
|
||||
|
||||
x = []
|
||||
x = [1]
|
||||
x = [1 or 2 or 3]
|
||||
x = [1 or 2 or 3, 2, 3]
|
||||
x = []
|
||||
|
||||
x = {}
|
||||
x = {'one': 1}
|
||||
x = {'one': 1,}
|
||||
x = {'one' or 'two': 1 or 2}
|
||||
x = {'one': 1, 'two': 2}
|
||||
x = {'one': 1, 'two': 2,}
|
||||
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
|
||||
|
||||
x = `x`
|
||||
x = `1 or 2 or 3`
|
||||
self.assertEqual(`1,2`, '(1, 2)')
|
||||
|
||||
x = x
|
||||
x = 'x'
|
||||
x = 123
|
||||
|
||||
### exprlist: expr (',' expr)* [',']
|
||||
### testlist: test (',' test)* [',']
|
||||
# These have been exercised enough above
|
||||
|
||||
def testClassdef(self):
|
||||
# 'class' NAME ['(' [testlist] ')'] ':' suite
|
||||
class B: pass
|
||||
class B2(): pass
|
||||
class C1(B): pass
|
||||
class C2(B): pass
|
||||
class D(C1, C2, B): pass
|
||||
class C:
|
||||
def meth1(self): pass
|
||||
def meth2(self, arg): pass
|
||||
def meth3(self, a1, a2): pass
|
||||
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
# decorators: decorator+
|
||||
# decorated: decorators (classdef | funcdef)
|
||||
def class_decorator(x):
|
||||
x.decorated = True
|
||||
return x
|
||||
@class_decorator
|
||||
class G:
|
||||
pass
|
||||
self.assertEqual(G.decorated, True)
|
||||
|
||||
def testListcomps(self):
|
||||
# list comprehension tests
|
||||
nums = [1, 2, 3, 4, 5]
|
||||
strs = ["Apple", "Banana", "Coconut"]
|
||||
spcs = [" Apple", " Banana ", "Coco nut "]
|
||||
|
||||
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
|
||||
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
|
||||
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
|
||||
self.assertEqual([(i, s) for i in nums for s in strs],
|
||||
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
|
||||
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
|
||||
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
|
||||
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
|
||||
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
|
||||
|
||||
def test_in_func(l):
|
||||
return [None < x < 3 for x in l if x > 2]
|
||||
|
||||
self.assertEqual(test_in_func(nums), [False, False, False])
|
||||
|
||||
def test_nested_front():
|
||||
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
|
||||
[[1, 2], [3, 4], [5, 6]])
|
||||
|
||||
test_nested_front()
|
||||
|
||||
check_syntax_error(self, "[i, s for i in nums for s in strs]")
|
||||
check_syntax_error(self, "[x if y]")
|
||||
|
||||
suppliers = [
|
||||
(1, "Boeing"),
|
||||
(2, "Ford"),
|
||||
(3, "Macdonalds")
|
||||
]
|
||||
|
||||
parts = [
|
||||
(10, "Airliner"),
|
||||
(20, "Engine"),
|
||||
(30, "Cheeseburger")
|
||||
]
|
||||
|
||||
suppart = [
|
||||
(1, 10), (1, 20), (2, 20), (3, 30)
|
||||
]
|
||||
|
||||
x = [
|
||||
(sname, pname)
|
||||
for (sno, sname) in suppliers
|
||||
for (pno, pname) in parts
|
||||
for (sp_sno, sp_pno) in suppart
|
||||
if sno == sp_sno and pno == sp_pno
|
||||
]
|
||||
|
||||
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
|
||||
('Macdonalds', 'Cheeseburger')])
|
||||
|
||||
def testGenexps(self):
|
||||
# generator expression tests
|
||||
g = ([x for x in range(10)] for x in range(1))
|
||||
self.assertEqual(g.next(), [x for x in range(10)])
|
||||
try:
|
||||
g.next()
|
||||
self.fail('should produce StopIteration exception')
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
a = 1
|
||||
try:
|
||||
g = (a for d in a)
|
||||
g.next()
|
||||
self.fail('should produce TypeError')
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
|
||||
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
|
||||
|
||||
a = [x for x in range(10)]
|
||||
b = (x for x in (y for y in a))
|
||||
self.assertEqual(sum(b), sum([x for x in range(10)]))
|
||||
|
||||
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
|
||||
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
|
||||
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
|
||||
check_syntax_error(self, "foo(x for x in range(10), 100)")
|
||||
check_syntax_error(self, "foo(100, x for x in range(10))")
|
||||
|
||||
def testComprehensionSpecials(self):
|
||||
# test for outmost iterable precomputation
|
||||
x = 10; g = (i for i in range(x)); x = 5
|
||||
self.assertEqual(len(list(g)), 10)
|
||||
|
||||
# This should hold, since we're only precomputing outmost iterable.
|
||||
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
|
||||
x = 5; t = True;
|
||||
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
|
||||
|
||||
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
|
||||
# even though it's silly. Make sure it works (ifelse broke this.)
|
||||
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
|
||||
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
|
||||
|
||||
# verify unpacking single element tuples in listcomp/genexp.
|
||||
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
|
||||
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
|
||||
|
||||
def test_with_statement(self):
|
||||
class manager(object):
|
||||
def __enter__(self):
|
||||
return (1, 2)
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
with manager():
|
||||
pass
|
||||
with manager() as x:
|
||||
pass
|
||||
with manager() as (x, y):
|
||||
pass
|
||||
with manager(), manager():
|
||||
pass
|
||||
with manager() as x, manager() as y:
|
||||
pass
|
||||
with manager() as x, manager():
|
||||
pass
|
||||
|
||||
def testIfElseExpr(self):
|
||||
# Test ifelse expressions in various cases
|
||||
def _checkeval(msg, ret):
|
||||
"helper to check that evaluation of expressions is done correctly"
|
||||
print x
|
||||
return ret
|
||||
|
||||
self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
|
||||
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
|
||||
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
|
||||
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
|
||||
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
|
||||
self.assertEqual((5 and 6 if 0 else 1), 1)
|
||||
self.assertEqual(((5 and 6) if 0 else 1), 1)
|
||||
self.assertEqual((5 and (6 if 1 else 1)), 6)
|
||||
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
|
||||
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
|
||||
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
|
||||
self.assertEqual((not 5 if 1 else 1), False)
|
||||
self.assertEqual((not 5 if 0 else 1), 1)
|
||||
self.assertEqual((6 + 1 if 1 else 2), 7)
|
||||
self.assertEqual((6 - 1 if 1 else 2), 5)
|
||||
self.assertEqual((6 * 2 if 1 else 4), 12)
|
||||
self.assertEqual((6 / 2 if 1 else 3), 3)
|
||||
self.assertEqual((6 < 4 if 0 else 2), 2)
|
||||
|
||||
def testStringLiterals(self):
|
||||
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
|
||||
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
|
||||
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
|
||||
x = "doesn't \"shrink\" does it"
|
||||
y = 'doesn\'t "shrink" does it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = "does \"shrink\" doesn't it"
|
||||
y = 'does "shrink" doesn\'t it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = """
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
"""
|
||||
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
|
||||
self.assertEqual(x, y)
|
||||
y = '''
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
'''
|
||||
self.assertEqual(x, y)
|
||||
y = "\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the 'lazy' dog.\n\
|
||||
"
|
||||
self.assertEqual(x, y)
|
||||
y = '\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the \'lazy\' dog.\n\
|
||||
'
|
||||
self.assertEqual(x, y)
|
||||
|
||||
|
||||
|
||||
def test_main():
|
||||
run_unittest(TokenTests, GrammarTests)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_main()
|
||||
|
||||
Vendored
+945
@@ -0,0 +1,945 @@
|
||||
# Python test set -- part 1, grammar.
|
||||
# This just tests whether the parser accepts them all.
|
||||
|
||||
# NOTE: When you run this test as a script from the command line, you
|
||||
# get warnings about certain hex/oct constants. Since those are
|
||||
# issued by the parser, you can't suppress them by adding a
|
||||
# filterwarnings() call to this module. Therefore, to shut up the
|
||||
# regression test, the filterwarnings() call has been added to
|
||||
# regrtest.py.
|
||||
|
||||
from test.support import run_unittest, check_syntax_error
|
||||
import unittest
|
||||
import sys
|
||||
# testing import *
|
||||
from sys import *
|
||||
|
||||
class TokenTests(unittest.TestCase):
|
||||
|
||||
def testBackslash(self):
|
||||
# Backslash means line continuation:
|
||||
x = 1 \
|
||||
+ 1
|
||||
self.assertEqual(x, 2, 'backslash for line continuation')
|
||||
|
||||
# Backslash does not means continuation in comments :\
|
||||
x = 0
|
||||
self.assertEqual(x, 0, 'backslash ending comment')
|
||||
|
||||
def testPlainIntegers(self):
|
||||
self.assertEqual(type(000), type(0))
|
||||
self.assertEqual(0xff, 255)
|
||||
self.assertEqual(0o377, 255)
|
||||
self.assertEqual(2147483647, 0o17777777777)
|
||||
self.assertEqual(0b1001, 9)
|
||||
# "0x" is not a valid literal
|
||||
self.assertRaises(SyntaxError, eval, "0x")
|
||||
from sys import maxsize
|
||||
if maxsize == 2147483647:
|
||||
self.assertEqual(-2147483647-1, -0o20000000000)
|
||||
# XXX -2147483648
|
||||
self.assert_(0o37777777777 > 0)
|
||||
self.assert_(0xffffffff > 0)
|
||||
self.assert_(0b1111111111111111111111111111111 > 0)
|
||||
for s in ('2147483648', '0o40000000000', '0x100000000',
|
||||
'0b10000000000000000000000000000000'):
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
elif maxsize == 9223372036854775807:
|
||||
self.assertEqual(-9223372036854775807-1, -0o1000000000000000000000)
|
||||
self.assert_(0o1777777777777777777777 > 0)
|
||||
self.assert_(0xffffffffffffffff > 0)
|
||||
self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
|
||||
for s in '9223372036854775808', '0o2000000000000000000000', \
|
||||
'0x10000000000000000', \
|
||||
'0b100000000000000000000000000000000000000000000000000000000000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
else:
|
||||
self.fail('Weird maxsize value %r' % maxsize)
|
||||
|
||||
def testLongIntegers(self):
|
||||
x = 0
|
||||
x = 0xffffffffffffffff
|
||||
x = 0Xffffffffffffffff
|
||||
x = 0o77777777777777777
|
||||
x = 0O77777777777777777
|
||||
x = 123456789012345678901234567890
|
||||
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
|
||||
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
|
||||
|
||||
def testUnderscoresInNumbers(self):
|
||||
# Integers
|
||||
x = 1_0
|
||||
x = 123_456_7_89
|
||||
x = 0xabc_123_4_5
|
||||
x = 0X_abc_123
|
||||
x = 0B11_01
|
||||
x = 0b_11_01
|
||||
x = 0o45_67
|
||||
x = 0O_45_67
|
||||
|
||||
# Floats
|
||||
x = 3_1.4
|
||||
x = 03_1.4
|
||||
x = 3_1.
|
||||
x = .3_1
|
||||
x = 3.1_4
|
||||
x = 0_3.1_4
|
||||
x = 3e1_4
|
||||
x = 3_1e+4_1
|
||||
x = 3_1E-4_1
|
||||
|
||||
def testFloats(self):
|
||||
x = 3.14
|
||||
x = 314.
|
||||
x = 0.314
|
||||
# XXX x = 000.314
|
||||
x = .314
|
||||
x = 3e14
|
||||
x = 3E14
|
||||
x = 3e-14
|
||||
x = 3e+14
|
||||
x = 3.e14
|
||||
x = .3e14
|
||||
x = 3.1e4
|
||||
|
||||
def testEllipsis(self):
|
||||
x = ...
|
||||
self.assert_(x is Ellipsis)
|
||||
self.assertRaises(SyntaxError, eval, ".. .")
|
||||
|
||||
class GrammarTests(unittest.TestCase):
|
||||
|
||||
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
||||
# XXX can't test in a script -- this rule is only used when interactive
|
||||
|
||||
# file_input: (NEWLINE | stmt)* ENDMARKER
|
||||
# Being tested as this very moment this very module
|
||||
|
||||
# expr_input: testlist NEWLINE
|
||||
# XXX Hard to test -- used only in calls to input()
|
||||
|
||||
def testEvalInput(self):
|
||||
# testlist ENDMARKER
|
||||
x = eval('1, 0 or 1')
|
||||
|
||||
def testFuncdef(self):
|
||||
### [decorators] 'def' NAME parameters ['->' test] ':' suite
|
||||
### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
### decorators: decorator+
|
||||
### parameters: '(' [typedargslist] ')'
|
||||
### typedargslist: ((tfpdef ['=' test] ',')*
|
||||
### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
|
||||
### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
|
||||
### tfpdef: NAME [':' test]
|
||||
### varargslist: ((vfpdef ['=' test] ',')*
|
||||
### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
|
||||
### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
|
||||
### vfpdef: NAME
|
||||
def f1(): pass
|
||||
f1()
|
||||
f1(*())
|
||||
f1(*(), **{})
|
||||
def f2(one_argument): pass
|
||||
def f3(two, arguments): pass
|
||||
self.assertEqual(f2.__code__.co_varnames, ('one_argument',))
|
||||
self.assertEqual(f3.__code__.co_varnames, ('two', 'arguments'))
|
||||
def a1(one_arg,): pass
|
||||
def a2(two, args,): pass
|
||||
def v0(*rest): pass
|
||||
def v1(a, *rest): pass
|
||||
def v2(a, b, *rest): pass
|
||||
|
||||
f1()
|
||||
f2(1)
|
||||
f2(1,)
|
||||
f3(1, 2)
|
||||
f3(1, 2,)
|
||||
v0()
|
||||
v0(1)
|
||||
v0(1,)
|
||||
v0(1,2)
|
||||
v0(1,2,3,4,5,6,7,8,9,0)
|
||||
v1(1)
|
||||
v1(1,)
|
||||
v1(1,2)
|
||||
v1(1,2,3)
|
||||
v1(1,2,3,4,5,6,7,8,9,0)
|
||||
v2(1,2)
|
||||
v2(1,2,3)
|
||||
v2(1,2,3,4)
|
||||
v2(1,2,3,4,5,6,7,8,9,0)
|
||||
|
||||
def d01(a=1): pass
|
||||
d01()
|
||||
d01(1)
|
||||
d01(*(1,))
|
||||
d01(**{'a':2})
|
||||
def d11(a, b=1): pass
|
||||
d11(1)
|
||||
d11(1, 2)
|
||||
d11(1, **{'b':2})
|
||||
def d21(a, b, c=1): pass
|
||||
d21(1, 2)
|
||||
d21(1, 2, 3)
|
||||
d21(*(1, 2, 3))
|
||||
d21(1, *(2, 3))
|
||||
d21(1, 2, *(3,))
|
||||
d21(1, 2, **{'c':3})
|
||||
def d02(a=1, b=2): pass
|
||||
d02()
|
||||
d02(1)
|
||||
d02(1, 2)
|
||||
d02(*(1, 2))
|
||||
d02(1, *(2,))
|
||||
d02(1, **{'b':2})
|
||||
d02(**{'a': 1, 'b': 2})
|
||||
def d12(a, b=1, c=2): pass
|
||||
d12(1)
|
||||
d12(1, 2)
|
||||
d12(1, 2, 3)
|
||||
def d22(a, b, c=1, d=2): pass
|
||||
d22(1, 2)
|
||||
d22(1, 2, 3)
|
||||
d22(1, 2, 3, 4)
|
||||
def d01v(a=1, *rest): pass
|
||||
d01v()
|
||||
d01v(1)
|
||||
d01v(1, 2)
|
||||
d01v(*(1, 2, 3, 4))
|
||||
d01v(*(1,))
|
||||
d01v(**{'a':2})
|
||||
def d11v(a, b=1, *rest): pass
|
||||
d11v(1)
|
||||
d11v(1, 2)
|
||||
d11v(1, 2, 3)
|
||||
def d21v(a, b, c=1, *rest): pass
|
||||
d21v(1, 2)
|
||||
d21v(1, 2, 3)
|
||||
d21v(1, 2, 3, 4)
|
||||
d21v(*(1, 2, 3, 4))
|
||||
d21v(1, 2, **{'c': 3})
|
||||
def d02v(a=1, b=2, *rest): pass
|
||||
d02v()
|
||||
d02v(1)
|
||||
d02v(1, 2)
|
||||
d02v(1, 2, 3)
|
||||
d02v(1, *(2, 3, 4))
|
||||
d02v(**{'a': 1, 'b': 2})
|
||||
def d12v(a, b=1, c=2, *rest): pass
|
||||
d12v(1)
|
||||
d12v(1, 2)
|
||||
d12v(1, 2, 3)
|
||||
d12v(1, 2, 3, 4)
|
||||
d12v(*(1, 2, 3, 4))
|
||||
d12v(1, 2, *(3, 4, 5))
|
||||
d12v(1, *(2,), **{'c': 3})
|
||||
def d22v(a, b, c=1, d=2, *rest): pass
|
||||
d22v(1, 2)
|
||||
d22v(1, 2, 3)
|
||||
d22v(1, 2, 3, 4)
|
||||
d22v(1, 2, 3, 4, 5)
|
||||
d22v(*(1, 2, 3, 4))
|
||||
d22v(1, 2, *(3, 4, 5))
|
||||
d22v(1, *(2, 3), **{'d': 4})
|
||||
|
||||
# keyword argument type tests
|
||||
try:
|
||||
str('x', **{b'foo':1 })
|
||||
except TypeError:
|
||||
pass
|
||||
else:
|
||||
self.fail('Bytes should not work as keyword argument names')
|
||||
# keyword only argument tests
|
||||
def pos0key1(*, key): return key
|
||||
pos0key1(key=100)
|
||||
def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
|
||||
pos2key2(1, 2, k1=100)
|
||||
pos2key2(1, 2, k1=100, k2=200)
|
||||
pos2key2(1, 2, k2=100, k1=200)
|
||||
def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
|
||||
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
|
||||
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
|
||||
|
||||
# keyword arguments after *arglist
|
||||
def f(*args, **kwargs):
|
||||
return args, kwargs
|
||||
self.assertEqual(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
|
||||
{'x':2, 'y':5}))
|
||||
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
|
||||
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
|
||||
|
||||
# argument annotation tests
|
||||
def f(x) -> list: pass
|
||||
self.assertEqual(f.__annotations__, {'return': list})
|
||||
def f(x:int): pass
|
||||
self.assertEqual(f.__annotations__, {'x': int})
|
||||
def f(*x:str): pass
|
||||
self.assertEqual(f.__annotations__, {'x': str})
|
||||
def f(**x:float): pass
|
||||
self.assertEqual(f.__annotations__, {'x': float})
|
||||
def f(x, y:1+2): pass
|
||||
self.assertEqual(f.__annotations__, {'y': 3})
|
||||
def f(a, b:1, c:2, d): pass
|
||||
self.assertEqual(f.__annotations__, {'b': 1, 'c': 2})
|
||||
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
|
||||
self.assertEqual(f.__annotations__,
|
||||
{'b': 1, 'c': 2, 'e': 3, 'g': 6})
|
||||
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
|
||||
**k:11) -> 12: pass
|
||||
self.assertEqual(f.__annotations__,
|
||||
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
|
||||
'k': 11, 'return': 12})
|
||||
# Check for SF Bug #1697248 - mixing decorators and a return annotation
|
||||
def null(x): return x
|
||||
@null
|
||||
def f(x) -> list: pass
|
||||
self.assertEqual(f.__annotations__, {'return': list})
|
||||
|
||||
# test closures with a variety of oparg's
|
||||
closure = 1
|
||||
def f(): return closure
|
||||
def f(x=1): return closure
|
||||
def f(*, k=1): return closure
|
||||
def f() -> int: return closure
|
||||
|
||||
# Check ast errors in *args and *kwargs
|
||||
check_syntax_error(self, "f(*g(1=2))")
|
||||
check_syntax_error(self, "f(**g(1=2))")
|
||||
|
||||
def testLambdef(self):
|
||||
### lambdef: 'lambda' [varargslist] ':' test
|
||||
l1 = lambda : 0
|
||||
self.assertEqual(l1(), 0)
|
||||
l2 = lambda : a[d] # XXX just testing the expression
|
||||
l3 = lambda : [2 < x for x in [-1, 3, 0]]
|
||||
self.assertEqual(l3(), [0, 1, 0])
|
||||
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
|
||||
self.assertEqual(l4(), 1)
|
||||
l5 = lambda x, y, z=2: x + y + z
|
||||
self.assertEqual(l5(1, 2), 5)
|
||||
self.assertEqual(l5(1, 2, 3), 6)
|
||||
check_syntax_error(self, "lambda x: x = 2")
|
||||
check_syntax_error(self, "lambda (None,): None")
|
||||
l6 = lambda x, y, *, k=20: x+y+k
|
||||
self.assertEqual(l6(1,2), 1+2+20)
|
||||
self.assertEqual(l6(1,2,k=10), 1+2+10)
|
||||
|
||||
|
||||
### stmt: simple_stmt | compound_stmt
|
||||
# Tested below
|
||||
|
||||
def testSimpleStmt(self):
|
||||
### simple_stmt: small_stmt (';' small_stmt)* [';']
|
||||
x = 1; pass; del x
|
||||
def foo():
|
||||
# verify statements that end with semi-colons
|
||||
x = 1; pass; del x;
|
||||
foo()
|
||||
|
||||
### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
|
||||
# Tested below
|
||||
|
||||
def testExprStmt(self):
|
||||
# (exprlist '=')* exprlist
|
||||
1
|
||||
1, 2, 3
|
||||
x = 1
|
||||
x = 1, 2, 3
|
||||
x = y = z = 1, 2, 3
|
||||
x, y, z = 1, 2, 3
|
||||
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
|
||||
|
||||
check_syntax_error(self, "x + 1 = 1")
|
||||
check_syntax_error(self, "a + 1 = b + 2")
|
||||
|
||||
def testDelStmt(self):
|
||||
# 'del' exprlist
|
||||
abc = [1,2,3]
|
||||
x, y, z = abc
|
||||
xyz = x, y, z
|
||||
|
||||
del abc
|
||||
del x, y, (z, xyz)
|
||||
|
||||
def testPassStmt(self):
|
||||
# 'pass'
|
||||
pass
|
||||
|
||||
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
|
||||
# Tested below
|
||||
|
||||
def testBreakStmt(self):
|
||||
# 'break'
|
||||
while 1: break
|
||||
|
||||
def testContinueStmt(self):
|
||||
# 'continue'
|
||||
i = 1
|
||||
while i: i = 0; continue
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "ok"
|
||||
try:
|
||||
continue
|
||||
msg = "continue failed to continue inside try"
|
||||
except:
|
||||
msg = "continue inside try called except block"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "finally block not called"
|
||||
try:
|
||||
continue
|
||||
finally:
|
||||
msg = "ok"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
def test_break_continue_loop(self):
|
||||
# This test warrants an explanation. It is a test specifically for SF bugs
|
||||
# #463359 and #462937. The bug is that a 'break' statement executed or
|
||||
# exception raised inside a try/except inside a loop, *after* a continue
|
||||
# statement has been executed in that loop, will cause the wrong number of
|
||||
# arguments to be popped off the stack and the instruction pointer reset to
|
||||
# a very small number (usually 0.) Because of this, the following test
|
||||
# *must* written as a function, and the tracking vars *must* be function
|
||||
# arguments with default values. Otherwise, the test will loop and loop.
|
||||
|
||||
def test_inner(extra_burning_oil = 1, count=0):
|
||||
big_hippo = 2
|
||||
while big_hippo:
|
||||
count += 1
|
||||
try:
|
||||
if extra_burning_oil and big_hippo == 1:
|
||||
extra_burning_oil -= 1
|
||||
break
|
||||
big_hippo -= 1
|
||||
continue
|
||||
except:
|
||||
raise
|
||||
if count > 2 or big_hippo != 1:
|
||||
self.fail("continue then break in try/except in loop broken!")
|
||||
test_inner()
|
||||
|
||||
def testReturn(self):
|
||||
# 'return' [testlist]
|
||||
def g1(): return
|
||||
def g2(): return 1
|
||||
g1()
|
||||
x = g2()
|
||||
check_syntax_error(self, "class foo:return 1")
|
||||
|
||||
def testYield(self):
|
||||
check_syntax_error(self, "class foo:yield 1")
|
||||
|
||||
def testRaise(self):
|
||||
# 'raise' test [',' test]
|
||||
try: raise RuntimeError('just testing')
|
||||
except RuntimeError: pass
|
||||
try: raise KeyboardInterrupt
|
||||
except KeyboardInterrupt: pass
|
||||
|
||||
def testImport(self):
|
||||
# 'import' dotted_as_names
|
||||
import sys
|
||||
import time, sys
|
||||
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
|
||||
from time import time
|
||||
from time import (time)
|
||||
# not testable inside a function, but already done at top of the module
|
||||
# from sys import *
|
||||
from sys import path, argv
|
||||
from sys import (path, argv)
|
||||
from sys import (path, argv,)
|
||||
|
||||
def testGlobal(self):
|
||||
# 'global' NAME (',' NAME)*
|
||||
global a
|
||||
global a, b
|
||||
global one, two, three, four, five, six, seven, eight, nine, ten
|
||||
|
||||
def testNonlocal(self):
|
||||
# 'nonlocal' NAME (',' NAME)*
|
||||
x = 0
|
||||
y = 0
|
||||
def f():
|
||||
nonlocal x
|
||||
nonlocal x, y
|
||||
|
||||
def testAssert(self):
|
||||
# assert_stmt: 'assert' test [',' test]
|
||||
assert 1
|
||||
assert 1, 1
|
||||
assert lambda x:x
|
||||
assert 1, lambda x:x+1
|
||||
try:
|
||||
assert 0, "msg"
|
||||
except AssertionError as e:
|
||||
self.assertEqual(e.args[0], "msg")
|
||||
else:
|
||||
if __debug__:
|
||||
self.fail("AssertionError not raised by assert 0")
|
||||
|
||||
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
|
||||
# Tested below
|
||||
|
||||
def testIf(self):
|
||||
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||
if 1: pass
|
||||
if 1: pass
|
||||
else: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
else: pass
|
||||
|
||||
def testWhile(self):
|
||||
# 'while' test ':' suite ['else' ':' suite]
|
||||
while 0: pass
|
||||
while 0: pass
|
||||
else: pass
|
||||
|
||||
# Issue1920: "while 0" is optimized away,
|
||||
# ensure that the "else" clause is still present.
|
||||
x = 0
|
||||
while 0:
|
||||
x = 1
|
||||
else:
|
||||
x = 2
|
||||
self.assertEqual(x, 2)
|
||||
|
||||
def testFor(self):
|
||||
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
|
||||
for i in 1, 2, 3: pass
|
||||
for i, j, k in (): pass
|
||||
else: pass
|
||||
class Squares:
|
||||
def __init__(self, max):
|
||||
self.max = max
|
||||
self.sofar = []
|
||||
def __len__(self): return len(self.sofar)
|
||||
def __getitem__(self, i):
|
||||
if not 0 <= i < self.max: raise IndexError
|
||||
n = len(self.sofar)
|
||||
while n <= i:
|
||||
self.sofar.append(n*n)
|
||||
n = n+1
|
||||
return self.sofar[i]
|
||||
n = 0
|
||||
for x in Squares(10): n = n+x
|
||||
if n != 285:
|
||||
self.fail('for over growing sequence')
|
||||
|
||||
result = []
|
||||
for x, in [(1,), (2,), (3,)]:
|
||||
result.append(x)
|
||||
self.assertEqual(result, [1, 2, 3])
|
||||
|
||||
def testTry(self):
|
||||
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
|
||||
### | 'try' ':' suite 'finally' ':' suite
|
||||
### except_clause: 'except' [expr ['as' expr]]
|
||||
try:
|
||||
1/0
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
try: 1/0
|
||||
except EOFError: pass
|
||||
except TypeError as msg: pass
|
||||
except RuntimeError as msg: pass
|
||||
except: pass
|
||||
else: pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError): pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError) as msg: pass
|
||||
try: pass
|
||||
finally: pass
|
||||
|
||||
def testSuite(self):
|
||||
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
|
||||
if 1: pass
|
||||
if 1:
|
||||
pass
|
||||
if 1:
|
||||
#
|
||||
#
|
||||
#
|
||||
pass
|
||||
pass
|
||||
#
|
||||
pass
|
||||
#
|
||||
|
||||
def testTest(self):
|
||||
### and_test ('or' and_test)*
|
||||
### and_test: not_test ('and' not_test)*
|
||||
### not_test: 'not' not_test | comparison
|
||||
if not 1: pass
|
||||
if 1 and 1: pass
|
||||
if 1 or 1: pass
|
||||
if not not not 1: pass
|
||||
if not 1 and 1 and 1: pass
|
||||
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
|
||||
|
||||
def testComparison(self):
|
||||
### comparison: expr (comp_op expr)*
|
||||
### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||||
if 1: pass
|
||||
x = (1 == 1)
|
||||
if 1 == 1: pass
|
||||
if 1 != 1: pass
|
||||
if 1 < 1: pass
|
||||
if 1 > 1: pass
|
||||
if 1 <= 1: pass
|
||||
if 1 >= 1: pass
|
||||
if 1 is 1: pass
|
||||
if 1 is not 1: pass
|
||||
if 1 in (): pass
|
||||
if 1 not in (): pass
|
||||
if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
|
||||
|
||||
def testBinaryMaskOps(self):
|
||||
x = 1 & 1
|
||||
x = 1 ^ 1
|
||||
x = 1 | 1
|
||||
|
||||
def testShiftOps(self):
|
||||
x = 1 << 1
|
||||
x = 1 >> 1
|
||||
x = 1 << 1 >> 1
|
||||
|
||||
def testAdditiveOps(self):
|
||||
x = 1
|
||||
x = 1 + 1
|
||||
x = 1 - 1 - 1
|
||||
x = 1 - 1 + 1 - 1 + 1
|
||||
|
||||
def testMultiplicativeOps(self):
|
||||
x = 1 * 1
|
||||
x = 1 / 1
|
||||
x = 1 % 1
|
||||
x = 1 / 1 * 1 % 1
|
||||
|
||||
def testUnaryOps(self):
|
||||
x = +1
|
||||
x = -1
|
||||
x = ~1
|
||||
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
|
||||
x = -1*1/1 + 1*1 - ---1*1
|
||||
|
||||
def testSelectors(self):
|
||||
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
|
||||
### subscript: expr | [expr] ':' [expr]
|
||||
|
||||
import sys, time
|
||||
c = sys.path[0]
|
||||
x = time.time()
|
||||
x = sys.modules['time'].time()
|
||||
a = '01234'
|
||||
c = a[0]
|
||||
c = a[-1]
|
||||
s = a[0:5]
|
||||
s = a[:5]
|
||||
s = a[0:]
|
||||
s = a[:]
|
||||
s = a[-5:]
|
||||
s = a[:-1]
|
||||
s = a[-4:-3]
|
||||
# A rough test of SF bug 1333982. http://python.org/sf/1333982
|
||||
# The testing here is fairly incomplete.
|
||||
# Test cases should include: commas with 1 and 2 colons
|
||||
d = {}
|
||||
d[1] = 1
|
||||
d[1,] = 2
|
||||
d[1,2] = 3
|
||||
d[1,2,3] = 4
|
||||
L = list(d)
|
||||
L.sort(key=lambda x: x if isinstance(x, tuple) else ())
|
||||
self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
|
||||
|
||||
def testAtoms(self):
|
||||
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
|
||||
### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
|
||||
|
||||
x = (1)
|
||||
x = (1 or 2 or 3)
|
||||
x = (1 or 2 or 3, 2, 3)
|
||||
|
||||
x = []
|
||||
x = [1]
|
||||
x = [1 or 2 or 3]
|
||||
x = [1 or 2 or 3, 2, 3]
|
||||
x = []
|
||||
|
||||
x = {}
|
||||
x = {'one': 1}
|
||||
x = {'one': 1,}
|
||||
x = {'one' or 'two': 1 or 2}
|
||||
x = {'one': 1, 'two': 2}
|
||||
x = {'one': 1, 'two': 2,}
|
||||
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
|
||||
|
||||
x = {'one'}
|
||||
x = {'one', 1,}
|
||||
x = {'one', 'two', 'three'}
|
||||
x = {2, 3, 4,}
|
||||
|
||||
x = x
|
||||
x = 'x'
|
||||
x = 123
|
||||
|
||||
### exprlist: expr (',' expr)* [',']
|
||||
### testlist: test (',' test)* [',']
|
||||
# These have been exercised enough above
|
||||
|
||||
def testClassdef(self):
|
||||
# 'class' NAME ['(' [testlist] ')'] ':' suite
|
||||
class B: pass
|
||||
class B2(): pass
|
||||
class C1(B): pass
|
||||
class C2(B): pass
|
||||
class D(C1, C2, B): pass
|
||||
class C:
|
||||
def meth1(self): pass
|
||||
def meth2(self, arg): pass
|
||||
def meth3(self, a1, a2): pass
|
||||
|
||||
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
# decorators: decorator+
|
||||
# decorated: decorators (classdef | funcdef)
|
||||
def class_decorator(x): return x
|
||||
@class_decorator
|
||||
class G: pass
|
||||
|
||||
def testDictcomps(self):
|
||||
# dictorsetmaker: ( (test ':' test (comp_for |
|
||||
# (',' test ':' test)* [','])) |
|
||||
# (test (comp_for | (',' test)* [','])) )
|
||||
nums = [1, 2, 3]
|
||||
self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
|
||||
|
||||
def testListcomps(self):
|
||||
# list comprehension tests
|
||||
nums = [1, 2, 3, 4, 5]
|
||||
strs = ["Apple", "Banana", "Coconut"]
|
||||
spcs = [" Apple", " Banana ", "Coco nut "]
|
||||
|
||||
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
|
||||
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
|
||||
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
|
||||
self.assertEqual([(i, s) for i in nums for s in strs],
|
||||
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
|
||||
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
|
||||
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
|
||||
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
|
||||
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
|
||||
|
||||
def test_in_func(l):
|
||||
return [0 < x < 3 for x in l if x > 2]
|
||||
|
||||
self.assertEqual(test_in_func(nums), [False, False, False])
|
||||
|
||||
def test_nested_front():
|
||||
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
|
||||
[[1, 2], [3, 4], [5, 6]])
|
||||
|
||||
test_nested_front()
|
||||
|
||||
check_syntax_error(self, "[i, s for i in nums for s in strs]")
|
||||
check_syntax_error(self, "[x if y]")
|
||||
|
||||
suppliers = [
|
||||
(1, "Boeing"),
|
||||
(2, "Ford"),
|
||||
(3, "Macdonalds")
|
||||
]
|
||||
|
||||
parts = [
|
||||
(10, "Airliner"),
|
||||
(20, "Engine"),
|
||||
(30, "Cheeseburger")
|
||||
]
|
||||
|
||||
suppart = [
|
||||
(1, 10), (1, 20), (2, 20), (3, 30)
|
||||
]
|
||||
|
||||
x = [
|
||||
(sname, pname)
|
||||
for (sno, sname) in suppliers
|
||||
for (pno, pname) in parts
|
||||
for (sp_sno, sp_pno) in suppart
|
||||
if sno == sp_sno and pno == sp_pno
|
||||
]
|
||||
|
||||
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
|
||||
('Macdonalds', 'Cheeseburger')])
|
||||
|
||||
def testGenexps(self):
|
||||
# generator expression tests
|
||||
g = ([x for x in range(10)] for x in range(1))
|
||||
self.assertEqual(next(g), [x for x in range(10)])
|
||||
try:
|
||||
next(g)
|
||||
self.fail('should produce StopIteration exception')
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
a = 1
|
||||
try:
|
||||
g = (a for d in a)
|
||||
next(g)
|
||||
self.fail('should produce TypeError')
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
|
||||
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
|
||||
|
||||
a = [x for x in range(10)]
|
||||
b = (x for x in (y for y in a))
|
||||
self.assertEqual(sum(b), sum([x for x in range(10)]))
|
||||
|
||||
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
|
||||
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
|
||||
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
|
||||
check_syntax_error(self, "foo(x for x in range(10), 100)")
|
||||
check_syntax_error(self, "foo(100, x for x in range(10))")
|
||||
|
||||
def testComprehensionSpecials(self):
|
||||
# test for outmost iterable precomputation
|
||||
x = 10; g = (i for i in range(x)); x = 5
|
||||
self.assertEqual(len(list(g)), 10)
|
||||
|
||||
# This should hold, since we're only precomputing outmost iterable.
|
||||
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
|
||||
x = 5; t = True;
|
||||
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
|
||||
|
||||
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
|
||||
# even though it's silly. Make sure it works (ifelse broke this.)
|
||||
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
|
||||
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
|
||||
|
||||
# verify unpacking single element tuples in listcomp/genexp.
|
||||
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
|
||||
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
|
||||
|
||||
def test_with_statement(self):
|
||||
class manager(object):
|
||||
def __enter__(self):
|
||||
return (1, 2)
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
with manager():
|
||||
pass
|
||||
with manager() as x:
|
||||
pass
|
||||
with manager() as (x, y):
|
||||
pass
|
||||
with manager(), manager():
|
||||
pass
|
||||
with manager() as x, manager() as y:
|
||||
pass
|
||||
with manager() as x, manager():
|
||||
pass
|
||||
|
||||
def testIfElseExpr(self):
|
||||
# Test ifelse expressions in various cases
|
||||
def _checkeval(msg, ret):
|
||||
"helper to check that evaluation of expressions is done correctly"
|
||||
print(x)
|
||||
return ret
|
||||
|
||||
# the next line is not allowed anymore
|
||||
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
|
||||
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
|
||||
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
|
||||
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
|
||||
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
|
||||
self.assertEqual((5 and 6 if 0 else 1), 1)
|
||||
self.assertEqual(((5 and 6) if 0 else 1), 1)
|
||||
self.assertEqual((5 and (6 if 1 else 1)), 6)
|
||||
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
|
||||
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
|
||||
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
|
||||
self.assertEqual((not 5 if 1 else 1), False)
|
||||
self.assertEqual((not 5 if 0 else 1), 1)
|
||||
self.assertEqual((6 + 1 if 1 else 2), 7)
|
||||
self.assertEqual((6 - 1 if 1 else 2), 5)
|
||||
self.assertEqual((6 * 2 if 1 else 4), 12)
|
||||
self.assertEqual((6 / 2 if 1 else 3), 3)
|
||||
self.assertEqual((6 < 4 if 0 else 2), 2)
|
||||
|
||||
def testStringLiterals(self):
|
||||
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
|
||||
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
|
||||
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
|
||||
x = "doesn't \"shrink\" does it"
|
||||
y = 'doesn\'t "shrink" does it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = "does \"shrink\" doesn't it"
|
||||
y = 'does "shrink" doesn\'t it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = """
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
"""
|
||||
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
|
||||
self.assertEqual(x, y)
|
||||
y = '''
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
'''
|
||||
self.assertEqual(x, y)
|
||||
y = "\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the 'lazy' dog.\n\
|
||||
"
|
||||
self.assertEqual(x, y)
|
||||
y = '\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the \'lazy\' dog.\n\
|
||||
'
|
||||
self.assertEqual(x, y)
|
||||
|
||||
|
||||
def test_main():
|
||||
run_unittest(TokenTests, GrammarTests)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_main()
|
||||
+945
@@ -0,0 +1,945 @@
|
||||
# Python test set -- part 1, grammar.
|
||||
# This just tests whether the parser accepts them all.
|
||||
|
||||
# NOTE: When you run this test as a script from the command line, you
|
||||
# get warnings about certain hex/oct constants. Since those are
|
||||
# issued by the parser, you can't suppress them by adding a
|
||||
# filterwarnings() call to this module. Therefore, to shut up the
|
||||
# regression test, the filterwarnings() call has been added to
|
||||
# regrtest.py.
|
||||
|
||||
from test.support import run_unittest, check_syntax_error
|
||||
import unittest
|
||||
import sys
|
||||
# testing import *
|
||||
from sys import *
|
||||
|
||||
class TokenTests(unittest.TestCase):
|
||||
|
||||
def testBackslash(self):
|
||||
# Backslash means line continuation:
|
||||
x = 1 \
|
||||
+ 1
|
||||
self.assertEqual(x, 2, 'backslash for line continuation')
|
||||
|
||||
# Backslash does not means continuation in comments :\
|
||||
x = 0
|
||||
self.assertEqual(x, 0, 'backslash ending comment')
|
||||
|
||||
def testPlainIntegers(self):
|
||||
self.assertEqual(type(000), type(0))
|
||||
self.assertEqual(0xff, 255)
|
||||
self.assertEqual(0o377, 255)
|
||||
self.assertEqual(2147483647, 0o17777777777)
|
||||
self.assertEqual(0b1001, 9)
|
||||
# "0x" is not a valid literal
|
||||
self.assertRaises(SyntaxError, eval, "0x")
|
||||
from sys import maxsize
|
||||
if maxsize == 2147483647:
|
||||
self.assertEqual(-2147483647-1, -0o20000000000)
|
||||
# XXX -2147483648
|
||||
self.assert_(0o37777777777 > 0)
|
||||
self.assert_(0xffffffff > 0)
|
||||
self.assert_(0b1111111111111111111111111111111 > 0)
|
||||
for s in ('2147483648', '0o40000000000', '0x100000000',
|
||||
'0b10000000000000000000000000000000'):
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
elif maxsize == 9223372036854775807:
|
||||
self.assertEqual(-9223372036854775807-1, -0o1000000000000000000000)
|
||||
self.assert_(0o1777777777777777777777 > 0)
|
||||
self.assert_(0xffffffffffffffff > 0)
|
||||
self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
|
||||
for s in '9223372036854775808', '0o2000000000000000000000', \
|
||||
'0x10000000000000000', \
|
||||
'0b100000000000000000000000000000000000000000000000000000000000000':
|
||||
try:
|
||||
x = eval(s)
|
||||
except OverflowError:
|
||||
self.fail("OverflowError on huge integer literal %r" % s)
|
||||
else:
|
||||
self.fail('Weird maxsize value %r' % maxsize)
|
||||
|
||||
def testLongIntegers(self):
|
||||
x = 0
|
||||
x = 0xffffffffffffffff
|
||||
x = 0Xffffffffffffffff
|
||||
x = 0o77777777777777777
|
||||
x = 0O77777777777777777
|
||||
x = 123456789012345678901234567890
|
||||
x = 0b100000000000000000000000000000000000000000000000000000000000000000000
|
||||
x = 0B111111111111111111111111111111111111111111111111111111111111111111111
|
||||
|
||||
def testUnderscoresInNumbers(self):
|
||||
# Integers
|
||||
x = 1_0
|
||||
x = 123_456_7_89
|
||||
x = 0xabc_123_4_5
|
||||
x = 0X_abc_123
|
||||
x = 0B11_01
|
||||
x = 0b_11_01
|
||||
x = 0o45_67
|
||||
x = 0O_45_67
|
||||
|
||||
# Floats
|
||||
x = 3_1.4
|
||||
x = 03_1.4
|
||||
x = 3_1.
|
||||
x = .3_1
|
||||
x = 3.1_4
|
||||
x = 0_3.1_4
|
||||
x = 3e1_4
|
||||
x = 3_1e+4_1
|
||||
x = 3_1E-4_1
|
||||
|
||||
def testFloats(self):
|
||||
x = 3.14
|
||||
x = 314.
|
||||
x = 0.314
|
||||
# XXX x = 000.314
|
||||
x = .314
|
||||
x = 3e14
|
||||
x = 3E14
|
||||
x = 3e-14
|
||||
x = 3e+14
|
||||
x = 3.e14
|
||||
x = .3e14
|
||||
x = 3.1e4
|
||||
|
||||
def testEllipsis(self):
|
||||
x = ...
|
||||
self.assert_(x is Ellipsis)
|
||||
self.assertRaises(SyntaxError, eval, ".. .")
|
||||
|
||||
class GrammarTests(unittest.TestCase):
|
||||
|
||||
# single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
|
||||
# XXX can't test in a script -- this rule is only used when interactive
|
||||
|
||||
# file_input: (NEWLINE | stmt)* ENDMARKER
|
||||
# Being tested as this very moment this very module
|
||||
|
||||
# expr_input: testlist NEWLINE
|
||||
# XXX Hard to test -- used only in calls to input()
|
||||
|
||||
def testEvalInput(self):
|
||||
# testlist ENDMARKER
|
||||
x = eval('1, 0 or 1')
|
||||
|
||||
def testFuncdef(self):
|
||||
### [decorators] 'def' NAME parameters ['->' test] ':' suite
|
||||
### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
### decorators: decorator+
|
||||
### parameters: '(' [typedargslist] ')'
|
||||
### typedargslist: ((tfpdef ['=' test] ',')*
|
||||
### ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
|
||||
### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
|
||||
### tfpdef: NAME [':' test]
|
||||
### varargslist: ((vfpdef ['=' test] ',')*
|
||||
### ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
|
||||
### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
|
||||
### vfpdef: NAME
|
||||
def f1(): pass
|
||||
f1()
|
||||
f1(*())
|
||||
f1(*(), **{})
|
||||
def f2(one_argument): pass
|
||||
def f3(two, arguments): pass
|
||||
self.assertEqual(f2.__code__.co_varnames, ('one_argument',))
|
||||
self.assertEqual(f3.__code__.co_varnames, ('two', 'arguments'))
|
||||
def a1(one_arg,): pass
|
||||
def a2(two, args,): pass
|
||||
def v0(*rest): pass
|
||||
def v1(a, *rest): pass
|
||||
def v2(a, b, *rest): pass
|
||||
|
||||
f1()
|
||||
f2(1)
|
||||
f2(1,)
|
||||
f3(1, 2)
|
||||
f3(1, 2,)
|
||||
v0()
|
||||
v0(1)
|
||||
v0(1,)
|
||||
v0(1,2)
|
||||
v0(1,2,3,4,5,6,7,8,9,0)
|
||||
v1(1)
|
||||
v1(1,)
|
||||
v1(1,2)
|
||||
v1(1,2,3)
|
||||
v1(1,2,3,4,5,6,7,8,9,0)
|
||||
v2(1,2)
|
||||
v2(1,2,3)
|
||||
v2(1,2,3,4)
|
||||
v2(1,2,3,4,5,6,7,8,9,0)
|
||||
|
||||
def d01(a=1): pass
|
||||
d01()
|
||||
d01(1)
|
||||
d01(*(1,))
|
||||
d01(**{'a':2})
|
||||
def d11(a, b=1): pass
|
||||
d11(1)
|
||||
d11(1, 2)
|
||||
d11(1, **{'b':2})
|
||||
def d21(a, b, c=1): pass
|
||||
d21(1, 2)
|
||||
d21(1, 2, 3)
|
||||
d21(*(1, 2, 3))
|
||||
d21(1, *(2, 3))
|
||||
d21(1, 2, *(3,))
|
||||
d21(1, 2, **{'c':3})
|
||||
def d02(a=1, b=2): pass
|
||||
d02()
|
||||
d02(1)
|
||||
d02(1, 2)
|
||||
d02(*(1, 2))
|
||||
d02(1, *(2,))
|
||||
d02(1, **{'b':2})
|
||||
d02(**{'a': 1, 'b': 2})
|
||||
def d12(a, b=1, c=2): pass
|
||||
d12(1)
|
||||
d12(1, 2)
|
||||
d12(1, 2, 3)
|
||||
def d22(a, b, c=1, d=2): pass
|
||||
d22(1, 2)
|
||||
d22(1, 2, 3)
|
||||
d22(1, 2, 3, 4)
|
||||
def d01v(a=1, *rest): pass
|
||||
d01v()
|
||||
d01v(1)
|
||||
d01v(1, 2)
|
||||
d01v(*(1, 2, 3, 4))
|
||||
d01v(*(1,))
|
||||
d01v(**{'a':2})
|
||||
def d11v(a, b=1, *rest): pass
|
||||
d11v(1)
|
||||
d11v(1, 2)
|
||||
d11v(1, 2, 3)
|
||||
def d21v(a, b, c=1, *rest): pass
|
||||
d21v(1, 2)
|
||||
d21v(1, 2, 3)
|
||||
d21v(1, 2, 3, 4)
|
||||
d21v(*(1, 2, 3, 4))
|
||||
d21v(1, 2, **{'c': 3})
|
||||
def d02v(a=1, b=2, *rest): pass
|
||||
d02v()
|
||||
d02v(1)
|
||||
d02v(1, 2)
|
||||
d02v(1, 2, 3)
|
||||
d02v(1, *(2, 3, 4))
|
||||
d02v(**{'a': 1, 'b': 2})
|
||||
def d12v(a, b=1, c=2, *rest): pass
|
||||
d12v(1)
|
||||
d12v(1, 2)
|
||||
d12v(1, 2, 3)
|
||||
d12v(1, 2, 3, 4)
|
||||
d12v(*(1, 2, 3, 4))
|
||||
d12v(1, 2, *(3, 4, 5))
|
||||
d12v(1, *(2,), **{'c': 3})
|
||||
def d22v(a, b, c=1, d=2, *rest): pass
|
||||
d22v(1, 2)
|
||||
d22v(1, 2, 3)
|
||||
d22v(1, 2, 3, 4)
|
||||
d22v(1, 2, 3, 4, 5)
|
||||
d22v(*(1, 2, 3, 4))
|
||||
d22v(1, 2, *(3, 4, 5))
|
||||
d22v(1, *(2, 3), **{'d': 4})
|
||||
|
||||
# keyword argument type tests
|
||||
try:
|
||||
str('x', **{b'foo':1 })
|
||||
except TypeError:
|
||||
pass
|
||||
else:
|
||||
self.fail('Bytes should not work as keyword argument names')
|
||||
# keyword only argument tests
|
||||
def pos0key1(*, key): return key
|
||||
pos0key1(key=100)
|
||||
def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
|
||||
pos2key2(1, 2, k1=100)
|
||||
pos2key2(1, 2, k1=100, k2=200)
|
||||
pos2key2(1, 2, k2=100, k1=200)
|
||||
def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
|
||||
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
|
||||
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
|
||||
|
||||
# keyword arguments after *arglist
|
||||
def f(*args, **kwargs):
|
||||
return args, kwargs
|
||||
self.assertEqual(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
|
||||
{'x':2, 'y':5}))
|
||||
self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
|
||||
self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
|
||||
|
||||
# argument annotation tests
|
||||
def f(x) -> list: pass
|
||||
self.assertEqual(f.__annotations__, {'return': list})
|
||||
def f(x:int): pass
|
||||
self.assertEqual(f.__annotations__, {'x': int})
|
||||
def f(*x:str): pass
|
||||
self.assertEqual(f.__annotations__, {'x': str})
|
||||
def f(**x:float): pass
|
||||
self.assertEqual(f.__annotations__, {'x': float})
|
||||
def f(x, y:1+2): pass
|
||||
self.assertEqual(f.__annotations__, {'y': 3})
|
||||
def f(a, b:1, c:2, d): pass
|
||||
self.assertEqual(f.__annotations__, {'b': 1, 'c': 2})
|
||||
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
|
||||
self.assertEqual(f.__annotations__,
|
||||
{'b': 1, 'c': 2, 'e': 3, 'g': 6})
|
||||
def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
|
||||
**k:11) -> 12: pass
|
||||
self.assertEqual(f.__annotations__,
|
||||
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
|
||||
'k': 11, 'return': 12})
|
||||
# Check for SF Bug #1697248 - mixing decorators and a return annotation
|
||||
def null(x): return x
|
||||
@null
|
||||
def f(x) -> list: pass
|
||||
self.assertEqual(f.__annotations__, {'return': list})
|
||||
|
||||
# test closures with a variety of oparg's
|
||||
closure = 1
|
||||
def f(): return closure
|
||||
def f(x=1): return closure
|
||||
def f(*, k=1): return closure
|
||||
def f() -> int: return closure
|
||||
|
||||
# Check ast errors in *args and *kwargs
|
||||
check_syntax_error(self, "f(*g(1=2))")
|
||||
check_syntax_error(self, "f(**g(1=2))")
|
||||
|
||||
def testLambdef(self):
|
||||
### lambdef: 'lambda' [varargslist] ':' test
|
||||
l1 = lambda : 0
|
||||
self.assertEqual(l1(), 0)
|
||||
l2 = lambda : a[d] # XXX just testing the expression
|
||||
l3 = lambda : [2 < x for x in [-1, 3, 0]]
|
||||
self.assertEqual(l3(), [0, 1, 0])
|
||||
l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
|
||||
self.assertEqual(l4(), 1)
|
||||
l5 = lambda x, y, z=2: x + y + z
|
||||
self.assertEqual(l5(1, 2), 5)
|
||||
self.assertEqual(l5(1, 2, 3), 6)
|
||||
check_syntax_error(self, "lambda x: x = 2")
|
||||
check_syntax_error(self, "lambda (None,): None")
|
||||
l6 = lambda x, y, *, k=20: x+y+k
|
||||
self.assertEqual(l6(1,2), 1+2+20)
|
||||
self.assertEqual(l6(1,2,k=10), 1+2+10)
|
||||
|
||||
|
||||
### stmt: simple_stmt | compound_stmt
|
||||
# Tested below
|
||||
|
||||
def testSimpleStmt(self):
|
||||
### simple_stmt: small_stmt (';' small_stmt)* [';']
|
||||
x = 1; pass; del x
|
||||
def foo():
|
||||
# verify statements that end with semi-colons
|
||||
x = 1; pass; del x;
|
||||
foo()
|
||||
|
||||
### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
|
||||
# Tested below
|
||||
|
||||
def testExprStmt(self):
|
||||
# (exprlist '=')* exprlist
|
||||
1
|
||||
1, 2, 3
|
||||
x = 1
|
||||
x = 1, 2, 3
|
||||
x = y = z = 1, 2, 3
|
||||
x, y, z = 1, 2, 3
|
||||
abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
|
||||
|
||||
check_syntax_error(self, "x + 1 = 1")
|
||||
check_syntax_error(self, "a + 1 = b + 2")
|
||||
|
||||
def testDelStmt(self):
|
||||
# 'del' exprlist
|
||||
abc = [1,2,3]
|
||||
x, y, z = abc
|
||||
xyz = x, y, z
|
||||
|
||||
del abc
|
||||
del x, y, (z, xyz)
|
||||
|
||||
def testPassStmt(self):
|
||||
# 'pass'
|
||||
pass
|
||||
|
||||
# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
|
||||
# Tested below
|
||||
|
||||
def testBreakStmt(self):
|
||||
# 'break'
|
||||
while 1: break
|
||||
|
||||
def testContinueStmt(self):
|
||||
# 'continue'
|
||||
i = 1
|
||||
while i: i = 0; continue
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "ok"
|
||||
try:
|
||||
continue
|
||||
msg = "continue failed to continue inside try"
|
||||
except:
|
||||
msg = "continue inside try called except block"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
msg = ""
|
||||
while not msg:
|
||||
msg = "finally block not called"
|
||||
try:
|
||||
continue
|
||||
finally:
|
||||
msg = "ok"
|
||||
if msg != "ok":
|
||||
self.fail(msg)
|
||||
|
||||
def test_break_continue_loop(self):
|
||||
# This test warrants an explanation. It is a test specifically for SF bugs
|
||||
# #463359 and #462937. The bug is that a 'break' statement executed or
|
||||
# exception raised inside a try/except inside a loop, *after* a continue
|
||||
# statement has been executed in that loop, will cause the wrong number of
|
||||
# arguments to be popped off the stack and the instruction pointer reset to
|
||||
# a very small number (usually 0.) Because of this, the following test
|
||||
# *must* written as a function, and the tracking vars *must* be function
|
||||
# arguments with default values. Otherwise, the test will loop and loop.
|
||||
|
||||
def test_inner(extra_burning_oil = 1, count=0):
|
||||
big_hippo = 2
|
||||
while big_hippo:
|
||||
count += 1
|
||||
try:
|
||||
if extra_burning_oil and big_hippo == 1:
|
||||
extra_burning_oil -= 1
|
||||
break
|
||||
big_hippo -= 1
|
||||
continue
|
||||
except:
|
||||
raise
|
||||
if count > 2 or big_hippo != 1:
|
||||
self.fail("continue then break in try/except in loop broken!")
|
||||
test_inner()
|
||||
|
||||
def testReturn(self):
|
||||
# 'return' [testlist]
|
||||
def g1(): return
|
||||
def g2(): return 1
|
||||
g1()
|
||||
x = g2()
|
||||
check_syntax_error(self, "class foo:return 1")
|
||||
|
||||
def testYield(self):
|
||||
check_syntax_error(self, "class foo:yield 1")
|
||||
|
||||
def testRaise(self):
|
||||
# 'raise' test [',' test]
|
||||
try: raise RuntimeError('just testing')
|
||||
except RuntimeError: pass
|
||||
try: raise KeyboardInterrupt
|
||||
except KeyboardInterrupt: pass
|
||||
|
||||
def testImport(self):
|
||||
# 'import' dotted_as_names
|
||||
import sys
|
||||
import time, sys
|
||||
# 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
|
||||
from time import time
|
||||
from time import (time)
|
||||
# not testable inside a function, but already done at top of the module
|
||||
# from sys import *
|
||||
from sys import path, argv
|
||||
from sys import (path, argv)
|
||||
from sys import (path, argv,)
|
||||
|
||||
def testGlobal(self):
|
||||
# 'global' NAME (',' NAME)*
|
||||
global a
|
||||
global a, b
|
||||
global one, two, three, four, five, six, seven, eight, nine, ten
|
||||
|
||||
def testNonlocal(self):
|
||||
# 'nonlocal' NAME (',' NAME)*
|
||||
x = 0
|
||||
y = 0
|
||||
def f():
|
||||
nonlocal x
|
||||
nonlocal x, y
|
||||
|
||||
def testAssert(self):
|
||||
# assert_stmt: 'assert' test [',' test]
|
||||
assert 1
|
||||
assert 1, 1
|
||||
assert lambda x:x
|
||||
assert 1, lambda x:x+1
|
||||
try:
|
||||
assert 0, "msg"
|
||||
except AssertionError as e:
|
||||
self.assertEqual(e.args[0], "msg")
|
||||
else:
|
||||
if __debug__:
|
||||
self.fail("AssertionError not raised by assert 0")
|
||||
|
||||
### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
|
||||
# Tested below
|
||||
|
||||
def testIf(self):
|
||||
# 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
|
||||
if 1: pass
|
||||
if 1: pass
|
||||
else: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
if 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
elif 0: pass
|
||||
else: pass
|
||||
|
||||
def testWhile(self):
|
||||
# 'while' test ':' suite ['else' ':' suite]
|
||||
while 0: pass
|
||||
while 0: pass
|
||||
else: pass
|
||||
|
||||
# Issue1920: "while 0" is optimized away,
|
||||
# ensure that the "else" clause is still present.
|
||||
x = 0
|
||||
while 0:
|
||||
x = 1
|
||||
else:
|
||||
x = 2
|
||||
self.assertEqual(x, 2)
|
||||
|
||||
def testFor(self):
|
||||
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
|
||||
for i in 1, 2, 3: pass
|
||||
for i, j, k in (): pass
|
||||
else: pass
|
||||
class Squares:
|
||||
def __init__(self, max):
|
||||
self.max = max
|
||||
self.sofar = []
|
||||
def __len__(self): return len(self.sofar)
|
||||
def __getitem__(self, i):
|
||||
if not 0 <= i < self.max: raise IndexError
|
||||
n = len(self.sofar)
|
||||
while n <= i:
|
||||
self.sofar.append(n*n)
|
||||
n = n+1
|
||||
return self.sofar[i]
|
||||
n = 0
|
||||
for x in Squares(10): n = n+x
|
||||
if n != 285:
|
||||
self.fail('for over growing sequence')
|
||||
|
||||
result = []
|
||||
for x, in [(1,), (2,), (3,)]:
|
||||
result.append(x)
|
||||
self.assertEqual(result, [1, 2, 3])
|
||||
|
||||
def testTry(self):
|
||||
### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
|
||||
### | 'try' ':' suite 'finally' ':' suite
|
||||
### except_clause: 'except' [expr ['as' expr]]
|
||||
try:
|
||||
1/0
|
||||
except ZeroDivisionError:
|
||||
pass
|
||||
else:
|
||||
pass
|
||||
try: 1/0
|
||||
except EOFError: pass
|
||||
except TypeError as msg: pass
|
||||
except RuntimeError as msg: pass
|
||||
except: pass
|
||||
else: pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError): pass
|
||||
try: 1/0
|
||||
except (EOFError, TypeError, ZeroDivisionError) as msg: pass
|
||||
try: pass
|
||||
finally: pass
|
||||
|
||||
def testSuite(self):
|
||||
# simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
|
||||
if 1: pass
|
||||
if 1:
|
||||
pass
|
||||
if 1:
|
||||
#
|
||||
#
|
||||
#
|
||||
pass
|
||||
pass
|
||||
#
|
||||
pass
|
||||
#
|
||||
|
||||
def testTest(self):
|
||||
### and_test ('or' and_test)*
|
||||
### and_test: not_test ('and' not_test)*
|
||||
### not_test: 'not' not_test | comparison
|
||||
if not 1: pass
|
||||
if 1 and 1: pass
|
||||
if 1 or 1: pass
|
||||
if not not not 1: pass
|
||||
if not 1 and 1 and 1: pass
|
||||
if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
|
||||
|
||||
def testComparison(self):
|
||||
### comparison: expr (comp_op expr)*
|
||||
### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||||
if 1: pass
|
||||
x = (1 == 1)
|
||||
if 1 == 1: pass
|
||||
if 1 != 1: pass
|
||||
if 1 < 1: pass
|
||||
if 1 > 1: pass
|
||||
if 1 <= 1: pass
|
||||
if 1 >= 1: pass
|
||||
if 1 is 1: pass
|
||||
if 1 is not 1: pass
|
||||
if 1 in (): pass
|
||||
if 1 not in (): pass
|
||||
if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
|
||||
|
||||
def testBinaryMaskOps(self):
|
||||
x = 1 & 1
|
||||
x = 1 ^ 1
|
||||
x = 1 | 1
|
||||
|
||||
def testShiftOps(self):
|
||||
x = 1 << 1
|
||||
x = 1 >> 1
|
||||
x = 1 << 1 >> 1
|
||||
|
||||
def testAdditiveOps(self):
|
||||
x = 1
|
||||
x = 1 + 1
|
||||
x = 1 - 1 - 1
|
||||
x = 1 - 1 + 1 - 1 + 1
|
||||
|
||||
def testMultiplicativeOps(self):
|
||||
x = 1 * 1
|
||||
x = 1 / 1
|
||||
x = 1 % 1
|
||||
x = 1 / 1 * 1 % 1
|
||||
|
||||
def testUnaryOps(self):
|
||||
x = +1
|
||||
x = -1
|
||||
x = ~1
|
||||
x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
|
||||
x = -1*1/1 + 1*1 - ---1*1
|
||||
|
||||
def testSelectors(self):
|
||||
### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
|
||||
### subscript: expr | [expr] ':' [expr]
|
||||
|
||||
import sys, time
|
||||
c = sys.path[0]
|
||||
x = time.time()
|
||||
x = sys.modules['time'].time()
|
||||
a = '01234'
|
||||
c = a[0]
|
||||
c = a[-1]
|
||||
s = a[0:5]
|
||||
s = a[:5]
|
||||
s = a[0:]
|
||||
s = a[:]
|
||||
s = a[-5:]
|
||||
s = a[:-1]
|
||||
s = a[-4:-3]
|
||||
# A rough test of SF bug 1333982. http://python.org/sf/1333982
|
||||
# The testing here is fairly incomplete.
|
||||
# Test cases should include: commas with 1 and 2 colons
|
||||
d = {}
|
||||
d[1] = 1
|
||||
d[1,] = 2
|
||||
d[1,2] = 3
|
||||
d[1,2,3] = 4
|
||||
L = list(d)
|
||||
L.sort(key=lambda x: x if isinstance(x, tuple) else ())
|
||||
self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
|
||||
|
||||
def testAtoms(self):
|
||||
### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
|
||||
### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
|
||||
|
||||
x = (1)
|
||||
x = (1 or 2 or 3)
|
||||
x = (1 or 2 or 3, 2, 3)
|
||||
|
||||
x = []
|
||||
x = [1]
|
||||
x = [1 or 2 or 3]
|
||||
x = [1 or 2 or 3, 2, 3]
|
||||
x = []
|
||||
|
||||
x = {}
|
||||
x = {'one': 1}
|
||||
x = {'one': 1,}
|
||||
x = {'one' or 'two': 1 or 2}
|
||||
x = {'one': 1, 'two': 2}
|
||||
x = {'one': 1, 'two': 2,}
|
||||
x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
|
||||
|
||||
x = {'one'}
|
||||
x = {'one', 1,}
|
||||
x = {'one', 'two', 'three'}
|
||||
x = {2, 3, 4,}
|
||||
|
||||
x = x
|
||||
x = 'x'
|
||||
x = 123
|
||||
|
||||
### exprlist: expr (',' expr)* [',']
|
||||
### testlist: test (',' test)* [',']
|
||||
# These have been exercised enough above
|
||||
|
||||
def testClassdef(self):
|
||||
# 'class' NAME ['(' [testlist] ')'] ':' suite
|
||||
class B: pass
|
||||
class B2(): pass
|
||||
class C1(B): pass
|
||||
class C2(B): pass
|
||||
class D(C1, C2, B): pass
|
||||
class C:
|
||||
def meth1(self): pass
|
||||
def meth2(self, arg): pass
|
||||
def meth3(self, a1, a2): pass
|
||||
|
||||
# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||||
# decorators: decorator+
|
||||
# decorated: decorators (classdef | funcdef)
|
||||
def class_decorator(x): return x
|
||||
@class_decorator
|
||||
class G: pass
|
||||
|
||||
def testDictcomps(self):
|
||||
# dictorsetmaker: ( (test ':' test (comp_for |
|
||||
# (',' test ':' test)* [','])) |
|
||||
# (test (comp_for | (',' test)* [','])) )
|
||||
nums = [1, 2, 3]
|
||||
self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
|
||||
|
||||
def testListcomps(self):
|
||||
# list comprehension tests
|
||||
nums = [1, 2, 3, 4, 5]
|
||||
strs = ["Apple", "Banana", "Coconut"]
|
||||
spcs = [" Apple", " Banana ", "Coco nut "]
|
||||
|
||||
self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco nut'])
|
||||
self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
|
||||
self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
|
||||
self.assertEqual([(i, s) for i in nums for s in strs],
|
||||
[(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
|
||||
(2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
|
||||
(4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
|
||||
[(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
|
||||
(3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
|
||||
(5, 'Banana'), (5, 'Coconut')])
|
||||
self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
|
||||
[[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
|
||||
|
||||
def test_in_func(l):
|
||||
return [0 < x < 3 for x in l if x > 2]
|
||||
|
||||
self.assertEqual(test_in_func(nums), [False, False, False])
|
||||
|
||||
def test_nested_front():
|
||||
self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
|
||||
[[1, 2], [3, 4], [5, 6]])
|
||||
|
||||
test_nested_front()
|
||||
|
||||
check_syntax_error(self, "[i, s for i in nums for s in strs]")
|
||||
check_syntax_error(self, "[x if y]")
|
||||
|
||||
suppliers = [
|
||||
(1, "Boeing"),
|
||||
(2, "Ford"),
|
||||
(3, "Macdonalds")
|
||||
]
|
||||
|
||||
parts = [
|
||||
(10, "Airliner"),
|
||||
(20, "Engine"),
|
||||
(30, "Cheeseburger")
|
||||
]
|
||||
|
||||
suppart = [
|
||||
(1, 10), (1, 20), (2, 20), (3, 30)
|
||||
]
|
||||
|
||||
x = [
|
||||
(sname, pname)
|
||||
for (sno, sname) in suppliers
|
||||
for (pno, pname) in parts
|
||||
for (sp_sno, sp_pno) in suppart
|
||||
if sno == sp_sno and pno == sp_pno
|
||||
]
|
||||
|
||||
self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
|
||||
('Macdonalds', 'Cheeseburger')])
|
||||
|
||||
def testGenexps(self):
|
||||
# generator expression tests
|
||||
g = ([x for x in range(10)] for x in range(1))
|
||||
self.assertEqual(next(g), [x for x in range(10)])
|
||||
try:
|
||||
next(g)
|
||||
self.fail('should produce StopIteration exception')
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
a = 1
|
||||
try:
|
||||
g = (a for d in a)
|
||||
next(g)
|
||||
self.fail('should produce TypeError')
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
|
||||
self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
|
||||
|
||||
a = [x for x in range(10)]
|
||||
b = (x for x in (y for y in a))
|
||||
self.assertEqual(sum(b), sum([x for x in range(10)]))
|
||||
|
||||
self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
|
||||
self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
|
||||
self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
|
||||
self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
|
||||
check_syntax_error(self, "foo(x for x in range(10), 100)")
|
||||
check_syntax_error(self, "foo(100, x for x in range(10))")
|
||||
|
||||
def testComprehensionSpecials(self):
|
||||
# test for outmost iterable precomputation
|
||||
x = 10; g = (i for i in range(x)); x = 5
|
||||
self.assertEqual(len(list(g)), 10)
|
||||
|
||||
# This should hold, since we're only precomputing outmost iterable.
|
||||
x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
|
||||
x = 5; t = True;
|
||||
self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
|
||||
|
||||
# Grammar allows multiple adjacent 'if's in listcomps and genexps,
|
||||
# even though it's silly. Make sure it works (ifelse broke this.)
|
||||
self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
|
||||
self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
|
||||
|
||||
# verify unpacking single element tuples in listcomp/genexp.
|
||||
self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
|
||||
self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
|
||||
|
||||
def test_with_statement(self):
|
||||
class manager(object):
|
||||
def __enter__(self):
|
||||
return (1, 2)
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
with manager():
|
||||
pass
|
||||
with manager() as x:
|
||||
pass
|
||||
with manager() as (x, y):
|
||||
pass
|
||||
with manager(), manager():
|
||||
pass
|
||||
with manager() as x, manager() as y:
|
||||
pass
|
||||
with manager() as x, manager():
|
||||
pass
|
||||
|
||||
def testIfElseExpr(self):
|
||||
# Test ifelse expressions in various cases
|
||||
def _checkeval(msg, ret):
|
||||
"helper to check that evaluation of expressions is done correctly"
|
||||
print(x)
|
||||
return ret
|
||||
|
||||
# the next line is not allowed anymore
|
||||
#self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
|
||||
self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
|
||||
self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
|
||||
self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
|
||||
self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
|
||||
self.assertEqual((5 and 6 if 0 else 1), 1)
|
||||
self.assertEqual(((5 and 6) if 0 else 1), 1)
|
||||
self.assertEqual((5 and (6 if 1 else 1)), 6)
|
||||
self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
|
||||
self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
|
||||
self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
|
||||
self.assertEqual((not 5 if 1 else 1), False)
|
||||
self.assertEqual((not 5 if 0 else 1), 1)
|
||||
self.assertEqual((6 + 1 if 1 else 2), 7)
|
||||
self.assertEqual((6 - 1 if 1 else 2), 5)
|
||||
self.assertEqual((6 * 2 if 1 else 4), 12)
|
||||
self.assertEqual((6 / 2 if 1 else 3), 3)
|
||||
self.assertEqual((6 < 4 if 0 else 2), 2)
|
||||
|
||||
def testStringLiterals(self):
|
||||
x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
|
||||
x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
|
||||
x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
|
||||
x = "doesn't \"shrink\" does it"
|
||||
y = 'doesn\'t "shrink" does it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = "does \"shrink\" doesn't it"
|
||||
y = 'does "shrink" doesn\'t it'
|
||||
self.assert_(len(x) == 24 and x == y)
|
||||
x = f"""
|
||||
The "quick"
|
||||
brown fo{ok()}x
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
"""
|
||||
y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
|
||||
self.assertEqual(x, y)
|
||||
y = '''
|
||||
The "quick"
|
||||
brown fox
|
||||
jumps over
|
||||
the 'lazy' dog.
|
||||
'''
|
||||
self.assertEqual(x, y)
|
||||
y = "\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the 'lazy' dog.\n\
|
||||
"
|
||||
self.assertEqual(x, y)
|
||||
y = '\n\
|
||||
The \"quick\"\n\
|
||||
brown fox\n\
|
||||
jumps over\n\
|
||||
the \'lazy\' dog.\n\
|
||||
'
|
||||
self.assertEqual(x, y)
|
||||
|
||||
|
||||
def test_main():
|
||||
run_unittest(TokenTests, GrammarTests)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_main()
|
||||
+1572
File diff suppressed because it is too large
Load Diff
+1
@@ -0,0 +1 @@
|
||||
pass; print "hi"
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
def set_password(args):
|
||||
password = args.password
|
||||
while not password :
|
||||
password1 = getpass("" if args.quiet else "Provide password: ")
|
||||
password_repeat = getpass("" if args.quiet else "Repeat password: ")
|
||||
if password1 != password_repeat:
|
||||
print("Passwords do not match, try again")
|
||||
elif len(password1) < 4:
|
||||
print("Please provide at least 4 characters")
|
||||
else:
|
||||
password = password1
|
||||
|
||||
password_hash = passwd(password)
|
||||
cfg = BaseJSONConfigManager(config_dir=jupyter_config_dir())
|
||||
cfg.update('jupyter_notebook_config', {
|
||||
'NotebookApp': {
|
||||
'password': password_hash,
|
||||
}
|
||||
})
|
||||
if not args.quiet:
|
||||
print("password stored in config dir: %s" % jupyter_config_dir())
|
||||
|
||||
def main(argv):
|
||||
parser = argparse.ArgumentParser(argv[0])
|
||||
subparsers = parser.add_subparsers()
|
||||
parser_password = subparsers.add_parser('password', help='sets a password for your notebook server')
|
||||
parser_password.add_argument("password", help="password to set, if not given, a password will be queried for (NOTE: this may not be safe)",
|
||||
nargs="?")
|
||||
parser_password.add_argument("--quiet", help="suppress messages", action="store_true")
|
||||
parser_password.set_defaults(function=set_password)
|
||||
args = parser.parse_args(argv[1:])
|
||||
args.function(args)
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
print a
|
||||
|
||||
if b:
|
||||
if c:
|
||||
d
|
||||
e
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"name": "tree-sitter-python",
|
||||
"version": "0.20.2",
|
||||
"description": "Python grammar for tree-sitter",
|
||||
"main": "bindings/node",
|
||||
"keywords": [
|
||||
"parser",
|
||||
"lexer"
|
||||
],
|
||||
"author": "Max Brunsfeld",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nan": "^2.15.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tree-sitter-cli": "^0.20.1"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tree-sitter generate && node-gyp build",
|
||||
"test": "tree-sitter test && script/parse-examples",
|
||||
"parse": "tree-sitter parse",
|
||||
"test-windows": "tree-sitter test"
|
||||
},
|
||||
"repository": "https://github.com/tree-sitter/tree-sitter-python",
|
||||
"tree-sitter": [
|
||||
{
|
||||
"scope": "source.python",
|
||||
"file-types": [
|
||||
"py"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
; Identifier naming conventions
|
||||
|
||||
((identifier) @constructor
|
||||
(#match? @constructor "^[A-Z]"))
|
||||
|
||||
((identifier) @constant
|
||||
(#match? @constant "^[A-Z][A-Z_]*$"))
|
||||
|
||||
; Builtin functions
|
||||
|
||||
((call
|
||||
function: (identifier) @function.builtin)
|
||||
(#match?
|
||||
@function.builtin
|
||||
"^(abs|all|any|ascii|bin|bool|breakpoint|bytearray|bytes|callable|chr|classmethod|compile|complex|delattr|dict|dir|divmod|enumerate|eval|exec|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|isinstance|issubclass|iter|len|list|locals|map|max|memoryview|min|next|object|oct|open|ord|pow|print|property|range|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|vars|zip|__import__)$"))
|
||||
|
||||
; Function calls
|
||||
|
||||
(decorator) @function
|
||||
|
||||
(call
|
||||
function: (attribute attribute: (identifier) @function.method))
|
||||
(call
|
||||
function: (identifier) @function)
|
||||
|
||||
; Function definitions
|
||||
|
||||
(function_definition
|
||||
name: (identifier) @function)
|
||||
|
||||
(identifier) @variable
|
||||
(attribute attribute: (identifier) @property)
|
||||
(type (identifier) @type)
|
||||
|
||||
; Literals
|
||||
|
||||
[
|
||||
(none)
|
||||
(true)
|
||||
(false)
|
||||
] @constant.builtin
|
||||
|
||||
[
|
||||
(integer)
|
||||
(float)
|
||||
] @number
|
||||
|
||||
(comment) @comment
|
||||
(string) @string
|
||||
(escape_sequence) @escape
|
||||
|
||||
(interpolation
|
||||
"{" @punctuation.special
|
||||
"}" @punctuation.special) @embedded
|
||||
|
||||
[
|
||||
"-"
|
||||
"-="
|
||||
"!="
|
||||
"*"
|
||||
"**"
|
||||
"**="
|
||||
"*="
|
||||
"/"
|
||||
"//"
|
||||
"//="
|
||||
"/="
|
||||
"&"
|
||||
"%"
|
||||
"%="
|
||||
"^"
|
||||
"+"
|
||||
"->"
|
||||
"+="
|
||||
"<"
|
||||
"<<"
|
||||
"<="
|
||||
"<>"
|
||||
"="
|
||||
":="
|
||||
"=="
|
||||
">"
|
||||
">="
|
||||
">>"
|
||||
"|"
|
||||
"~"
|
||||
"and"
|
||||
"in"
|
||||
"is"
|
||||
"not"
|
||||
"or"
|
||||
] @operator
|
||||
|
||||
[
|
||||
"as"
|
||||
"assert"
|
||||
"async"
|
||||
"await"
|
||||
"break"
|
||||
"class"
|
||||
"continue"
|
||||
"def"
|
||||
"del"
|
||||
"elif"
|
||||
"else"
|
||||
"except"
|
||||
"exec"
|
||||
"finally"
|
||||
"for"
|
||||
"from"
|
||||
"global"
|
||||
"if"
|
||||
"import"
|
||||
"lambda"
|
||||
"nonlocal"
|
||||
"pass"
|
||||
"print"
|
||||
"raise"
|
||||
"return"
|
||||
"try"
|
||||
"while"
|
||||
"with"
|
||||
"yield"
|
||||
"match"
|
||||
"case"
|
||||
] @keyword
|
||||
@@ -0,0 +1,12 @@
|
||||
(class_definition
|
||||
name: (identifier) @name) @definition.class
|
||||
|
||||
(function_definition
|
||||
name: (identifier) @name) @definition.function
|
||||
|
||||
(call
|
||||
function: [
|
||||
(identifier) @name
|
||||
(attribute
|
||||
attribute: (identifier) @name)
|
||||
]) @reference.call
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
function checkout() {
|
||||
repo=$1; url=$2; sha=$3
|
||||
|
||||
if [ ! -d "$repo" ]; then
|
||||
git clone "https://github.com/$url" "$repo"
|
||||
fi
|
||||
|
||||
pushd "$repo"
|
||||
git fetch && git reset --hard "$sha"
|
||||
popd
|
||||
}
|
||||
|
||||
checkout examples/numpy numpy/numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
|
||||
checkout examples/django django/django 01974d7f7549b2dca2a729c3c1a1ea7d4585eb3a
|
||||
checkout examples/flask pallets/flask de464c03e134127140e5622e230790806a133ff9
|
||||
|
||||
known_failures="$(cat script/known_failures.txt)"
|
||||
|
||||
tree-sitter parse -q \
|
||||
'examples/**/*.py' \
|
||||
$(for file in $known_failures; do echo "!${file}"; done)
|
||||
|
||||
example_count=$(find examples -name '*.py' | wc -l)
|
||||
failure_count=$(wc -w <<< "$known_failures")
|
||||
success_count=$(( $example_count - $failure_count ))
|
||||
success_percent=$(bc -l <<< "100*${success_count}/${example_count}")
|
||||
|
||||
printf \
|
||||
"Successfully parsed %d of %d example files (%.1f%%)\n" \
|
||||
$success_count $example_count $success_percent
|
||||
+5395
File diff suppressed because it is too large
Load Diff
+3152
File diff suppressed because it is too large
Load Diff
+107777
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,414 @@
|
||||
#include <tree_sitter/parser.h>
|
||||
#include <vector>
|
||||
#include <cwctype>
|
||||
#include <cstring>
|
||||
#include <cassert>
|
||||
#include <stdio.h>
|
||||
namespace {
|
||||
|
||||
using std::vector;
|
||||
using std::iswspace;
|
||||
using std::memcpy;
|
||||
|
||||
enum TokenType {
|
||||
NEWLINE,
|
||||
INDENT,
|
||||
DEDENT,
|
||||
STRING_START,
|
||||
STRING_CONTENT,
|
||||
STRING_END,
|
||||
COMMENT,
|
||||
CLOSE_PAREN,
|
||||
CLOSE_BRACKET,
|
||||
CLOSE_BRACE,
|
||||
};
|
||||
|
||||
struct Delimiter {
|
||||
enum {
|
||||
SingleQuote = 1 << 0,
|
||||
DoubleQuote = 1 << 1,
|
||||
BackQuote = 1 << 2,
|
||||
Raw = 1 << 3,
|
||||
Format = 1 << 4,
|
||||
Triple = 1 << 5,
|
||||
Bytes = 1 << 6,
|
||||
};
|
||||
|
||||
Delimiter() : flags(0) {}
|
||||
|
||||
bool is_format() const {
|
||||
return flags & Format;
|
||||
}
|
||||
|
||||
bool is_raw() const {
|
||||
return flags & Raw;
|
||||
}
|
||||
|
||||
bool is_triple() const {
|
||||
return flags & Triple;
|
||||
}
|
||||
|
||||
bool is_bytes() const {
|
||||
return flags & Bytes;
|
||||
}
|
||||
|
||||
int32_t end_character() const {
|
||||
if (flags & SingleQuote) return '\'';
|
||||
if (flags & DoubleQuote) return '"';
|
||||
if (flags & BackQuote) return '`';
|
||||
return 0;
|
||||
}
|
||||
|
||||
void set_format() {
|
||||
flags |= Format;
|
||||
}
|
||||
|
||||
void set_raw() {
|
||||
flags |= Raw;
|
||||
}
|
||||
|
||||
void set_triple() {
|
||||
flags |= Triple;
|
||||
}
|
||||
|
||||
void set_bytes() {
|
||||
flags |= Bytes;
|
||||
}
|
||||
|
||||
void set_end_character(int32_t character) {
|
||||
switch (character) {
|
||||
case '\'':
|
||||
flags |= SingleQuote;
|
||||
break;
|
||||
case '"':
|
||||
flags |= DoubleQuote;
|
||||
break;
|
||||
case '`':
|
||||
flags |= BackQuote;
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
char flags;
|
||||
};
|
||||
|
||||
struct Scanner {
|
||||
Scanner() {
|
||||
assert(sizeof(Delimiter) == sizeof(char));
|
||||
deserialize(NULL, 0);
|
||||
}
|
||||
|
||||
unsigned serialize(char *buffer) {
|
||||
size_t i = 0;
|
||||
|
||||
size_t delimiter_count = delimiter_stack.size();
|
||||
if (delimiter_count > UINT8_MAX) delimiter_count = UINT8_MAX;
|
||||
buffer[i++] = delimiter_count;
|
||||
|
||||
if (delimiter_count > 0) {
|
||||
memcpy(&buffer[i], delimiter_stack.data(), delimiter_count);
|
||||
}
|
||||
i += delimiter_count;
|
||||
|
||||
vector<uint16_t>::iterator
|
||||
iter = indent_length_stack.begin() + 1,
|
||||
end = indent_length_stack.end();
|
||||
|
||||
for (; iter != end && i < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; ++iter) {
|
||||
buffer[i++] = *iter;
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
void deserialize(const char *buffer, unsigned length) {
|
||||
delimiter_stack.clear();
|
||||
indent_length_stack.clear();
|
||||
indent_length_stack.push_back(0);
|
||||
|
||||
if (length > 0) {
|
||||
size_t i = 0;
|
||||
|
||||
size_t delimiter_count = (uint8_t)buffer[i++];
|
||||
delimiter_stack.resize(delimiter_count);
|
||||
if (delimiter_count > 0) {
|
||||
memcpy(delimiter_stack.data(), &buffer[i], delimiter_count);
|
||||
}
|
||||
i += delimiter_count;
|
||||
|
||||
for (; i < length; i++) {
|
||||
indent_length_stack.push_back(buffer[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void advance(TSLexer *lexer) {
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
|
||||
void skip(TSLexer *lexer) {
|
||||
lexer->advance(lexer, true);
|
||||
}
|
||||
|
||||
bool scan(TSLexer *lexer, const bool *valid_symbols) {
|
||||
bool error_recovery_mode = valid_symbols[STRING_CONTENT] && valid_symbols[INDENT];
|
||||
bool within_brackets = valid_symbols[CLOSE_BRACE] || valid_symbols[CLOSE_PAREN] || valid_symbols[CLOSE_BRACKET];
|
||||
|
||||
if (valid_symbols[STRING_CONTENT] && !delimiter_stack.empty() && !error_recovery_mode) {
|
||||
Delimiter delimiter = delimiter_stack.back();
|
||||
int32_t end_character = delimiter.end_character();
|
||||
bool has_content = false;
|
||||
while (lexer->lookahead) {
|
||||
if ((lexer->lookahead == '{' || lexer->lookahead == '}') && delimiter.is_format()) {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
return has_content;
|
||||
} else if (lexer->lookahead == '\\') {
|
||||
if (delimiter.is_raw()) {
|
||||
// Step over the backslash.
|
||||
lexer->advance(lexer, false);
|
||||
// Step over any escaped quotes.
|
||||
if (lexer->lookahead == delimiter.end_character() || lexer->lookahead == '\\') {
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
continue;
|
||||
} else if (delimiter.is_bytes()) {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->advance(lexer, false);
|
||||
if (lexer->lookahead == 'N' || lexer->lookahead == 'u' || lexer->lookahead == 'U') {
|
||||
// In bytes string, \N{...}, \uXXXX and \UXXXXXXXX are not escape sequences
|
||||
// https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
|
||||
lexer->advance(lexer, false);
|
||||
} else {
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
return has_content;
|
||||
}
|
||||
} else {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
return has_content;
|
||||
}
|
||||
} else if (lexer->lookahead == end_character) {
|
||||
if (delimiter.is_triple()) {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->advance(lexer, false);
|
||||
if (lexer->lookahead == end_character) {
|
||||
lexer->advance(lexer, false);
|
||||
if (lexer->lookahead == end_character) {
|
||||
if (has_content) {
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
} else {
|
||||
lexer->advance(lexer, false);
|
||||
lexer->mark_end(lexer);
|
||||
delimiter_stack.pop_back();
|
||||
lexer->result_symbol = STRING_END;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
lexer->mark_end(lexer);
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if (has_content) {
|
||||
lexer->result_symbol = STRING_CONTENT;
|
||||
} else {
|
||||
lexer->advance(lexer, false);
|
||||
delimiter_stack.pop_back();
|
||||
lexer->result_symbol = STRING_END;
|
||||
}
|
||||
lexer->mark_end(lexer);
|
||||
return true;
|
||||
}
|
||||
} else if (lexer->lookahead == '\n' && has_content && !delimiter.is_triple()) {
|
||||
return false;
|
||||
}
|
||||
advance(lexer);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
|
||||
lexer->mark_end(lexer);
|
||||
|
||||
bool found_end_of_line = false;
|
||||
uint32_t indent_length = 0;
|
||||
int32_t first_comment_indent_length = -1;
|
||||
for (;;) {
|
||||
if (lexer->lookahead == '\n') {
|
||||
found_end_of_line = true;
|
||||
indent_length = 0;
|
||||
skip(lexer);
|
||||
} else if (lexer->lookahead == ' ') {
|
||||
indent_length++;
|
||||
skip(lexer);
|
||||
} else if (lexer->lookahead == '\r') {
|
||||
indent_length = 0;
|
||||
skip(lexer);
|
||||
} else if (lexer->lookahead == '\t') {
|
||||
indent_length += 8;
|
||||
skip(lexer);
|
||||
} else if (lexer->lookahead == '#') {
|
||||
if (first_comment_indent_length == -1) {
|
||||
first_comment_indent_length = (int32_t)indent_length;
|
||||
}
|
||||
while (lexer->lookahead && lexer->lookahead != '\n') {
|
||||
skip(lexer);
|
||||
}
|
||||
skip(lexer);
|
||||
indent_length = 0;
|
||||
} else if (lexer->lookahead == '\\') {
|
||||
skip(lexer);
|
||||
if (lexer->lookahead == '\r') {
|
||||
skip(lexer);
|
||||
}
|
||||
if (lexer->lookahead == '\n') {
|
||||
skip(lexer);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (lexer->lookahead == '\f') {
|
||||
indent_length = 0;
|
||||
skip(lexer);
|
||||
} else if (lexer->lookahead == 0) {
|
||||
indent_length = 0;
|
||||
found_end_of_line = true;
|
||||
break;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (found_end_of_line) {
|
||||
if (!indent_length_stack.empty()) {
|
||||
uint16_t current_indent_length = indent_length_stack.back();
|
||||
|
||||
if (
|
||||
valid_symbols[INDENT] &&
|
||||
indent_length > current_indent_length
|
||||
) {
|
||||
indent_length_stack.push_back(indent_length);
|
||||
lexer->result_symbol = INDENT;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
(valid_symbols[DEDENT] || (!valid_symbols[NEWLINE] && !within_brackets)) &&
|
||||
indent_length < current_indent_length &&
|
||||
|
||||
// Wait to create a dedent token until we've consumed any comments
|
||||
// whose indentation matches the current block.
|
||||
first_comment_indent_length < (int32_t)current_indent_length
|
||||
) {
|
||||
indent_length_stack.pop_back();
|
||||
lexer->result_symbol = DEDENT;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (valid_symbols[NEWLINE] && !error_recovery_mode) {
|
||||
lexer->result_symbol = NEWLINE;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (first_comment_indent_length == -1 && valid_symbols[STRING_START]) {
|
||||
Delimiter delimiter;
|
||||
|
||||
bool has_flags = false;
|
||||
while (lexer->lookahead) {
|
||||
if (lexer->lookahead == 'f' || lexer->lookahead == 'F') {
|
||||
delimiter.set_format();
|
||||
} else if (lexer->lookahead == 'r' || lexer->lookahead == 'R') {
|
||||
delimiter.set_raw();
|
||||
} else if (lexer->lookahead == 'b' || lexer->lookahead == 'B') {
|
||||
delimiter.set_bytes();
|
||||
} else if (lexer->lookahead != 'u' && lexer->lookahead != 'U') {
|
||||
break;
|
||||
}
|
||||
has_flags = true;
|
||||
advance(lexer);
|
||||
}
|
||||
|
||||
if (lexer->lookahead == '`') {
|
||||
delimiter.set_end_character('`');
|
||||
advance(lexer);
|
||||
lexer->mark_end(lexer);
|
||||
} else if (lexer->lookahead == '\'') {
|
||||
delimiter.set_end_character('\'');
|
||||
advance(lexer);
|
||||
lexer->mark_end(lexer);
|
||||
if (lexer->lookahead == '\'') {
|
||||
advance(lexer);
|
||||
if (lexer->lookahead == '\'') {
|
||||
advance(lexer);
|
||||
lexer->mark_end(lexer);
|
||||
delimiter.set_triple();
|
||||
}
|
||||
}
|
||||
} else if (lexer->lookahead == '"') {
|
||||
delimiter.set_end_character('"');
|
||||
advance(lexer);
|
||||
lexer->mark_end(lexer);
|
||||
if (lexer->lookahead == '"') {
|
||||
advance(lexer);
|
||||
if (lexer->lookahead == '"') {
|
||||
advance(lexer);
|
||||
lexer->mark_end(lexer);
|
||||
delimiter.set_triple();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (delimiter.end_character()) {
|
||||
delimiter_stack.push_back(delimiter);
|
||||
lexer->result_symbol = STRING_START;
|
||||
return true;
|
||||
} else if (has_flags) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
vector<uint16_t> indent_length_stack;
|
||||
vector<Delimiter> delimiter_stack;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
|
||||
void *tree_sitter_python_external_scanner_create() {
|
||||
return new Scanner();
|
||||
}
|
||||
|
||||
bool tree_sitter_python_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
const bool *valid_symbols) {
|
||||
Scanner *scanner = static_cast<Scanner *>(payload);
|
||||
return scanner->scan(lexer, valid_symbols);
|
||||
}
|
||||
|
||||
unsigned tree_sitter_python_external_scanner_serialize(void *payload, char *buffer) {
|
||||
Scanner *scanner = static_cast<Scanner *>(payload);
|
||||
return scanner->serialize(buffer);
|
||||
}
|
||||
|
||||
void tree_sitter_python_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {
|
||||
Scanner *scanner = static_cast<Scanner *>(payload);
|
||||
scanner->deserialize(buffer, length);
|
||||
}
|
||||
|
||||
void tree_sitter_python_external_scanner_destroy(void *payload) {
|
||||
Scanner *scanner = static_cast<Scanner *>(payload);
|
||||
delete scanner;
|
||||
}
|
||||
|
||||
}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
#ifndef TREE_SITTER_PARSER_H_
|
||||
#define TREE_SITTER_PARSER_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ts_builtin_sym_error ((TSSymbol)-1)
|
||||
#define ts_builtin_sym_end 0
|
||||
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
|
||||
|
||||
typedef uint16_t TSStateId;
|
||||
|
||||
#ifndef TREE_SITTER_API_H_
|
||||
typedef uint16_t TSSymbol;
|
||||
typedef uint16_t TSFieldId;
|
||||
typedef struct TSLanguage TSLanguage;
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
TSFieldId field_id;
|
||||
uint8_t child_index;
|
||||
bool inherited;
|
||||
} TSFieldMapEntry;
|
||||
|
||||
typedef struct {
|
||||
uint16_t index;
|
||||
uint16_t length;
|
||||
} TSFieldMapSlice;
|
||||
|
||||
typedef struct {
|
||||
bool visible;
|
||||
bool named;
|
||||
bool supertype;
|
||||
} TSSymbolMetadata;
|
||||
|
||||
typedef struct TSLexer TSLexer;
|
||||
|
||||
struct TSLexer {
|
||||
int32_t lookahead;
|
||||
TSSymbol result_symbol;
|
||||
void (*advance)(TSLexer *, bool);
|
||||
void (*mark_end)(TSLexer *);
|
||||
uint32_t (*get_column)(TSLexer *);
|
||||
bool (*is_at_included_range_start)(const TSLexer *);
|
||||
bool (*eof)(const TSLexer *);
|
||||
};
|
||||
|
||||
typedef enum {
|
||||
TSParseActionTypeShift,
|
||||
TSParseActionTypeReduce,
|
||||
TSParseActionTypeAccept,
|
||||
TSParseActionTypeRecover,
|
||||
} TSParseActionType;
|
||||
|
||||
typedef union {
|
||||
struct {
|
||||
uint8_t type;
|
||||
TSStateId state;
|
||||
bool extra;
|
||||
bool repetition;
|
||||
} shift;
|
||||
struct {
|
||||
uint8_t type;
|
||||
uint8_t child_count;
|
||||
TSSymbol symbol;
|
||||
int16_t dynamic_precedence;
|
||||
uint16_t production_id;
|
||||
} reduce;
|
||||
uint8_t type;
|
||||
} TSParseAction;
|
||||
|
||||
typedef struct {
|
||||
uint16_t lex_state;
|
||||
uint16_t external_lex_state;
|
||||
} TSLexMode;
|
||||
|
||||
typedef union {
|
||||
TSParseAction action;
|
||||
struct {
|
||||
uint8_t count;
|
||||
bool reusable;
|
||||
} entry;
|
||||
} TSParseActionEntry;
|
||||
|
||||
struct TSLanguage {
|
||||
uint32_t version;
|
||||
uint32_t symbol_count;
|
||||
uint32_t alias_count;
|
||||
uint32_t token_count;
|
||||
uint32_t external_token_count;
|
||||
uint32_t state_count;
|
||||
uint32_t large_state_count;
|
||||
uint32_t production_id_count;
|
||||
uint32_t field_count;
|
||||
uint16_t max_alias_sequence_length;
|
||||
const uint16_t *parse_table;
|
||||
const uint16_t *small_parse_table;
|
||||
const uint32_t *small_parse_table_map;
|
||||
const TSParseActionEntry *parse_actions;
|
||||
const char * const *symbol_names;
|
||||
const char * const *field_names;
|
||||
const TSFieldMapSlice *field_map_slices;
|
||||
const TSFieldMapEntry *field_map_entries;
|
||||
const TSSymbolMetadata *symbol_metadata;
|
||||
const TSSymbol *public_symbol_map;
|
||||
const uint16_t *alias_map;
|
||||
const TSSymbol *alias_sequences;
|
||||
const TSLexMode *lex_modes;
|
||||
bool (*lex_fn)(TSLexer *, TSStateId);
|
||||
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
|
||||
TSSymbol keyword_capture_token;
|
||||
struct {
|
||||
const bool *states;
|
||||
const TSSymbol *symbol_map;
|
||||
void *(*create)(void);
|
||||
void (*destroy)(void *);
|
||||
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
|
||||
unsigned (*serialize)(void *, char *);
|
||||
void (*deserialize)(void *, const char *, unsigned);
|
||||
} external_scanner;
|
||||
const TSStateId *primary_state_ids;
|
||||
};
|
||||
|
||||
/*
|
||||
* Lexer Macros
|
||||
*/
|
||||
|
||||
#define START_LEXER() \
|
||||
bool result = false; \
|
||||
bool skip = false; \
|
||||
bool eof = false; \
|
||||
int32_t lookahead; \
|
||||
goto start; \
|
||||
next_state: \
|
||||
lexer->advance(lexer, skip); \
|
||||
start: \
|
||||
skip = false; \
|
||||
lookahead = lexer->lookahead;
|
||||
|
||||
#define ADVANCE(state_value) \
|
||||
{ \
|
||||
state = state_value; \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define SKIP(state_value) \
|
||||
{ \
|
||||
skip = true; \
|
||||
state = state_value; \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define ACCEPT_TOKEN(symbol_value) \
|
||||
result = true; \
|
||||
lexer->result_symbol = symbol_value; \
|
||||
lexer->mark_end(lexer);
|
||||
|
||||
#define END_STATE() return result;
|
||||
|
||||
/*
|
||||
* Parse Table Macros
|
||||
*/
|
||||
|
||||
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
|
||||
|
||||
#define STATE(id) id
|
||||
|
||||
#define ACTIONS(id) id
|
||||
|
||||
#define SHIFT(state_value) \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.state = state_value \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define SHIFT_REPEAT(state_value) \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.state = state_value, \
|
||||
.repetition = true \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define SHIFT_EXTRA() \
|
||||
{{ \
|
||||
.shift = { \
|
||||
.type = TSParseActionTypeShift, \
|
||||
.extra = true \
|
||||
} \
|
||||
}}
|
||||
|
||||
#define REDUCE(symbol_val, child_count_val, ...) \
|
||||
{{ \
|
||||
.reduce = { \
|
||||
.type = TSParseActionTypeReduce, \
|
||||
.symbol = symbol_val, \
|
||||
.child_count = child_count_val, \
|
||||
__VA_ARGS__ \
|
||||
}, \
|
||||
}}
|
||||
|
||||
#define RECOVER() \
|
||||
{{ \
|
||||
.type = TSParseActionTypeRecover \
|
||||
}}
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{{ \
|
||||
.type = TSParseActionTypeAccept \
|
||||
}}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TREE_SITTER_PARSER_H_
|
||||
@@ -0,0 +1,27 @@
|
||||
====================================
|
||||
An error before a string literal
|
||||
====================================
|
||||
|
||||
def a(b):
|
||||
c.
|
||||
|
||||
"""
|
||||
d
|
||||
"""
|
||||
|
||||
e
|
||||
|
||||
---
|
||||
|
||||
(module
|
||||
(function_definition
|
||||
(identifier)
|
||||
(parameters
|
||||
(identifier))
|
||||
(ERROR
|
||||
(identifier))
|
||||
(block
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(identifier)))))
|
||||
+1003
File diff suppressed because it is too large
Load Diff
+828
@@ -0,0 +1,828 @@
|
||||
================================================================================
|
||||
Integers
|
||||
================================================================================
|
||||
|
||||
-1
|
||||
0xDEAD
|
||||
0XDEAD
|
||||
1j
|
||||
-1j
|
||||
0o123
|
||||
0O123
|
||||
0b001
|
||||
0B001
|
||||
1_1
|
||||
0B1_1
|
||||
0O1_1
|
||||
0L
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(unary_operator
|
||||
(integer)))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(unary_operator
|
||||
(integer)))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer))
|
||||
(expression_statement
|
||||
(integer)))
|
||||
|
||||
================================================================================
|
||||
Floats
|
||||
================================================================================
|
||||
|
||||
-.6_6
|
||||
+.1_1
|
||||
123.4123
|
||||
123.123J
|
||||
1_1.3_1
|
||||
1_1.
|
||||
1e+3_4j
|
||||
.3e1_4
|
||||
1_0.l
|
||||
.1l
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(unary_operator
|
||||
(float)))
|
||||
(expression_statement
|
||||
(unary_operator
|
||||
(float)))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float)))
|
||||
|
||||
================================================================================
|
||||
Scientific Notation Floats
|
||||
================================================================================
|
||||
|
||||
1e322
|
||||
1e-3
|
||||
1e+3
|
||||
1.8e10
|
||||
1.e10
|
||||
-1e10
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(float))
|
||||
(expression_statement
|
||||
(unary_operator
|
||||
(float))))
|
||||
|
||||
================================================================================
|
||||
Strings
|
||||
================================================================================
|
||||
|
||||
"I'm ok"
|
||||
'"ok"'
|
||||
UR'bye'
|
||||
b'sup'
|
||||
B"sup"
|
||||
`1`
|
||||
"\\"
|
||||
"/"
|
||||
"multiline \
|
||||
string"
|
||||
b"\x12\u12\U12\x13\N{WINKING FACE}"
|
||||
"\xab\123\'\"\a\b\f\r\n\t\v\\"
|
||||
"\xgh\o123\p\q\c\d\e\u12\U1234"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence))))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence))))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence)
|
||||
(escape_sequence))))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence))))
|
||||
(expression_statement
|
||||
(string (string_content))))
|
||||
|
||||
================================================================================
|
||||
Raw strings
|
||||
================================================================================
|
||||
|
||||
'ab\x00cd'
|
||||
"\n"
|
||||
|
||||
# no escape sequences in these
|
||||
r'ab\x00cd'
|
||||
ur"\n"
|
||||
|
||||
# raw f-string
|
||||
fr"\{0}"
|
||||
|
||||
r"\\"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence))))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence))))
|
||||
(comment)
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(comment)
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(integer))))
|
||||
(expression_statement
|
||||
(string)))
|
||||
|
||||
================================================================================
|
||||
Raw strings with escaped quotes
|
||||
================================================================================
|
||||
|
||||
re.compile(r"(\n|\A)#include\s*['\"]"
|
||||
r"(?P<name>[\w\d./\\]+[.]src)['\"]")
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(concatenated_string
|
||||
(string (string_content))
|
||||
(string (string_content)))))))
|
||||
|
||||
================================================================================
|
||||
Format strings
|
||||
================================================================================
|
||||
|
||||
# nested!
|
||||
f"a {b(f'c {e} d')} e"
|
||||
f"""a"{b}c"""
|
||||
f"""a""{b}c"""
|
||||
f"a {{}} e"
|
||||
f"a {b}}}"
|
||||
f"a {{{b}"
|
||||
f"a {{b}}"
|
||||
f"a {{{b}}}"
|
||||
f"{c,}"
|
||||
f"{yield d}"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(comment)
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))
|
||||
(string_content)))))
|
||||
(string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))
|
||||
(string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))
|
||||
(string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))
|
||||
(string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier))
|
||||
(string_content)))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation
|
||||
(expression_list (identifier)))))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation
|
||||
(yield
|
||||
(identifier))))))
|
||||
|
||||
================================================================================
|
||||
Format strings with format specifiers
|
||||
================================================================================
|
||||
|
||||
f"a {b:2} {c:34.5}"
|
||||
f"{b:{c.d}.{d.e}}"
|
||||
f"{a:#06x}"
|
||||
f"{a=}"
|
||||
f"{a=:.2f}"
|
||||
f"{value:{width + padding!r}.{precision}}"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier)
|
||||
(format_specifier))
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier)
|
||||
(format_specifier))))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation
|
||||
(identifier)
|
||||
(format_specifier
|
||||
(format_expression
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier)))
|
||||
(format_expression
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier)))))))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation (identifier) (format_specifier))))
|
||||
(expression_statement
|
||||
(string (interpolation (identifier))))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation (identifier) (format_specifier))))
|
||||
(expression_statement
|
||||
(string
|
||||
(interpolation
|
||||
(identifier)
|
||||
(format_specifier
|
||||
(format_expression
|
||||
(binary_operator
|
||||
(identifier)
|
||||
(identifier))
|
||||
(type_conversion))
|
||||
(format_expression
|
||||
(identifier)))))))
|
||||
|
||||
================================================================================
|
||||
Unicode escape sequences
|
||||
================================================================================
|
||||
|
||||
"\x12 \123 \u1234"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)))))
|
||||
|
||||
================================================================================
|
||||
Other primitives
|
||||
================================================================================
|
||||
|
||||
True
|
||||
False
|
||||
None
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(true))
|
||||
(expression_statement
|
||||
(false))
|
||||
(expression_statement
|
||||
(none)))
|
||||
|
||||
================================================================================
|
||||
Concatenated strings
|
||||
================================================================================
|
||||
|
||||
"one" "two" "three"
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(concatenated_string
|
||||
(string (string_content))
|
||||
(string (string_content))
|
||||
(string (string_content)))))
|
||||
|
||||
================================================================================
|
||||
Multi-line strings
|
||||
================================================================================
|
||||
|
||||
"""
|
||||
A double quote hello,
|
||||
without double or single quotes.
|
||||
"""
|
||||
|
||||
"""
|
||||
A double quote "hello",
|
||||
with double quotes.
|
||||
"""
|
||||
|
||||
"""
|
||||
A double quote 'hello',
|
||||
with single quotes.
|
||||
"""
|
||||
|
||||
'''
|
||||
A single quote hello,
|
||||
without double or single quotes.
|
||||
'''
|
||||
|
||||
'''
|
||||
A single quote 'hello',
|
||||
with single quotes.
|
||||
'''
|
||||
|
||||
'''
|
||||
A single quote "hello",
|
||||
with double quotes.
|
||||
'''
|
||||
|
||||
"""
|
||||
A double quote hello\n\
|
||||
with an escaped newline\n\
|
||||
and another escaped newline\n\
|
||||
"""
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content)))
|
||||
(expression_statement
|
||||
(string (string_content
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)
|
||||
(escape_sequence)))))
|
||||
|
||||
================================================================================
|
||||
Lists
|
||||
================================================================================
|
||||
|
||||
[a, b, [c, d]]
|
||||
[*()]
|
||||
[*[]]
|
||||
[*a]
|
||||
[*a.b]
|
||||
[*a[b].c]
|
||||
[*a()]
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(list
|
||||
(identifier)
|
||||
(identifier)
|
||||
(list
|
||||
(identifier)
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(tuple))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(list))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier)))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(attribute
|
||||
(subscript
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier)))))
|
||||
(expression_statement
|
||||
(list
|
||||
(list_splat
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list))))))
|
||||
|
||||
================================================================================
|
||||
List comprehensions
|
||||
================================================================================
|
||||
|
||||
[a + b for (a, b) in items]
|
||||
[a for b in c for a in b]
|
||||
[(x,y) for x in [1,2,3] for y in [1,2,3] if True]
|
||||
[a for a in lambda: True, lambda: False if a()]
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(list_comprehension
|
||||
(binary_operator
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(tuple_pattern
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(list_comprehension
|
||||
(identifier)
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(list_comprehension
|
||||
(tuple
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(list
|
||||
(integer)
|
||||
(integer)
|
||||
(integer)))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(list
|
||||
(integer)
|
||||
(integer)
|
||||
(integer)))
|
||||
(if_clause
|
||||
(true))))
|
||||
(expression_statement
|
||||
(list_comprehension
|
||||
(identifier)
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(lambda
|
||||
(true))
|
||||
(lambda
|
||||
(false)))
|
||||
(if_clause
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list))))))
|
||||
|
||||
================================================================================
|
||||
Dictionaries
|
||||
================================================================================
|
||||
|
||||
{a: 1, b: 2}
|
||||
{}
|
||||
{**{}}
|
||||
{**a}
|
||||
{**a.b}
|
||||
{**a[b].c}
|
||||
{**a()}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(pair
|
||||
(identifier)
|
||||
(integer))
|
||||
(pair
|
||||
(identifier)
|
||||
(integer))))
|
||||
(expression_statement
|
||||
(dictionary))
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(dictionary_splat
|
||||
(dictionary))))
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(dictionary_splat
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(dictionary_splat
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier)))))
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(dictionary_splat
|
||||
(attribute
|
||||
(subscript
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier)))))
|
||||
(expression_statement
|
||||
(dictionary
|
||||
(dictionary_splat
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list))))))
|
||||
|
||||
================================================================================
|
||||
Dictionary comprehensions
|
||||
================================================================================
|
||||
|
||||
{a: b for a, b in items}
|
||||
{a: b for c in d for e in items}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(dictionary_comprehension
|
||||
(pair
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(dictionary_comprehension
|
||||
(pair
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier)))))
|
||||
|
||||
================================================================================
|
||||
Sets
|
||||
================================================================================
|
||||
|
||||
{a, b, c,}
|
||||
{*{}}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(set
|
||||
(identifier)
|
||||
(identifier)
|
||||
(identifier)))
|
||||
(expression_statement
|
||||
(set
|
||||
(list_splat
|
||||
(dictionary)))))
|
||||
|
||||
================================================================================
|
||||
Set comprehensions
|
||||
================================================================================
|
||||
|
||||
{a[b][c] for a, b, c in items}
|
||||
{r for s in qs for n in ms}
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(set_comprehension
|
||||
(subscript
|
||||
(subscript
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(set_comprehension
|
||||
(identifier)
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier)))))
|
||||
|
||||
================================================================================
|
||||
Simple Tuples
|
||||
================================================================================
|
||||
|
||||
()
|
||||
(a, b)
|
||||
(a, b, c,)
|
||||
(print, exec)
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(tuple))
|
||||
(expression_statement
|
||||
(tuple
|
||||
(identifier)
|
||||
(identifier)))
|
||||
(expression_statement
|
||||
(tuple
|
||||
(identifier)
|
||||
(identifier)
|
||||
(identifier)))
|
||||
(expression_statement
|
||||
(tuple
|
||||
(identifier)
|
||||
(identifier))))
|
||||
|
||||
================================================================================
|
||||
Generator expression
|
||||
================================================================================
|
||||
|
||||
(a[b][c] for a, b, c in items)
|
||||
dict((a, b) for a, b in d)
|
||||
(a for b in c for d in e,)
|
||||
(x for x in range(1, 10))
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(generator_expression
|
||||
(subscript
|
||||
(subscript
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(call
|
||||
(identifier)
|
||||
(generator_expression
|
||||
(tuple
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier)))))
|
||||
(expression_statement
|
||||
(generator_expression
|
||||
(identifier)
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(generator_expression
|
||||
(identifier)
|
||||
(for_in_clause
|
||||
(identifier)
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(integer)
|
||||
(integer)))))))
|
||||
+588
@@ -0,0 +1,588 @@
|
||||
================================================================================
|
||||
Matching specific values
|
||||
================================================================================
|
||||
|
||||
match command.split():
|
||||
case ["quit"]:
|
||||
print("Goodbye!")
|
||||
quit_game()
|
||||
case ["look"]:
|
||||
current_room.describe()
|
||||
case ["get", obj]:
|
||||
character.get(obj, current_room)
|
||||
case ["go", direction]:
|
||||
current_room = current_room.neighbor(direction)
|
||||
# The rest of your commands go here
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))))
|
||||
(block
|
||||
(expression_statement
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(string (string_content)))))
|
||||
(expression_statement
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list)))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))))
|
||||
(block
|
||||
(expression_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list)))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(identifier)))
|
||||
(block
|
||||
(expression_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(identifier)
|
||||
(identifier))))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(identifier)))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(identifier))))))))
|
||||
(comment))
|
||||
|
||||
================================================================================
|
||||
Matching multiple values
|
||||
================================================================================
|
||||
|
||||
match command.split():
|
||||
case ["drop", *objects]:
|
||||
for obj in objects:
|
||||
character.drop(obj, current_room)
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(list_splat
|
||||
(identifier))))
|
||||
(block
|
||||
(for_statement
|
||||
(identifier)
|
||||
(identifier)
|
||||
(block
|
||||
(expression_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(identifier)
|
||||
(identifier))))))))))
|
||||
|
||||
================================================================================
|
||||
Adding a wild card
|
||||
================================================================================
|
||||
|
||||
match command.split():
|
||||
# ^ conditional
|
||||
case ["quit"]: ... # Code omitted for brevity
|
||||
case ["go", direction]: pass
|
||||
case ["drop", *objects]: pass
|
||||
case _:
|
||||
print(f"Sorry, I couldn't understand {command!r}")
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list))
|
||||
(comment)
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))))
|
||||
(block
|
||||
(expression_statement
|
||||
(ellipsis))
|
||||
(comment)))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(identifier)))
|
||||
(block
|
||||
(pass_statement)))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(list_splat
|
||||
(identifier))))
|
||||
(block
|
||||
(pass_statement)))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(block
|
||||
(expression_statement
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(string
|
||||
(string_content)
|
||||
(interpolation
|
||||
(identifier)
|
||||
(type_conversion))))))))))
|
||||
|
||||
================================================================================
|
||||
Or patterns
|
||||
================================================================================
|
||||
|
||||
match command.split():
|
||||
case ["north"] | ["go", "north"]:
|
||||
current_room = current_room.neighbor("north")
|
||||
case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]:
|
||||
pass
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(binary_operator
|
||||
(list
|
||||
(string (string_content)))
|
||||
(list
|
||||
(string (string_content))
|
||||
(string (string_content)))))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(string (string_content))))))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(binary_operator
|
||||
(binary_operator
|
||||
(list
|
||||
(string (string_content))
|
||||
(identifier))
|
||||
(list
|
||||
(string (string_content))
|
||||
(string (string_content))
|
||||
(identifier)))
|
||||
(list
|
||||
(string (string_content))
|
||||
(identifier)
|
||||
(string (string_content)))))
|
||||
(block
|
||||
(pass_statement)))))
|
||||
|
||||
================================================================================
|
||||
As patterns
|
||||
================================================================================
|
||||
match command.split():
|
||||
case ["go", ("north" | "south" | "east" | "west") as direction]:
|
||||
current_room = current_room.neighbor(direction)
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(list
|
||||
(string (string_content))
|
||||
(as_pattern
|
||||
(parenthesized_expression
|
||||
(binary_operator
|
||||
(binary_operator
|
||||
(binary_operator
|
||||
(string (string_content))
|
||||
(string (string_content)))
|
||||
(string (string_content)))
|
||||
(string (string_content))))
|
||||
(as_pattern_target
|
||||
(identifier)))))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(identifier)))))))))
|
||||
|
||||
================================================================================
|
||||
Actually not match
|
||||
================================================================================
|
||||
match = 2
|
||||
match, a = 2, 3
|
||||
match: int = secret
|
||||
x, match = 2, "hey, what's up?"
|
||||
*match, last = [1, 2, 3]
|
||||
def foo(**match): pass
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer)))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier))
|
||||
(expression_list
|
||||
(integer)
|
||||
(integer))))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(type
|
||||
(identifier))
|
||||
(identifier)))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(pattern_list
|
||||
(identifier)
|
||||
(identifier))
|
||||
(expression_list
|
||||
(integer)
|
||||
(string (string_content)))))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(pattern_list
|
||||
(list_splat_pattern
|
||||
(identifier))
|
||||
(identifier))
|
||||
(list
|
||||
(integer)
|
||||
(integer)
|
||||
(integer))))
|
||||
(function_definition
|
||||
(identifier)
|
||||
(parameters
|
||||
(dictionary_splat_pattern
|
||||
(identifier)))
|
||||
(block
|
||||
(pass_statement))))
|
||||
|
||||
================================================================================
|
||||
Match is match but not pattern matching
|
||||
================================================================================
|
||||
|
||||
a = [match]
|
||||
match = [match]
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(list
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(list
|
||||
(identifier)))))
|
||||
|
||||
================================================================================
|
||||
Match kwargs
|
||||
================================================================================
|
||||
|
||||
field = call(match=r".*\.txt$")
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(keyword_argument
|
||||
(identifier)
|
||||
(string (string_content))))))))
|
||||
|
||||
================================================================================
|
||||
Match kwargs 2
|
||||
================================================================================
|
||||
|
||||
field = match(match=match, match)
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(keyword_argument
|
||||
(identifier)
|
||||
(identifier))
|
||||
(identifier))))))
|
||||
|
||||
================================================================================
|
||||
Case used as identifier
|
||||
================================================================================
|
||||
|
||||
a = [case]
|
||||
case = [case]
|
||||
just_in_case = call_me(case=True)
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(list
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(list
|
||||
(identifier))))
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list
|
||||
(keyword_argument
|
||||
(identifier)
|
||||
(true)))))))
|
||||
|
||||
================================================================================
|
||||
If guards
|
||||
================================================================================
|
||||
|
||||
match 0:
|
||||
case 0 if False:
|
||||
x = False
|
||||
case 0 if True:
|
||||
x = True
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(integer)
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(integer))
|
||||
(if_clause
|
||||
(false))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(false)))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(integer))
|
||||
(if_clause
|
||||
(true))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(true)))))))
|
||||
|
||||
================================================================================
|
||||
Comma separated cases
|
||||
================================================================================
|
||||
match (0, 1, 2):
|
||||
case 0,1:
|
||||
x = 0
|
||||
case 0, *x:
|
||||
x = 0
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(tuple
|
||||
(integer)
|
||||
(integer)
|
||||
(integer))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(integer))
|
||||
(case_pattern
|
||||
(integer))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer)))))
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(integer))
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer)))))))
|
||||
|
||||
================================================================================
|
||||
case terminating in comma
|
||||
================================================================================
|
||||
match x,:
|
||||
case *x,:
|
||||
y = 0
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(identifier)
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer)))))))
|
||||
|
||||
================================================================================
|
||||
Multiple match patterns
|
||||
================================================================================
|
||||
|
||||
match ..., ...:
|
||||
case a, b:
|
||||
return locals()
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(match_statement
|
||||
(ellipsis)
|
||||
(ellipsis)
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(block
|
||||
(return_statement
|
||||
(call
|
||||
(identifier)
|
||||
(argument_list)))))))
|
||||
|
||||
================================================================================
|
||||
Match match, case case
|
||||
================================================================================
|
||||
match = case = 0
|
||||
match match:
|
||||
case case:
|
||||
x = 0
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer))))
|
||||
(match_statement
|
||||
(identifier)
|
||||
(case_clause
|
||||
(case_pattern
|
||||
(identifier))
|
||||
(block
|
||||
(expression_statement
|
||||
(assignment
|
||||
(identifier)
|
||||
(integer)))))))
|
||||
|
||||
================================================================================
|
||||
Walrus match (Issue #150)
|
||||
================================================================================
|
||||
if match := re.fullmatch(r"(-)?(\d+:)?\d?\d:\d\d(\.\d*)?", time, flags=re.ASCII):
|
||||
return 42
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
(module
|
||||
(if_statement
|
||||
(named_expression
|
||||
(identifier)
|
||||
(call
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))
|
||||
(argument_list
|
||||
(string (string_content))
|
||||
(identifier)
|
||||
(keyword_argument
|
||||
(identifier)
|
||||
(attribute
|
||||
(identifier)
|
||||
(identifier))))))
|
||||
(block
|
||||
(return_statement
|
||||
(integer)))))
|
||||
+1352
File diff suppressed because it is too large
Load Diff
+30
@@ -0,0 +1,30 @@
|
||||
if foo():
|
||||
# <- keyword
|
||||
pass
|
||||
# <- keyword
|
||||
elif bar():
|
||||
# <- keyword
|
||||
pass
|
||||
else:
|
||||
# <- keyword
|
||||
foo
|
||||
|
||||
return
|
||||
# ^ keyword
|
||||
raise e
|
||||
# ^ keyword
|
||||
|
||||
for i in foo():
|
||||
# <- keyword
|
||||
# ^ variable
|
||||
# ^ operator
|
||||
# ^ function
|
||||
continue
|
||||
# <- keyword
|
||||
break
|
||||
# <- keyword
|
||||
|
||||
a and b or c
|
||||
# ^ operator
|
||||
# ^ variable
|
||||
# ^ operator
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
def g(h, i, /, j, *, k=100, **kwarg):
|
||||
# ^ operator
|
||||
# ^ operator
|
||||
pass
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
match command.split():
|
||||
# ^ keyword
|
||||
case ["quit"]:
|
||||
# ^ keyword
|
||||
print("Goodbye!")
|
||||
quit_game()
|
||||
case ["look"]:
|
||||
# ^ keyword
|
||||
current_room.describe()
|
||||
case ["get", obj]:
|
||||
# ^ keyword
|
||||
character.get(obj, current_room)
|
||||
case ["go", direction]:
|
||||
# ^ keyword
|
||||
current_room = current_room.neighbor(direction)
|
||||
# The rest of your commands go here
|
||||
|
||||
match command.split():
|
||||
# ^ keyword
|
||||
case ["drop", *objects]:
|
||||
# ^ keyword
|
||||
for obj in objects:
|
||||
character.drop(obj, current_room)
|
||||
|
||||
match command.split():
|
||||
# ^ keyword
|
||||
case ["quit"]: ... # Code omitted for brevity
|
||||
case ["go", direction]: pass
|
||||
case ["drop", *objects]: pass
|
||||
case _:
|
||||
print(f"Sorry, I couldn't understand {command!r}")
|
||||
|
||||
match command.split():
|
||||
# ^ keyword
|
||||
case ["north"] | ["go", "north"]:
|
||||
# ^ keyword
|
||||
current_room = current_room.neighbor("north")
|
||||
case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]:
|
||||
# ^ keyword
|
||||
pass
|
||||
|
||||
match = 2
|
||||
# ^ variable
|
||||
match, a = 2, 3
|
||||
# ^ variable
|
||||
match: int = secret
|
||||
# ^ variable
|
||||
x, match: str = 2, "hey, what's up?"
|
||||
# <- variable
|
||||
# ^ variable
|
||||
|
||||
if match := re.fullmatch(r"(-)?(\d+:)?\d?\d:\d\d(\.\d*)?", time, flags=re.ASCII):
|
||||
# ^ variable
|
||||
return match
|
||||
@@ -0,0 +1,102 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
import re
|
||||
from io import StringIO
|
||||
import tokenize
|
||||
|
||||
def remove_comments_and_docstrings(source,lang):
|
||||
if lang in ['python']:
|
||||
"""
|
||||
Returns 'source' minus comments and docstrings.
|
||||
"""
|
||||
io_obj = StringIO(source)
|
||||
out = ""
|
||||
prev_toktype = tokenize.INDENT
|
||||
last_lineno = -1
|
||||
last_col = 0
|
||||
for tok in tokenize.generate_tokens(io_obj.readline):
|
||||
token_type = tok[0]
|
||||
token_string = tok[1]
|
||||
start_line, start_col = tok[2]
|
||||
end_line, end_col = tok[3]
|
||||
ltext = tok[4]
|
||||
if start_line > last_lineno:
|
||||
last_col = 0
|
||||
if start_col > last_col:
|
||||
out += (" " * (start_col - last_col))
|
||||
# Remove comments:
|
||||
if token_type == tokenize.COMMENT:
|
||||
pass
|
||||
# This series of conditionals removes docstrings:
|
||||
elif token_type == tokenize.STRING:
|
||||
if prev_toktype != tokenize.INDENT:
|
||||
# This is likely a docstring; double-check we're not inside an operator:
|
||||
if prev_toktype != tokenize.NEWLINE:
|
||||
if start_col > 0:
|
||||
out += token_string
|
||||
else:
|
||||
out += token_string
|
||||
prev_toktype = token_type
|
||||
last_col = end_col
|
||||
last_lineno = end_line
|
||||
temp=[]
|
||||
for x in out.split('\n'):
|
||||
if x.strip()!="":
|
||||
temp.append(x)
|
||||
return '\n'.join(temp)
|
||||
elif lang in ['ruby']:
|
||||
return source
|
||||
else:
|
||||
def replacer(match):
|
||||
s = match.group(0)
|
||||
if s.startswith('/'):
|
||||
return " " # note: a space and not an empty string
|
||||
else:
|
||||
return s
|
||||
pattern = re.compile(
|
||||
r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
|
||||
re.DOTALL | re.MULTILINE
|
||||
)
|
||||
temp=[]
|
||||
for x in re.sub(pattern, replacer, source).split('\n'):
|
||||
if x.strip()!="":
|
||||
temp.append(x)
|
||||
return '\n'.join(temp)
|
||||
|
||||
def tree_to_token_index(root_node):
|
||||
if (len(root_node.children)==0 or root_node.type in ['string_literal','string','character_literal']) and root_node.type!='comment':
|
||||
return [(root_node.start_point,root_node.end_point)]
|
||||
else:
|
||||
code_tokens=[]
|
||||
for child in root_node.children:
|
||||
code_tokens+=tree_to_token_index(child)
|
||||
return code_tokens
|
||||
|
||||
def tree_to_variable_index(root_node,index_to_code):
|
||||
if (len(root_node.children)==0 or root_node.type in ['string_literal','string','character_literal']) and root_node.type!='comment':
|
||||
index=(root_node.start_point,root_node.end_point)
|
||||
_,code=index_to_code[index]
|
||||
if root_node.type!=code:
|
||||
return [(root_node.start_point,root_node.end_point)]
|
||||
else:
|
||||
return []
|
||||
else:
|
||||
code_tokens=[]
|
||||
for child in root_node.children:
|
||||
code_tokens+=tree_to_variable_index(child,index_to_code)
|
||||
return code_tokens
|
||||
|
||||
def index_to_code_token(index,code):
|
||||
start_point=index[0]
|
||||
end_point=index[1]
|
||||
if start_point[0]==end_point[0]:
|
||||
s=code[start_point[0]][start_point[1]:end_point[1]]
|
||||
else:
|
||||
s=""
|
||||
s+=code[start_point[0]][start_point[1]:]
|
||||
for i in range(start_point[0]+1,end_point[0]):
|
||||
s+=code[i]
|
||||
s+=code[end_point[0]][:end_point[1]]
|
||||
return s
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
python calc_code_bleu.py --refs reference_files --hyp candidate_file --language java ( or c_sharp) --params 0.25,0.25,0.25,0.25(default)
|
||||
@@ -0,0 +1,76 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
from codebleu.parser import DFG_python,DFG_java,DFG_ruby,DFG_go,DFG_php,DFG_javascript,DFG_csharp
|
||||
from codebleu.parser import (remove_comments_and_docstrings,
|
||||
tree_to_token_index,
|
||||
index_to_code_token,
|
||||
tree_to_variable_index)
|
||||
from tree_sitter import Language, Parser
|
||||
|
||||
dfg_function={
|
||||
'python':DFG_python,
|
||||
'java':DFG_java,
|
||||
'ruby':DFG_ruby,
|
||||
'go':DFG_go,
|
||||
'php':DFG_php,
|
||||
'javascript':DFG_javascript,
|
||||
'c_sharp':DFG_csharp,
|
||||
}
|
||||
|
||||
def calc_syntax_match(references, candidate, lang):
|
||||
return corpus_syntax_match([references], [candidate], lang)
|
||||
|
||||
def corpus_syntax_check(references, candidates, lang):
|
||||
LANGUAGE = Language('codebleu/parser/my-languages.so', lang)
|
||||
parser = Parser()
|
||||
parser.set_language(LANGUAGE)
|
||||
match_count = 0
|
||||
total_count = 0
|
||||
scores = []
|
||||
|
||||
for i in range(len(candidates)):
|
||||
references_sample = references[i]
|
||||
candidate = candidates[i]
|
||||
for reference in references_sample:
|
||||
try:
|
||||
candidate=remove_comments_and_docstrings(candidate,'java')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
reference=remove_comments_and_docstrings(reference,'java')
|
||||
except:
|
||||
pass
|
||||
|
||||
candidate_tree = parser.parse(bytes(candidate,'utf8')).root_node
|
||||
|
||||
reference_tree = parser.parse(bytes(reference,'utf8')).root_node
|
||||
|
||||
def get_all_sub_trees(root_node):
|
||||
node_stack = []
|
||||
sub_tree_sexp_list = []
|
||||
depth = 1
|
||||
node_stack.append([root_node, depth])
|
||||
while len(node_stack) != 0:
|
||||
cur_node, cur_depth = node_stack.pop()
|
||||
sub_tree_sexp_list.append([cur_node.sexp(), cur_depth])
|
||||
for child_node in cur_node.children:
|
||||
if len(child_node.children) != 0:
|
||||
depth = cur_depth + 1
|
||||
node_stack.append([child_node, depth])
|
||||
return sub_tree_sexp_list
|
||||
cand_sexps = [x[0] for x in get_all_sub_trees(candidate_tree)]
|
||||
# print(len(cand_sexps))
|
||||
# for i, sub_tree in enumerate(cand_sexps):
|
||||
# print(i, sub_tree)
|
||||
# exit()
|
||||
ref_sexps = get_all_sub_trees(reference_tree)
|
||||
|
||||
score = 0 - str(candidate_tree.sexp()).count("ERROR")
|
||||
# for sub_tree, depth in ref_sexps:
|
||||
# if sub_tree in cand_sexps:
|
||||
# match_count += 1
|
||||
# total_count += len(ref_sexps)
|
||||
|
||||
# score = match_count / total_count
|
||||
return score
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
from codebleu.parser import DFG_python,DFG_java,DFG_ruby,DFG_go,DFG_php,DFG_javascript,DFG_csharp
|
||||
from codebleu.parser import (remove_comments_and_docstrings,
|
||||
tree_to_token_index,
|
||||
index_to_code_token,
|
||||
tree_to_variable_index)
|
||||
from tree_sitter import Language, Parser
|
||||
|
||||
dfg_function={
|
||||
'python':DFG_python,
|
||||
'java':DFG_java,
|
||||
'ruby':DFG_ruby,
|
||||
'go':DFG_go,
|
||||
'php':DFG_php,
|
||||
'javascript':DFG_javascript,
|
||||
'c_sharp':DFG_csharp,
|
||||
}
|
||||
|
||||
def calc_syntax_match(references, candidate, lang):
|
||||
"""
|
||||
Calculate the syntax match score for a candidate code against references.
|
||||
|
||||
Args:
|
||||
references (list): A list of reference code samples.
|
||||
candidate (str): The candidate code to be evaluated.
|
||||
lang (str): The programming language of the code samples.
|
||||
|
||||
Returns:
|
||||
float: The syntax match score.
|
||||
"""
|
||||
return corpus_syntax_match([references], [candidate], lang)
|
||||
|
||||
def corpus_syntax_match(references, candidates, lang):
|
||||
"""
|
||||
Calculate the corpus-level syntax match score for candidates against references.
|
||||
|
||||
Args:
|
||||
references (list): A list of lists of reference code samples.
|
||||
candidates (list): A list of candidate code samples.
|
||||
lang (str): The programming language of the code samples.
|
||||
|
||||
Returns:
|
||||
float: The corpus-level syntax match score.
|
||||
"""
|
||||
'''
|
||||
Language.build_library(# Store the library in the `build` directory
|
||||
'build/my-languages.so',
|
||||
'vendor/tree-sitter-python')
|
||||
'''
|
||||
# LANGUAGE = Language('codebleu/parser/my-languages.so', lang)
|
||||
LANGUAGE = Language('./build/my-languages.so', lang)
|
||||
parser = Parser()
|
||||
parser.set_language(LANGUAGE)
|
||||
match_count = 0
|
||||
total_count = 0
|
||||
|
||||
for i in range(len(candidates)):
|
||||
references_sample = references[i]
|
||||
candidate = candidates[i]
|
||||
for reference in references_sample:
|
||||
try:
|
||||
candidate=remove_comments_and_docstrings(candidate,'java')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
reference=remove_comments_and_docstrings(reference,'java')
|
||||
except:
|
||||
pass
|
||||
|
||||
candidate_tree = parser.parse(bytes(candidate,'utf8')).root_node
|
||||
|
||||
reference_tree = parser.parse(bytes(reference,'utf8')).root_node
|
||||
'''
|
||||
print(candidate)
|
||||
print(reference_tree.sexp())
|
||||
print(candidate_tree.sexp())
|
||||
print(reference_tree.sexp() == candidate_tree.sexp())
|
||||
exit()
|
||||
'''
|
||||
|
||||
'''
|
||||
if candidate_tree.sexp() == reference_tree.sexp():
|
||||
total_match = 1/0.3
|
||||
else:
|
||||
total_match = -1/0.3
|
||||
'''
|
||||
total_match = 0
|
||||
|
||||
def get_all_sub_trees(root_node):
|
||||
"""
|
||||
Get all sub-trees of a given root node.
|
||||
|
||||
Args:
|
||||
root_node: The root node of the syntax tree.
|
||||
|
||||
Returns:
|
||||
list: List of sub-tree sexps and their depths.
|
||||
"""
|
||||
node_stack = []
|
||||
sub_tree_sexp_list = []
|
||||
depth = 1
|
||||
node_stack.append([root_node, depth])
|
||||
while len(node_stack) != 0:
|
||||
cur_node, cur_depth = node_stack.pop()
|
||||
sub_tree_sexp_list.append([cur_node.sexp(), cur_depth])
|
||||
for child_node in cur_node.children:
|
||||
if len(child_node.children) != 0:
|
||||
depth = cur_depth + 1
|
||||
node_stack.append([child_node, depth])
|
||||
return sub_tree_sexp_list
|
||||
cand_sexps = [x[0] for x in get_all_sub_trees(candidate_tree)]
|
||||
ref_sexps = get_all_sub_trees(reference_tree)
|
||||
|
||||
# print(cand_sexps)
|
||||
# print(ref_sexps)
|
||||
|
||||
for sub_tree, depth in ref_sexps:
|
||||
if sub_tree in cand_sexps:
|
||||
match_count += 1
|
||||
total_count += len(ref_sexps)
|
||||
|
||||
if total_count == 0:
|
||||
return 0 + total_match
|
||||
|
||||
score = match_count / total_count + total_match
|
||||
return score
|
||||
@@ -0,0 +1,110 @@
|
||||
# Natural Language Toolkit: Utility functions
|
||||
#
|
||||
# Copyright (C) 2001-2020 NLTK Project
|
||||
# Author: Steven Bird <stevenbird1@gmail.com>
|
||||
# URL: <http://nltk.org/>
|
||||
# For license information, see LICENSE.TXT
|
||||
|
||||
from itertools import chain
|
||||
|
||||
def pad_sequence(
|
||||
sequence,
|
||||
n,
|
||||
pad_left=False,
|
||||
pad_right=False,
|
||||
left_pad_symbol=None,
|
||||
right_pad_symbol=None,
|
||||
):
|
||||
"""
|
||||
Returns a padded sequence of items before ngram extraction.
|
||||
|
||||
Args:
|
||||
sequence (sequence or iter): The source data to be padded.
|
||||
n (int): The degree of the ngrams.
|
||||
pad_left (bool): Whether the ngrams should be left-padded.
|
||||
pad_right (bool): Whether the ngrams should be right-padded.
|
||||
left_pad_symbol (any): The symbol to use for left padding (default is None).
|
||||
right_pad_symbol (any): The symbol to use for right padding (default is None).
|
||||
|
||||
Yields:
|
||||
sequence or iter: Padded sequence of items.
|
||||
|
||||
Examples:
|
||||
>>> list(pad_sequence([1, 2, 3, 4, 5], 2, pad_left=True, pad_right=True, left_pad_symbol='<s>', right_pad_symbol='</s>'))
|
||||
['<s>', 1, 2, 3, 4, 5, '</s>']
|
||||
>>> list(pad_sequence([1, 2, 3, 4, 5], 2, pad_left=True, left_pad_symbol='<s>'))
|
||||
['<s>', 1, 2, 3, 4, 5]
|
||||
>>> list(pad_sequence([1, 2, 3, 4, 5], 2, pad_right=True, right_pad_symbol='</s>'))
|
||||
[1, 2, 3, 4, 5, '</s>']
|
||||
|
||||
Returns:
|
||||
sequence or iter: Padded sequence of items.
|
||||
"""
|
||||
sequence = iter(sequence)
|
||||
if pad_left:
|
||||
sequence = chain((left_pad_symbol,) * (n - 1), sequence)
|
||||
if pad_right:
|
||||
sequence = chain(sequence, (right_pad_symbol,) * (n - 1))
|
||||
return sequence
|
||||
|
||||
|
||||
# add a flag to pad the sequence so we get peripheral ngrams?
|
||||
|
||||
|
||||
def ngrams(
|
||||
sequence,
|
||||
n,
|
||||
pad_left=False,
|
||||
pad_right=False,
|
||||
left_pad_symbol=None,
|
||||
right_pad_symbol=None,
|
||||
):
|
||||
"""
|
||||
Generate ngrams from a sequence of items.
|
||||
|
||||
Args:
|
||||
sequence (sequence or iter): The source data to be converted into ngrams.
|
||||
n (int): The degree of the ngrams.
|
||||
pad_left (bool, optional): Whether the ngrams should be left-padded.
|
||||
pad_right (bool, optional): Whether the ngrams should be right-padded.
|
||||
left_pad_symbol (any, optional): The symbol to use for left padding (default is None).
|
||||
right_pad_symbol (any, optional): The symbol to use for right padding (default is None).
|
||||
|
||||
Yields:
|
||||
sequence or iter: Ngrams generated from the sequence.
|
||||
|
||||
Examples:
|
||||
>>> from nltk.util import ngrams
|
||||
>>> list(ngrams([1, 2, 3, 4, 5], 3))
|
||||
[(1, 2, 3), (2, 3, 4), (3, 4, 5)]
|
||||
|
||||
>>> list(ngrams([1, 2, 3, 4, 5], 2, pad_right=True))
|
||||
[(1, 2), (2, 3), (3, 4), (4, 5), (5, None)]
|
||||
|
||||
>>> list(ngrams([1, 2, 3, 4, 5], 2, pad_right=True, right_pad_symbol='</s>'))
|
||||
[(1, 2), (2, 3), (3, 4), (4, 5), (5, '</s>')]
|
||||
|
||||
>>> list(ngrams([1, 2, 3, 4, 5], 2, pad_left=True, left_pad_symbol='<s>'))
|
||||
[('<s>', 1), (1, 2), (2, 3), (3, 4), (4, 5)]
|
||||
|
||||
>>> list(ngrams([1, 2, 3, 4, 5], 2, pad_left=True, pad_right=True, left_pad_symbol='<s>', right_pad_symbol='</s>'))
|
||||
[('<s>', 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, '</s>')]
|
||||
"""
|
||||
sequence = pad_sequence(
|
||||
sequence, n, pad_left, pad_right, left_pad_symbol, right_pad_symbol
|
||||
)
|
||||
|
||||
history = []
|
||||
while n > 1:
|
||||
# PEP 479, prevent RuntimeError from being raised when StopIteration bubbles out of generator
|
||||
try:
|
||||
next_item = next(sequence)
|
||||
except StopIteration:
|
||||
# no more data, terminate the generator
|
||||
return
|
||||
history.append(next_item)
|
||||
n -= 1
|
||||
for item in sequence:
|
||||
history.append(item)
|
||||
yield tuple(history)
|
||||
del history[0]
|
||||
@@ -0,0 +1,558 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
# Natural Language Toolkit: BLEU Score
|
||||
#
|
||||
# Copyright (C) 2001-2020 NLTK Project
|
||||
# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
|
||||
# Contributors: Björn Mattsson, Dmitrijs Milajevs, Liling Tan
|
||||
# URL: <http://nltk.org/>
|
||||
# For license information, see LICENSE.TXT
|
||||
|
||||
"""BLEU score implementation."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
from fractions import Fraction
|
||||
import warnings
|
||||
from collections import Counter
|
||||
|
||||
from codebleu.utils import ngrams
|
||||
import pdb
|
||||
|
||||
|
||||
def sentence_bleu(
|
||||
references,
|
||||
hypothesis,
|
||||
weights=(0.25, 0.25, 0.25, 0.25),
|
||||
smoothing_function=None,
|
||||
auto_reweigh=False,
|
||||
):
|
||||
"""
|
||||
Calculate BLEU score (Bilingual Evaluation Understudy) from
|
||||
Papineni, Kishore, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002.
|
||||
"BLEU: a method for automatic evaluation of machine translation."
|
||||
In Proceedings of ACL. http://www.aclweb.org/anthology/P02-1040.pdf
|
||||
>>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
|
||||
... 'ensures', 'that', 'the', 'military', 'always',
|
||||
... 'obeys', 'the', 'commands', 'of', 'the', 'party']
|
||||
>>> hypothesis2 = ['It', 'is', 'to', 'insure', 'the', 'troops',
|
||||
... 'forever', 'hearing', 'the', 'activity', 'guidebook',
|
||||
... 'that', 'party', 'direct']
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
|
||||
... 'ensures', 'that', 'the', 'military', 'will', 'forever',
|
||||
... 'heed', 'Party', 'commands']
|
||||
>>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',
|
||||
... 'guarantees', 'the', 'military', 'forces', 'always',
|
||||
... 'being', 'under', 'the', 'command', 'of', 'the',
|
||||
... 'Party']
|
||||
>>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
|
||||
... 'army', 'always', 'to', 'heed', 'the', 'directions',
|
||||
... 'of', 'the', 'party']
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis1) # doctest: +ELLIPSIS
|
||||
0.5045...
|
||||
If there is no ngrams overlap for any order of n-grams, BLEU returns the
|
||||
value 0. This is because the precision for the order of n-grams without
|
||||
overlap is 0, and the geometric mean in the final BLEU score computation
|
||||
multiplies the 0 with the precision of other n-grams. This results in 0
|
||||
(independently of the precision of the othe n-gram orders). The following
|
||||
example has zero 3-gram and 4-gram overlaps:
|
||||
>>> round(sentence_bleu([reference1, reference2, reference3], hypothesis2),4) # doctest: +ELLIPSIS
|
||||
0.0
|
||||
To avoid this harsh behaviour when no ngram overlaps are found a smoothing
|
||||
function can be used.
|
||||
>>> chencherry = SmoothingFunction()
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis2,
|
||||
... smoothing_function=chencherry.method1) # doctest: +ELLIPSIS
|
||||
0.0370...
|
||||
The default BLEU calculates a score for up to 4-grams using uniform
|
||||
weights (this is called BLEU-4). To evaluate your translations with
|
||||
higher/lower order ngrams, use customized weights. E.g. when accounting
|
||||
for up to 5-grams with uniform weights (this is called BLEU-5) use:
|
||||
>>> weights = (1./5., 1./5., 1./5., 1./5., 1./5.)
|
||||
>>> sentence_bleu([reference1, reference2, reference3], hypothesis1, weights) # doctest: +ELLIPSIS
|
||||
0.3920...
|
||||
:param references: reference sentences
|
||||
:type references: list(list(str))
|
||||
:param hypothesis: a hypothesis sentence
|
||||
:type hypothesis: list(str)
|
||||
:param weights: weights for unigrams, bigrams, trigrams and so on
|
||||
:type weights: list(float)
|
||||
:param smoothing_function:
|
||||
:type smoothing_function: SmoothingFunction
|
||||
:param auto_reweigh: Option to re-normalize the weights uniformly.
|
||||
:type auto_reweigh: bool
|
||||
:return: The sentence-level BLEU score.
|
||||
:rtype: float
|
||||
"""
|
||||
return corpus_bleu(
|
||||
[references], [hypothesis], weights, smoothing_function, auto_reweigh
|
||||
)
|
||||
|
||||
|
||||
def corpus_bleu(
|
||||
list_of_references,
|
||||
hypotheses,
|
||||
weights=(0.25, 0.25, 0.25, 0.25),
|
||||
smoothing_function=None,
|
||||
auto_reweigh=False,
|
||||
):
|
||||
"""
|
||||
Calculate a single corpus-level BLEU score (aka. system-level BLEU) for all
|
||||
the hypotheses and their respective references.
|
||||
Instead of averaging the sentence level BLEU scores (i.e. marco-average
|
||||
precision), the original BLEU metric (Papineni et al. 2002) accounts for
|
||||
the micro-average precision (i.e. summing the numerators and denominators
|
||||
for each hypothesis-reference(s) pairs before the division).
|
||||
>>> hyp1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
|
||||
... 'ensures', 'that', 'the', 'military', 'always',
|
||||
... 'obeys', 'the', 'commands', 'of', 'the', 'party']
|
||||
>>> ref1a = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
|
||||
... 'ensures', 'that', 'the', 'military', 'will', 'forever',
|
||||
... 'heed', 'Party', 'commands']
|
||||
>>> ref1b = ['It', 'is', 'the', 'guiding', 'principle', 'which',
|
||||
... 'guarantees', 'the', 'military', 'forces', 'always',
|
||||
... 'being', 'under', 'the', 'command', 'of', 'the', 'Party']
|
||||
>>> ref1c = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
|
||||
... 'army', 'always', 'to', 'heed', 'the', 'directions',
|
||||
... 'of', 'the', 'party']
|
||||
>>> hyp2 = ['he', 'read', 'the', 'book', 'because', 'he', 'was',
|
||||
... 'interested', 'in', 'world', 'history']
|
||||
>>> ref2a = ['he', 'was', 'interested', 'in', 'world', 'history',
|
||||
... 'because', 'he', 'read', 'the', 'book']
|
||||
>>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
|
||||
>>> hypotheses = [hyp1, hyp2]
|
||||
>>> corpus_bleu(list_of_references, hypotheses) # doctest: +ELLIPSIS
|
||||
0.5920...
|
||||
The example below show that corpus_bleu() is different from averaging
|
||||
sentence_bleu() for hypotheses
|
||||
>>> score1 = sentence_bleu([ref1a, ref1b, ref1c], hyp1)
|
||||
>>> score2 = sentence_bleu([ref2a], hyp2)
|
||||
>>> (score1 + score2) / 2 # doctest: +ELLIPSIS
|
||||
0.6223...
|
||||
:param list_of_references: a corpus of lists of reference sentences, w.r.t. hypotheses
|
||||
:type list_of_references: list(list(list(str)))
|
||||
:param hypotheses: a list of hypothesis sentences
|
||||
:type hypotheses: list(list(str))
|
||||
:param weights: weights for unigrams, bigrams, trigrams and so on
|
||||
:type weights: list(float)
|
||||
:param smoothing_function:
|
||||
:type smoothing_function: SmoothingFunction
|
||||
:param auto_reweigh: Option to re-normalize the weights uniformly.
|
||||
:type auto_reweigh: bool
|
||||
:return: The corpus-level BLEU score.
|
||||
:rtype: float
|
||||
"""
|
||||
# Before proceeding to compute BLEU, perform sanity checks.
|
||||
|
||||
p_numerators = Counter() # Key = ngram order, and value = no. of ngram matches.
|
||||
p_denominators = Counter() # Key = ngram order, and value = no. of ngram in ref.
|
||||
hyp_lengths, ref_lengths = 0, 0
|
||||
|
||||
assert len(list_of_references) == len(hypotheses), (
|
||||
"The number of hypotheses and their reference(s) should be the " "same "
|
||||
)
|
||||
|
||||
# Iterate through each hypothesis and their corresponding references.
|
||||
for references, hypothesis in zip(list_of_references, hypotheses):
|
||||
# For each order of ngram, calculate the numerator and
|
||||
# denominator for the corpus-level modified precision.
|
||||
for i, _ in enumerate(weights, start=1):
|
||||
p_i_numeraotr, p_i_denominator = modified_recall(references, hypothesis, i)
|
||||
p_numerators[i] += p_i_numeraotr
|
||||
p_denominators[i] += p_i_denominator
|
||||
|
||||
# Calculate the hypothesis length and the closest reference length.
|
||||
# Adds them to the corpus-level hypothesis and reference counts.
|
||||
hyp_len = len(hypothesis)
|
||||
hyp_lengths += hyp_len
|
||||
ref_lengths += closest_ref_length(references, hyp_len)
|
||||
|
||||
# Calculate corpus-level brevity penalty.
|
||||
bp = brevity_penalty(ref_lengths, hyp_lengths)
|
||||
|
||||
# Uniformly re-weighting based on maximum hypothesis lengths if largest
|
||||
# order of n-grams < 4 and weights is set at default.
|
||||
if auto_reweigh:
|
||||
if hyp_lengths < 4 and weights == (0.25, 0.25, 0.25, 0.25):
|
||||
weights = (1 / hyp_lengths,) * hyp_lengths
|
||||
|
||||
# Collects the various recall values for the different ngram orders.
|
||||
p_n = [
|
||||
(p_numerators[i], p_denominators[i])
|
||||
for i, _ in enumerate(weights, start=1)
|
||||
]
|
||||
|
||||
# Returns 0 if there's no matching n-grams
|
||||
# We only need to check for p_numerators[1] == 0, since if there's
|
||||
# no unigrams, there won't be any higher order ngrams.
|
||||
if p_numerators[1] == 0:
|
||||
return 0
|
||||
|
||||
# If there's no smoothing, set use method0 from SmoothinFunction class.
|
||||
if not smoothing_function:
|
||||
smoothing_function = SmoothingFunction().method1
|
||||
# Smoothen the modified precision.
|
||||
# Note: smoothing_function() may convert values into floats;
|
||||
# it tries to retain the Fraction object as much as the
|
||||
# smoothing method allows.
|
||||
p_n = smoothing_function(
|
||||
p_n, references=references, hypothesis=hypothesis, hyp_len=hyp_lengths
|
||||
)
|
||||
# pdb.set_trace()
|
||||
s = (w_i * math.log(p_i[0]/p_i[1]) for w_i, p_i in zip(weights, p_n))
|
||||
s = bp * math.exp(math.fsum(s))
|
||||
return s
|
||||
|
||||
|
||||
def modified_recall(references, hypothesis, n):
|
||||
"""
|
||||
Calculate modified ngram recall.
|
||||
:param references: A list of reference translations.
|
||||
:type references: list(list(str))
|
||||
:param hypothesis: A hypothesis translation.
|
||||
:type hypothesis: list(str)
|
||||
:param n: The ngram order.
|
||||
:type n: int
|
||||
:return: BLEU's modified precision for the nth order ngram.
|
||||
:rtype: Fraction
|
||||
"""
|
||||
# Extracts all ngrams in hypothesis
|
||||
# Set an empty Counter if hypothesis is empty.
|
||||
# pdb.set_trace()
|
||||
numerator = 0
|
||||
denominator = 0
|
||||
|
||||
counts = Counter(ngrams(hypothesis, n)) if len(hypothesis) >= n else Counter()
|
||||
# Extract a union of references' counts.
|
||||
# max_counts = reduce(or_, [Counter(ngrams(ref, n)) for ref in references])
|
||||
max_counts = {}
|
||||
for reference_and_weights in references:
|
||||
reference = reference_and_weights[0]
|
||||
weights = reference_and_weights[1]
|
||||
reference_counts = (
|
||||
Counter(ngrams(reference, n)) if len(reference) >= n else Counter()
|
||||
)
|
||||
# for ngram in reference_counts:
|
||||
# max_counts[ngram] = max(max_counts.get(ngram, 0), counts[ngram])
|
||||
clipped_counts = {
|
||||
ngram: min(count, counts[ngram]) for ngram, count in reference_counts.items()
|
||||
}
|
||||
# reweight
|
||||
if n == 1 and len(weights) == len(reference_counts):
|
||||
def weighted_sum(weights, counts):
|
||||
sum_counts = 0
|
||||
for ngram, count in counts.items():
|
||||
sum_counts += count * (weights[ngram[0]] if ngram[0] in weights else 1)
|
||||
return sum_counts
|
||||
|
||||
numerator += weighted_sum(weights, clipped_counts)
|
||||
denominator += max(1, weighted_sum(weights, reference_counts))
|
||||
|
||||
else:
|
||||
numerator += sum(clipped_counts.values())
|
||||
denominator += max(1, sum(reference_counts.values()))
|
||||
|
||||
# # Assigns the intersection between hypothesis and references' counts.
|
||||
# clipped_counts = {
|
||||
# ngram: min(count, max_counts[ngram]) for ngram, count in counts.items()
|
||||
# }
|
||||
|
||||
# numerator += sum(clipped_counts.values())
|
||||
# # Ensures that denominator is minimum 1 to avoid ZeroDivisionError.
|
||||
# # Usually this happens when the ngram order is > len(reference).
|
||||
# denominator += max(1, sum(counts.values()))
|
||||
|
||||
#return Fraction(numerator, denominator, _normalize=False)
|
||||
return numerator, denominator
|
||||
|
||||
|
||||
def closest_ref_length(references, hyp_len):
|
||||
"""
|
||||
This function finds the reference that is the closest length to the
|
||||
hypothesis. The closest reference length is referred to as *r* variable
|
||||
from the brevity penalty formula in Papineni et. al. (2002)
|
||||
:param references: A list of reference translations.
|
||||
:type references: list(list(str))
|
||||
:param hyp_len: The length of the hypothesis.
|
||||
:type hyp_len: int
|
||||
:return: The length of the reference that's closest to the hypothesis.
|
||||
:rtype: int
|
||||
"""
|
||||
ref_lens = (len(reference) for reference in references)
|
||||
closest_ref_len = min(
|
||||
ref_lens, key=lambda ref_len: (abs(ref_len - hyp_len), ref_len)
|
||||
)
|
||||
return closest_ref_len
|
||||
|
||||
|
||||
def brevity_penalty(closest_ref_len, hyp_len):
|
||||
"""
|
||||
Calculate brevity penalty.
|
||||
As the modified n-gram precision still has the problem from the short
|
||||
length sentence, brevity penalty is used to modify the overall BLEU
|
||||
score according to length.
|
||||
An example from the paper. There are three references with length 12, 15
|
||||
and 17. And a concise hypothesis of the length 12. The brevity penalty is 1.
|
||||
>>> reference1 = list('aaaaaaaaaaaa') # i.e. ['a'] * 12
|
||||
>>> reference2 = list('aaaaaaaaaaaaaaa') # i.e. ['a'] * 15
|
||||
>>> reference3 = list('aaaaaaaaaaaaaaaaa') # i.e. ['a'] * 17
|
||||
>>> hypothesis = list('aaaaaaaaaaaa') # i.e. ['a'] * 12
|
||||
>>> references = [reference1, reference2, reference3]
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(closest_ref_len, hyp_len)
|
||||
1.0
|
||||
In case a hypothesis translation is shorter than the references, penalty is
|
||||
applied.
|
||||
>>> references = [['a'] * 28, ['a'] * 28]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(closest_ref_len, hyp_len)
|
||||
0.2635971381157267
|
||||
The length of the closest reference is used to compute the penalty. If the
|
||||
length of a hypothesis is 12, and the reference lengths are 13 and 2, the
|
||||
penalty is applied because the hypothesis length (12) is less then the
|
||||
closest reference length (13).
|
||||
>>> references = [['a'] * 13, ['a'] * 2]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS
|
||||
0.9200...
|
||||
The brevity penalty doesn't depend on reference order. More importantly,
|
||||
when two reference sentences are at the same distance, the shortest
|
||||
reference sentence length is used.
|
||||
>>> references = [['a'] * 13, ['a'] * 11]
|
||||
>>> hypothesis = ['a'] * 12
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> bp1 = brevity_penalty(closest_ref_len, hyp_len)
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(reversed(references), hyp_len)
|
||||
>>> bp2 = brevity_penalty(closest_ref_len, hyp_len)
|
||||
>>> bp1 == bp2 == 1
|
||||
True
|
||||
A test example from mteval-v13a.pl (starting from the line 705):
|
||||
>>> references = [['a'] * 11, ['a'] * 8]
|
||||
>>> hypothesis = ['a'] * 7
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(closest_ref_len, hyp_len) # doctest: +ELLIPSIS
|
||||
0.8668...
|
||||
>>> references = [['a'] * 11, ['a'] * 8, ['a'] * 6, ['a'] * 7]
|
||||
>>> hypothesis = ['a'] * 7
|
||||
>>> hyp_len = len(hypothesis)
|
||||
>>> closest_ref_len = closest_ref_length(references, hyp_len)
|
||||
>>> brevity_penalty(closest_ref_len, hyp_len)
|
||||
1.0
|
||||
:param hyp_len: The length of the hypothesis for a single sentence OR the
|
||||
sum of all the hypotheses' lengths for a corpus
|
||||
:type hyp_len: int
|
||||
:param closest_ref_len: The length of the closest reference for a single
|
||||
hypothesis OR the sum of all the closest references for every hypotheses.
|
||||
:type closest_ref_len: int
|
||||
:return: BLEU's brevity penalty.
|
||||
:rtype: float
|
||||
"""
|
||||
if hyp_len > closest_ref_len:
|
||||
return 1
|
||||
# If hypothesis is empty, brevity penalty = 0 should result in BLEU = 0.0
|
||||
elif hyp_len == 0:
|
||||
return 0
|
||||
else:
|
||||
return math.exp(1 - closest_ref_len / hyp_len)
|
||||
|
||||
|
||||
class SmoothingFunction:
|
||||
"""
|
||||
This is an implementation of the smoothing techniques
|
||||
for segment-level BLEU scores that was presented in
|
||||
Boxing Chen and Collin Cherry (2014) A Systematic Comparison of
|
||||
Smoothing Techniques for Sentence-Level BLEU. In WMT14.
|
||||
http://acl2014.org/acl2014/W14-33/pdf/W14-3346.pdf
|
||||
"""
|
||||
|
||||
def __init__(self, epsilon=0.1, alpha=5, k=5):
|
||||
"""
|
||||
This will initialize the parameters required for the various smoothing
|
||||
techniques, the default values are set to the numbers used in the
|
||||
experiments from Chen and Cherry (2014).
|
||||
>>> hypothesis1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which', 'ensures',
|
||||
... 'that', 'the', 'military', 'always', 'obeys', 'the',
|
||||
... 'commands', 'of', 'the', 'party']
|
||||
>>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that', 'ensures',
|
||||
... 'that', 'the', 'military', 'will', 'forever', 'heed',
|
||||
... 'Party', 'commands']
|
||||
>>> chencherry = SmoothingFunction()
|
||||
>>> print(sentence_bleu([reference1], hypothesis1)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method0)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method1)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method2)) # doctest: +ELLIPSIS
|
||||
0.4489...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method3)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method4)) # doctest: +ELLIPSIS
|
||||
0.4118...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method5)) # doctest: +ELLIPSIS
|
||||
0.4905...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method6)) # doctest: +ELLIPSIS
|
||||
0.4135...
|
||||
>>> print(sentence_bleu([reference1], hypothesis1, smoothing_function=chencherry.method7)) # doctest: +ELLIPSIS
|
||||
0.4905...
|
||||
:param epsilon: the epsilon value use in method 1
|
||||
:type epsilon: float
|
||||
:param alpha: the alpha value use in method 6
|
||||
:type alpha: int
|
||||
:param k: the k value use in method 4
|
||||
:type k: int
|
||||
"""
|
||||
self.epsilon = epsilon
|
||||
self.alpha = alpha
|
||||
self.k = k
|
||||
|
||||
def method0(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
No smoothing.
|
||||
"""
|
||||
p_n_new = []
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i[0] != 0:
|
||||
p_n_new.append(p_i)
|
||||
else:
|
||||
_msg = str(
|
||||
"\nThe hypothesis contains 0 counts of {}-gram overlaps.\n"
|
||||
"Therefore the BLEU score evaluates to 0, independently of\n"
|
||||
"how many N-gram overlaps of lower order it contains.\n"
|
||||
"Consider using lower n-gram order or use "
|
||||
"SmoothingFunction()"
|
||||
).format(i + 1)
|
||||
warnings.warn(_msg)
|
||||
# When numerator==0 where denonminator==0 or !=0, the result
|
||||
# for the precision score should be equal to 0 or undefined.
|
||||
# Due to BLEU geometric mean computation in logarithm space,
|
||||
# we we need to take the return sys.float_info.min such that
|
||||
# math.log(sys.float_info.min) returns a 0 precision score.
|
||||
p_n_new.append(sys.float_info.min)
|
||||
return p_n_new
|
||||
|
||||
def method1(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 1: Add *epsilon* counts to precision with 0 counts.
|
||||
"""
|
||||
return [
|
||||
((p_i[0] + self.epsilon), p_i[1])
|
||||
if p_i[0] == 0
|
||||
else p_i
|
||||
for p_i in p_n
|
||||
]
|
||||
|
||||
def method2(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 2: Add 1 to both numerator and denominator from
|
||||
Chin-Yew Lin and Franz Josef Och (2004) Automatic evaluation of
|
||||
machine translation quality using longest common subsequence and
|
||||
skip-bigram statistics. In ACL04.
|
||||
"""
|
||||
return [
|
||||
(p_i[0] + 1, p_i[1] + 1)
|
||||
for p_i in p_n
|
||||
]
|
||||
|
||||
def method3(self, p_n, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 3: NIST geometric sequence smoothing
|
||||
The smoothing is computed by taking 1 / ( 2^k ), instead of 0, for each
|
||||
precision score whose matching n-gram count is null.
|
||||
k is 1 for the first 'n' value for which the n-gram match count is null/
|
||||
For example, if the text contains:
|
||||
- one 2-gram match
|
||||
- and (consequently) two 1-gram matches
|
||||
the n-gram count for each individual precision score would be:
|
||||
- n=1 => prec_count = 2 (two unigrams)
|
||||
- n=2 => prec_count = 1 (one bigram)
|
||||
- n=3 => prec_count = 1/2 (no trigram, taking 'smoothed' value of 1 / ( 2^k ), with k=1)
|
||||
- n=4 => prec_count = 1/4 (no fourgram, taking 'smoothed' value of 1 / ( 2^k ), with k=2)
|
||||
"""
|
||||
incvnt = 1 # From the mteval-v13a.pl, it's referred to as k.
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i.numerator == 0:
|
||||
p_n[i] = 1 / (2 ** incvnt * p_i.denominator)
|
||||
incvnt += 1
|
||||
return p_n
|
||||
|
||||
def method4(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 4:
|
||||
Shorter translations may have inflated precision values due to having
|
||||
smaller denominators; therefore, we give them proportionally
|
||||
smaller smoothed counts. Instead of scaling to 1/(2^k), Chen and Cherry
|
||||
suggests dividing by 1/ln(len(T)), where T is the length of the translation.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
for i, p_i in enumerate(p_n):
|
||||
if p_i.numerator == 0 and hyp_len != 0:
|
||||
incvnt = i + 1 * self.k / math.log(
|
||||
hyp_len
|
||||
) # Note that this K is different from the K from NIST.
|
||||
p_n[i] = incvnt / p_i.denominator
|
||||
return p_n
|
||||
|
||||
def method5(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 5:
|
||||
The matched counts for similar values of n should be similar. To a
|
||||
calculate the n-gram matched count, it averages the n−1, n and n+1 gram
|
||||
matched counts.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
m = {}
|
||||
# Requires an precision value for an addition ngram order.
|
||||
p_n_plus1 = p_n + [modified_precision(references, hypothesis, 5)]
|
||||
m[-1] = p_n[0] + 1
|
||||
for i, p_i in enumerate(p_n):
|
||||
p_n[i] = (m[i - 1] + p_i + p_n_plus1[i + 1]) / 3
|
||||
m[i] = p_n[i]
|
||||
return p_n
|
||||
|
||||
def method6(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 6:
|
||||
Interpolates the maximum likelihood estimate of the precision *p_n* with
|
||||
a prior estimate *pi0*. The prior is estimated by assuming that the ratio
|
||||
between pn and pn−1 will be the same as that between pn−1 and pn−2; from
|
||||
Gao and He (2013) Training MRF-Based Phrase Translation Models using
|
||||
Gradient Ascent. In NAACL.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
# This smoothing only works when p_1 and p_2 is non-zero.
|
||||
# Raise an error with an appropriate message when the input is too short
|
||||
# to use this smoothing technique.
|
||||
assert p_n[2], "This smoothing method requires non-zero precision for bigrams."
|
||||
for i, p_i in enumerate(p_n):
|
||||
if i in [0, 1]: # Skips the first 2 orders of ngrams.
|
||||
continue
|
||||
else:
|
||||
pi0 = 0 if p_n[i - 2] == 0 else p_n[i - 1] ** 2 / p_n[i - 2]
|
||||
# No. of ngrams in translation that matches the reference.
|
||||
m = p_i.numerator
|
||||
# No. of ngrams in translation.
|
||||
l = sum(1 for _ in ngrams(hypothesis, i + 1))
|
||||
# Calculates the interpolated precision.
|
||||
p_n[i] = (m + self.alpha * pi0) / (l + self.alpha)
|
||||
return p_n
|
||||
|
||||
def method7(self, p_n, references, hypothesis, hyp_len=None, *args, **kwargs):
|
||||
"""
|
||||
Smoothing method 7:
|
||||
Interpolates methods 4 and 5.
|
||||
"""
|
||||
hyp_len = hyp_len if hyp_len else len(hypothesis)
|
||||
p_n = self.method4(p_n, references, hypothesis, hyp_len)
|
||||
p_n = self.method5(p_n, references, hypothesis, hyp_len)
|
||||
return p_n
|
||||
Reference in New Issue
Block a user