wehub-resource-sync
131 lines
3.4 KiB
Python
131 lines
3.4 KiB
Python
# Copyright (c) Microsoft. All rights reserved.
|
|
|
|
from pytest import mark, raises
|
|
|
|
from semantic_kernel.exceptions import CodeBlockSyntaxError
|
|
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
|
from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer
|
|
|
|
|
|
def test_it_parses_empty_text():
|
|
target = CodeTokenizer()
|
|
|
|
assert not target.tokenize(None)
|
|
assert not target.tokenize("")
|
|
assert not target.tokenize(" ")
|
|
assert not target.tokenize(" \n ")
|
|
|
|
|
|
@mark.parametrize(
|
|
"template, content",
|
|
[
|
|
("$foo", "$foo"),
|
|
("$foo ", "$foo"),
|
|
(" $foo", "$foo"),
|
|
(" $bar ", "$bar"),
|
|
],
|
|
)
|
|
def test_it_parses_var_blocks(template, content):
|
|
target = CodeTokenizer()
|
|
blocks = target.tokenize(template)
|
|
|
|
assert len(blocks) == 1
|
|
assert blocks[0].content == content
|
|
assert blocks[0].type == BlockTypes.VARIABLE
|
|
|
|
|
|
@mark.parametrize(
|
|
"template, content",
|
|
[
|
|
("'foo'", "'foo'"),
|
|
("'foo' ", "'foo'"),
|
|
(" 'foo'", "'foo'"),
|
|
(' "bar" ', '"bar"'),
|
|
],
|
|
)
|
|
def test_it_parses_val_blocks(template, content):
|
|
blocks = CodeTokenizer.tokenize(template)
|
|
|
|
assert len(blocks) == 1
|
|
assert blocks[0].content == content
|
|
assert blocks[0].type == BlockTypes.VALUE
|
|
|
|
|
|
@mark.parametrize(
|
|
"template, content",
|
|
[
|
|
("f", "f"),
|
|
(" x ", "x"),
|
|
("foo", "foo"),
|
|
("fo.o ", "fo.o"),
|
|
(" f.oo", "f.oo"),
|
|
(" bar ", "bar"),
|
|
],
|
|
)
|
|
def test_it_parses_function_id_blocks(template, content):
|
|
blocks = CodeTokenizer.tokenize(template)
|
|
|
|
assert len(blocks) == 1
|
|
assert blocks[0].content == content
|
|
assert blocks[0].type == BlockTypes.FUNCTION_ID
|
|
|
|
|
|
def test_it_parses_function_calls():
|
|
template1 = "x.y $foo"
|
|
template2 = "xy $foo"
|
|
template3 = "xy '$value'"
|
|
|
|
blocks1 = CodeTokenizer.tokenize(template1)
|
|
blocks2 = CodeTokenizer.tokenize(template2)
|
|
blocks3 = CodeTokenizer.tokenize(template3)
|
|
|
|
assert len(blocks1) == 2
|
|
assert len(blocks2) == 2
|
|
assert len(blocks3) == 2
|
|
|
|
assert blocks1[0].content == "x.y"
|
|
assert blocks2[0].content == "xy"
|
|
assert blocks3[0].content == "xy"
|
|
|
|
assert blocks1[0].type == BlockTypes.FUNCTION_ID
|
|
assert blocks2[0].type == BlockTypes.FUNCTION_ID
|
|
assert blocks3[0].type == BlockTypes.FUNCTION_ID
|
|
|
|
assert blocks1[1].content == "$foo"
|
|
assert blocks2[1].content == "$foo"
|
|
assert blocks3[1].content == "'$value'"
|
|
|
|
assert blocks1[1].type == BlockTypes.VARIABLE
|
|
assert blocks2[1].type == BlockTypes.VARIABLE
|
|
assert blocks3[1].type == BlockTypes.VALUE
|
|
|
|
|
|
def test_it_supports_escaping():
|
|
template = "func 'f\\'oo'"
|
|
blocks = CodeTokenizer.tokenize(template)
|
|
|
|
assert len(blocks) == 2
|
|
assert blocks[0].content == "func"
|
|
assert blocks[1].content == "'f'oo'"
|
|
|
|
|
|
def test_it_throws_when_separators_are_missing():
|
|
template1 = "call 'f\\\\'xy'"
|
|
template2 = "call 'f\\\\'x"
|
|
|
|
with raises(CodeBlockSyntaxError):
|
|
CodeTokenizer.tokenize(template1)
|
|
|
|
with raises(CodeBlockSyntaxError):
|
|
CodeTokenizer.tokenize(template2)
|
|
|
|
|
|
def test_named_args():
|
|
template = 'plugin.function "direct" arg1=$arg1 arg2="arg2"'
|
|
blocks = CodeTokenizer.tokenize(template)
|
|
assert len(blocks) == 4
|
|
assert blocks[0].content == "plugin.function"
|
|
assert blocks[1].content == '"direct"'
|
|
assert blocks[2].content == "arg1=$arg1"
|
|
assert blocks[3].content == 'arg2="arg2"'
|