chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,21 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pydantic import ValidationError
|
||||
from pytest import raises
|
||||
|
||||
from semantic_kernel.template_engine.blocks.block import Block
|
||||
|
||||
|
||||
def test_init():
|
||||
block = Block(content="test content")
|
||||
assert block.content == "test content"
|
||||
|
||||
|
||||
def test_content_strip():
|
||||
block = Block(content=" test content ")
|
||||
assert block.content == "test content"
|
||||
|
||||
|
||||
def test_no_content():
|
||||
with raises(ValidationError):
|
||||
Block()
|
||||
@@ -0,0 +1,581 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import (
|
||||
CodeBlockRenderException,
|
||||
CodeBlockSyntaxError,
|
||||
CodeBlockTokenError,
|
||||
FunctionIdBlockSyntaxError,
|
||||
NamedArgBlockSyntaxError,
|
||||
ValBlockSyntaxError,
|
||||
VarBlockSyntaxError,
|
||||
)
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.functions.kernel_function_decorator import kernel_function
|
||||
from semantic_kernel.functions.kernel_function_from_method import KernelFunctionFromMethod
|
||||
from semantic_kernel.functions.kernel_plugin import KernelPlugin
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.code_block import CodeBlock
|
||||
from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
|
||||
from semantic_kernel.template_engine.blocks.named_arg_block import NamedArgBlock
|
||||
from semantic_kernel.template_engine.blocks.val_block import ValBlock
|
||||
from semantic_kernel.template_engine.blocks.var_block import VarBlock
|
||||
|
||||
|
||||
def test_init():
|
||||
target = CodeBlock(
|
||||
content="plugin.function 'value' arg1=$arg1",
|
||||
)
|
||||
assert len(target.tokens) == 3
|
||||
assert target.tokens[0] == FunctionIdBlock(content="plugin.function")
|
||||
assert target.tokens[1] == ValBlock(content="'value'")
|
||||
assert target.tokens[2] == NamedArgBlock(content="arg1=$arg1")
|
||||
assert target.type == BlockTypes.CODE
|
||||
|
||||
|
||||
class TestCodeBlockRendering:
|
||||
async def test_it_throws_if_a_plugins_are_empty(self, kernel: Kernel):
|
||||
target = CodeBlock(
|
||||
content="functionName",
|
||||
)
|
||||
assert target.tokens[0].type == BlockTypes.FUNCTION_ID
|
||||
with raises(CodeBlockRenderException, match="Function `functionName` not found"):
|
||||
await target.render_code(kernel, KernelArguments())
|
||||
|
||||
async def test_it_throws_if_a_function_doesnt_exist(self, kernel: Kernel):
|
||||
target = CodeBlock(
|
||||
content="functionName",
|
||||
)
|
||||
assert target.tokens[0].type == BlockTypes.FUNCTION_ID
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[]))
|
||||
with raises(CodeBlockRenderException, match="Function `functionName` not found"):
|
||||
await target.render_code(kernel, KernelArguments())
|
||||
|
||||
async def test_it_throws_if_a_function_call_throws(self, kernel: Kernel):
|
||||
@kernel_function(name="funcName")
|
||||
def invoke():
|
||||
raise Exception("function exception")
|
||||
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_function(plugin_name="test", function=function)
|
||||
|
||||
target = CodeBlock(
|
||||
content="test.funcName",
|
||||
)
|
||||
|
||||
with raises(CodeBlockRenderException, match="test.funcName"):
|
||||
await target.render_code(kernel, KernelArguments())
|
||||
|
||||
async def test_it_renders_code_block_consisting_of_just_a_var_block1(self, kernel: Kernel):
|
||||
code_block = CodeBlock(
|
||||
content="$var",
|
||||
)
|
||||
result = await code_block.render_code(kernel, KernelArguments(var="foo"))
|
||||
|
||||
assert result == "foo"
|
||||
|
||||
async def test_it_renders_code_block_consisting_of_just_a_val_block1(self, kernel: Kernel):
|
||||
code_block = CodeBlock(
|
||||
content="'ciao'",
|
||||
)
|
||||
result = await code_block.render_code(kernel, KernelArguments())
|
||||
|
||||
assert result == "ciao"
|
||||
|
||||
async def test_it_invokes_function_cloning_all_variables(self, kernel: Kernel):
|
||||
# Set up initial context variables
|
||||
arguments = KernelArguments(input="zero", var1="uno", var2="due")
|
||||
|
||||
# Create a FunctionIdBlock with the function name
|
||||
func_id = FunctionIdBlock(content="test.funcName")
|
||||
|
||||
# Set up a canary dictionary to track changes in the context variables
|
||||
canary = {"input": "", "var1": "", "var2": ""}
|
||||
|
||||
# Define the function to be invoked, which modifies the canary
|
||||
# and context variables
|
||||
@kernel_function(name="funcName")
|
||||
def invoke(arguments: KernelArguments):
|
||||
nonlocal canary
|
||||
canary["input"] = arguments["input"]
|
||||
canary["var1"] = arguments["var1"]
|
||||
canary["var2"] = arguments["var2"]
|
||||
|
||||
arguments["input"] = "overridden"
|
||||
arguments["var1"] = "overridden"
|
||||
arguments["var2"] = "overridden"
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and render it with the context
|
||||
code_block = CodeBlock(
|
||||
tokens=[func_id],
|
||||
content="",
|
||||
)
|
||||
await code_block.render_code(kernel, arguments)
|
||||
|
||||
# Check that the canary values match the original context variables
|
||||
assert canary["input"] == "zero"
|
||||
assert canary["var1"] == "uno"
|
||||
assert canary["var2"] == "due"
|
||||
|
||||
# Check that the original context variables were not modified
|
||||
assert arguments["input"] == "zero"
|
||||
assert arguments["var1"] == "uno"
|
||||
assert arguments["var2"] == "due"
|
||||
|
||||
async def test_it_invokes_function_with_custom_variable(self, kernel: Kernel):
|
||||
# Define custom variable name and value
|
||||
VAR_NAME = "varName"
|
||||
VAR_VALUE = "varValue"
|
||||
|
||||
# Set up initial context variables
|
||||
arguments = KernelArguments()
|
||||
arguments[VAR_NAME] = VAR_VALUE
|
||||
|
||||
# Create a FunctionIdBlock with the function name and a
|
||||
# VarBlock with the custom variable
|
||||
func_id = FunctionIdBlock(content="test.funcName")
|
||||
var_block = VarBlock(content=f"${VAR_NAME}")
|
||||
|
||||
# Set up a canary variable to track changes in the context input
|
||||
canary = ""
|
||||
|
||||
# Define the function to be invoked, which modifies the canary variable
|
||||
@kernel_function(name="funcName")
|
||||
def invoke(arguments: "KernelArguments"):
|
||||
nonlocal canary
|
||||
canary = arguments["varName"]
|
||||
return arguments["varName"]
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and VarBlock,
|
||||
# and render it with the context
|
||||
code_block = CodeBlock(
|
||||
tokens=[func_id, var_block],
|
||||
content="",
|
||||
)
|
||||
result = await code_block.render_code(kernel, arguments)
|
||||
|
||||
# Check that the result matches the custom variable value
|
||||
assert result == VAR_VALUE
|
||||
# Check that the canary value matches the custom variable value
|
||||
assert canary == VAR_VALUE
|
||||
|
||||
async def test_it_invokes_function_with_custom_value(self, kernel: Kernel):
|
||||
# Define a value to be used in the test
|
||||
VALUE = "value"
|
||||
|
||||
# Create a FunctionIdBlock with the function name and a ValBlock with the value
|
||||
func_id = FunctionIdBlock(content="test.funcName")
|
||||
val_block = ValBlock(content=f"'{VALUE}'")
|
||||
|
||||
# Set up a canary variable to track changes in the context input
|
||||
canary = ""
|
||||
|
||||
# Define the function to be invoked, which modifies the canary variable
|
||||
@kernel_function(name="funcName")
|
||||
def invoke(arguments):
|
||||
nonlocal canary
|
||||
canary = arguments["input"]
|
||||
return arguments["input"]
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and ValBlock,
|
||||
# and render it with the context
|
||||
code_block = CodeBlock(
|
||||
tokens=[func_id, val_block],
|
||||
content="",
|
||||
)
|
||||
result = await code_block.render_code(kernel, KernelArguments(input="value"))
|
||||
|
||||
# Check that the result matches the value
|
||||
assert str(result) == VALUE
|
||||
# Check that the canary value matches the value
|
||||
assert canary == VALUE
|
||||
|
||||
async def test_it_invokes_function_with_multiple_arguments(self, kernel: Kernel):
|
||||
# Define a value to be used in the test
|
||||
VALUE = "value"
|
||||
|
||||
code_block = CodeBlock(
|
||||
content=" ",
|
||||
tokens=[
|
||||
FunctionIdBlock(content="test.funcName", plugin_name="test", function_name="funcName", validated=True),
|
||||
ValBlock(content=f'"{VALUE}"'),
|
||||
NamedArgBlock(content="arg1=$arg1"),
|
||||
NamedArgBlock(content='arg2="arg2"'),
|
||||
],
|
||||
)
|
||||
# Set up a canary variable to track changes in the context input
|
||||
canary = ""
|
||||
|
||||
# Define the function to be invoked, which modifies the canary variable
|
||||
@kernel_function(name="funcName")
|
||||
def invoke(input, arg1, arg2):
|
||||
nonlocal canary
|
||||
canary = f"{input} {arg1} {arg2}"
|
||||
return input
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and ValBlock,
|
||||
# and render it with the context
|
||||
result = await code_block.render_code(kernel, KernelArguments(arg1="arg1"))
|
||||
|
||||
# Check that the result matches the value
|
||||
assert str(result) == VALUE
|
||||
# Check that the canary value matches the value
|
||||
assert canary == f"{VALUE} arg1 arg2"
|
||||
|
||||
async def test_it_invokes_function_with_only_named_arguments(self, kernel: Kernel):
|
||||
code_block = CodeBlock(
|
||||
content=" ",
|
||||
tokens=[
|
||||
FunctionIdBlock(content="test.funcName", plugin_name="test", function_name="funcName"),
|
||||
NamedArgBlock(content="arg1=$arg1"),
|
||||
NamedArgBlock(content='arg2="arg2"'),
|
||||
],
|
||||
)
|
||||
# Set up a canary variable to track changes in the context input
|
||||
canary = ""
|
||||
|
||||
# Define the function to be invoked, which modifies the canary variable
|
||||
@kernel_function(name="funcName")
|
||||
def invoke(arg1, arg2):
|
||||
nonlocal canary
|
||||
canary = f"{arg1} {arg2}"
|
||||
return arg1
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="pluginName",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and ValBlock,
|
||||
# and render it with the context
|
||||
result = await code_block.render_code(kernel, KernelArguments(arg1="arg1"))
|
||||
|
||||
# Check that the result matches the value
|
||||
assert str(result) == "arg1"
|
||||
# Check that the canary value matches the value
|
||||
assert canary == "arg1 arg2"
|
||||
|
||||
async def test_it_fails_on_function_without_args(self, kernel: Kernel):
|
||||
code_block = CodeBlock(
|
||||
content=" ",
|
||||
tokens=[
|
||||
FunctionIdBlock(content="test.funcName", plugin_name="test", function_name="funcName"),
|
||||
NamedArgBlock(content="arg1=$arg1"),
|
||||
NamedArgBlock(content='arg2="arg2"'),
|
||||
],
|
||||
)
|
||||
|
||||
@kernel_function(name="funcName")
|
||||
def invoke():
|
||||
return "function without args"
|
||||
|
||||
# Create an KernelFunction with the invoke function as its delegate
|
||||
function = KernelFunctionFromMethod(
|
||||
method=invoke,
|
||||
plugin_name="test",
|
||||
)
|
||||
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
# Create a CodeBlock with the FunctionIdBlock and ValBlock,
|
||||
# and render it with the context
|
||||
with raises(
|
||||
CodeBlockRenderException,
|
||||
match="Function test.funcName does not take any arguments \
|
||||
but it is being called in the template with 2 arguments.",
|
||||
):
|
||||
await code_block.render_code(kernel, KernelArguments(arg1="arg1"))
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"token2",
|
||||
[
|
||||
"",
|
||||
"arg2=$arg!2",
|
||||
"arg2='va\"l'",
|
||||
],
|
||||
ids=[
|
||||
"empty",
|
||||
"invalid_named_arg",
|
||||
"invalid_named_arg_val",
|
||||
],
|
||||
)
|
||||
@mark.parametrize(
|
||||
"token1",
|
||||
[
|
||||
"",
|
||||
"$var!",
|
||||
"\"val'",
|
||||
"arg1=$arg!1",
|
||||
"arg1='va\"l'",
|
||||
],
|
||||
ids=[
|
||||
"empty",
|
||||
"invalid_var",
|
||||
"invalid_val",
|
||||
"invalid_named_arg",
|
||||
"invalid_named_arg_val",
|
||||
],
|
||||
)
|
||||
@mark.parametrize(
|
||||
"token0",
|
||||
[
|
||||
"plugin.func.test",
|
||||
"$var!",
|
||||
'"va"l"',
|
||||
],
|
||||
ids=[
|
||||
"invalid_func",
|
||||
"invalid_var",
|
||||
"invalid_val",
|
||||
],
|
||||
)
|
||||
def test_block_validation(token0, token1, token2):
|
||||
with raises((
|
||||
FunctionIdBlockSyntaxError,
|
||||
VarBlockSyntaxError,
|
||||
ValBlockSyntaxError,
|
||||
NamedArgBlockSyntaxError,
|
||||
CodeBlockSyntaxError,
|
||||
)):
|
||||
CodeBlock(
|
||||
content=f"{token0} {token1} {token2}",
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"token2, token2valid",
|
||||
[
|
||||
("", True),
|
||||
("plugin.func", False),
|
||||
("$var", False),
|
||||
('"val"', False),
|
||||
("arg1=$arg1", True),
|
||||
("arg1='val'", True),
|
||||
],
|
||||
ids=[
|
||||
"empty",
|
||||
"func_invalid",
|
||||
"invalid_var",
|
||||
"invalid_val",
|
||||
"valid_named_arg",
|
||||
"valid_named_arg_val",
|
||||
],
|
||||
)
|
||||
@mark.parametrize(
|
||||
"token1, token1valid",
|
||||
[
|
||||
("", True),
|
||||
("plugin.func", False),
|
||||
("$var", True),
|
||||
('"val"', True),
|
||||
("arg1=$arg1", True),
|
||||
("arg1='val'", True),
|
||||
],
|
||||
ids=[
|
||||
"empty",
|
||||
"func_invalid",
|
||||
"var",
|
||||
"val",
|
||||
"valid_named_arg",
|
||||
"valid_named_arg_val",
|
||||
],
|
||||
)
|
||||
@mark.parametrize(
|
||||
"token0, token0valid",
|
||||
[
|
||||
("func", True),
|
||||
("plugin.func", True),
|
||||
("$var", True),
|
||||
('"val"', True),
|
||||
("arg1=$arg1", False),
|
||||
("arg1='val'", False),
|
||||
],
|
||||
ids=[
|
||||
"single_name_func",
|
||||
"FQN_func",
|
||||
"var",
|
||||
"val",
|
||||
"invalid_named_arg",
|
||||
"invalid_named_arg_val",
|
||||
],
|
||||
)
|
||||
def test_positional_validation(token0, token0valid, token1, token1valid, token2, token2valid):
|
||||
if not token1 and not token2valid:
|
||||
mark.skipif(f"{token0} {token1} {token2}", reason="Not applicable")
|
||||
return
|
||||
valid = token0valid and token1valid and token2valid
|
||||
if token0 in ["$var", '"val"']:
|
||||
valid = True
|
||||
content = f"{token0} {token1} {token2}"
|
||||
if valid:
|
||||
target = CodeBlock(
|
||||
content=content,
|
||||
)
|
||||
assert target.content == content.strip()
|
||||
else:
|
||||
with raises(CodeBlockTokenError):
|
||||
CodeBlock(
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"case, result",
|
||||
[
|
||||
(r"{$a", False),
|
||||
],
|
||||
)
|
||||
def test_edge_cases(case, result):
|
||||
if result:
|
||||
target = CodeBlock(
|
||||
content=case,
|
||||
)
|
||||
assert target.content == case
|
||||
else:
|
||||
with raises(FunctionIdBlockSyntaxError):
|
||||
CodeBlock(
|
||||
content=case,
|
||||
)
|
||||
|
||||
|
||||
def test_no_tokens():
|
||||
with raises(CodeBlockTokenError):
|
||||
CodeBlock(content="", tokens=[])
|
||||
|
||||
|
||||
class TestNonStringArguments:
|
||||
"""Test that non-string KernelArguments are preserved when passed to functions in templates."""
|
||||
|
||||
async def test_function_receives_int_type(self, kernel: Kernel):
|
||||
"""Test that an integer argument is passed as int, not converted to string."""
|
||||
received_value = None
|
||||
received_type = None
|
||||
|
||||
@kernel_function(name="check_type")
|
||||
def check_type(value: int):
|
||||
nonlocal received_value, received_type
|
||||
received_value = value
|
||||
received_type = type(value)
|
||||
return f"Received {type(value).__name__}: {value}"
|
||||
|
||||
function = KernelFunctionFromMethod(method=check_type, plugin_name="test")
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
code_block = CodeBlock(content="test.check_type value=$my_int")
|
||||
arguments = KernelArguments(my_int=42)
|
||||
|
||||
await code_block.render_code(kernel, arguments)
|
||||
|
||||
assert received_value == 42
|
||||
assert isinstance(received_value, int), f"Expected int but got {received_type}"
|
||||
|
||||
async def test_function_receives_list_type(self, kernel: Kernel):
|
||||
"""Test that a list argument is passed as list, not converted to string."""
|
||||
received_value = None
|
||||
received_type = None
|
||||
|
||||
@kernel_function(name="check_type")
|
||||
def check_type(items: list):
|
||||
nonlocal received_value, received_type
|
||||
received_value = items
|
||||
received_type = type(items)
|
||||
return f"Received {len(items)} items"
|
||||
|
||||
function = KernelFunctionFromMethod(method=check_type, plugin_name="test")
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
code_block = CodeBlock(content="test.check_type items=$my_list")
|
||||
arguments = KernelArguments(my_list=[1, 2, 3])
|
||||
|
||||
await code_block.render_code(kernel, arguments)
|
||||
|
||||
assert received_value == [1, 2, 3]
|
||||
assert isinstance(received_value, list), f"Expected list but got {received_type}"
|
||||
|
||||
async def test_function_receives_dict_type(self, kernel: Kernel):
|
||||
"""Test that a dict argument is passed as dict, not converted to string."""
|
||||
received_value = None
|
||||
received_type = None
|
||||
|
||||
@kernel_function(name="check_type")
|
||||
def check_type(data: dict):
|
||||
nonlocal received_value, received_type
|
||||
received_value = data
|
||||
received_type = type(data)
|
||||
return f"Received dict with keys: {list(data.keys())}"
|
||||
|
||||
function = KernelFunctionFromMethod(method=check_type, plugin_name="test")
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
code_block = CodeBlock(content="test.check_type data=$my_dict")
|
||||
arguments = KernelArguments(my_dict={"key": "value", "num": 123})
|
||||
|
||||
await code_block.render_code(kernel, arguments)
|
||||
|
||||
assert received_value == {"key": "value", "num": 123}
|
||||
assert isinstance(received_value, dict), f"Expected dict but got {received_type}"
|
||||
|
||||
async def test_named_arg_with_non_string_type(self, kernel: Kernel):
|
||||
"""Test that named arguments with non-string types are preserved."""
|
||||
received_count = None
|
||||
received_type = None
|
||||
|
||||
@kernel_function(name="process")
|
||||
def process(text: str, count: int):
|
||||
nonlocal received_count, received_type
|
||||
received_count = count
|
||||
received_type = type(count)
|
||||
return f"{text} x {count}"
|
||||
|
||||
function = KernelFunctionFromMethod(method=process, plugin_name="test")
|
||||
kernel.add_plugin(KernelPlugin(name="test", functions=[function]))
|
||||
|
||||
code_block = CodeBlock(content="test.process 'hello' count=$repetitions")
|
||||
arguments = KernelArguments(repetitions=5)
|
||||
|
||||
await code_block.render_code(kernel, arguments)
|
||||
|
||||
assert received_count == 5
|
||||
assert isinstance(received_count, int), f"Expected int but got {received_type}"
|
||||
@@ -0,0 +1,74 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import FunctionIdBlockSyntaxError
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
|
||||
|
||||
|
||||
def test_init():
|
||||
function_id_block = FunctionIdBlock(content="plugin.function")
|
||||
assert function_id_block.content == "plugin.function"
|
||||
assert function_id_block.plugin_name == "plugin"
|
||||
assert function_id_block.function_name == "function"
|
||||
assert function_id_block.type == BlockTypes.FUNCTION_ID
|
||||
|
||||
|
||||
def test_init_function_only():
|
||||
function_id_block = FunctionIdBlock(content="function")
|
||||
assert function_id_block.content == "function"
|
||||
assert not function_id_block.plugin_name
|
||||
assert function_id_block.function_name == "function"
|
||||
assert function_id_block.type == BlockTypes.FUNCTION_ID
|
||||
|
||||
|
||||
def test_it_trims_spaces():
|
||||
assert FunctionIdBlock(content=" aa ").content == "aa"
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"0",
|
||||
"1",
|
||||
"a",
|
||||
"_",
|
||||
"01",
|
||||
"01a",
|
||||
"a01",
|
||||
"_0",
|
||||
"a01_",
|
||||
"_a01",
|
||||
],
|
||||
)
|
||||
def test_valid_syntax(name):
|
||||
target = FunctionIdBlock(content=name)
|
||||
assert target.content == name
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"content",
|
||||
["", "plugin.nope.function", "func-tion", "plu-in.function", ".function"],
|
||||
ids=["empty", "three_parts", "invalid_function", "invalid_plugin", "no_plugin"],
|
||||
)
|
||||
def test_syntax_error(content):
|
||||
with raises(FunctionIdBlockSyntaxError, match=rf".*{content}.*"):
|
||||
FunctionIdBlock(content=content)
|
||||
|
||||
|
||||
def test_render():
|
||||
kernel = Kernel()
|
||||
function_id_block = FunctionIdBlock(content="plugin.function")
|
||||
rendered_value = function_id_block.render(kernel, KernelArguments())
|
||||
assert rendered_value == "plugin.function"
|
||||
|
||||
|
||||
def test_render_function_only():
|
||||
kernel = Kernel()
|
||||
function_id_block = FunctionIdBlock(content="function")
|
||||
rendered_value = function_id_block.render(kernel, KernelArguments())
|
||||
assert rendered_value == "function"
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import NamedArgBlockSyntaxError
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.named_arg_block import NamedArgBlock
|
||||
from semantic_kernel.template_engine.blocks.val_block import ValBlock
|
||||
from semantic_kernel.template_engine.blocks.var_block import VarBlock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_init_with_var():
|
||||
named_arg_block = NamedArgBlock(content="test=$test_var")
|
||||
assert named_arg_block.content == "test=$test_var"
|
||||
assert named_arg_block.name == "test"
|
||||
assert named_arg_block.variable.name == "test_var"
|
||||
assert isinstance(named_arg_block.variable, VarBlock)
|
||||
|
||||
|
||||
def test_init_with_val():
|
||||
named_arg_block = NamedArgBlock(content="test='test_val'")
|
||||
assert named_arg_block.content == "test='test_val'"
|
||||
assert named_arg_block.name == "test"
|
||||
assert named_arg_block.value.value == "test_val"
|
||||
assert isinstance(named_arg_block.value, ValBlock)
|
||||
|
||||
|
||||
def test_type_property():
|
||||
named_arg_block = NamedArgBlock(content="test=$test_var")
|
||||
assert named_arg_block.type == BlockTypes.NAMED_ARG
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
"=$test_var",
|
||||
"test=$test-var",
|
||||
"test='test_val\"",
|
||||
"test=''",
|
||||
"test=$",
|
||||
],
|
||||
ids=["no_name", "invalid_var", "invalid_val", "empty_val", "empty_var"],
|
||||
)
|
||||
def test_syntax_error(content):
|
||||
match = content.replace("$", "\\$") if "$" in content else content
|
||||
with raises(NamedArgBlockSyntaxError, match=rf".*{match}.*"):
|
||||
NamedArgBlock(content=content)
|
||||
|
||||
|
||||
def test_render():
|
||||
named_arg_block = NamedArgBlock(content="test=$test_var")
|
||||
rendered_value = named_arg_block.render(Kernel(), KernelArguments(test_var="test_value"))
|
||||
assert rendered_value == "test_value"
|
||||
|
||||
|
||||
def test_render_variable_not_found():
|
||||
named_arg_block = NamedArgBlock(content="test=$test_var")
|
||||
rendered_value = named_arg_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == ""
|
||||
|
||||
|
||||
def test_init_minimal_var():
|
||||
block = NamedArgBlock(content="a=$a")
|
||||
assert block.name == "a"
|
||||
assert block.variable.name == "a"
|
||||
|
||||
|
||||
def test_init_minimal_val():
|
||||
block = NamedArgBlock(content="a='a'")
|
||||
assert block.name == "a"
|
||||
assert block.value.value == "a"
|
||||
|
||||
|
||||
def test_init_empty():
|
||||
with raises(NamedArgBlockSyntaxError, match=r".*"):
|
||||
NamedArgBlock(content="")
|
||||
|
||||
|
||||
def test_it_trims_spaces():
|
||||
assert NamedArgBlock(content=" a=$x ").content == "a=$x"
|
||||
|
||||
|
||||
def test_it_ignores_spaces_around():
|
||||
target = NamedArgBlock(content=" a=$var \n ")
|
||||
assert target.content == "a=$var"
|
||||
|
||||
|
||||
def test_it_renders_to_empty_string_without_variables():
|
||||
target = NamedArgBlock(content=" a=$var \n ")
|
||||
result = target.render(Kernel(), None)
|
||||
assert result == ""
|
||||
|
||||
|
||||
def test_it_renders_to_empty_string_if_variable_is_missing():
|
||||
target = NamedArgBlock(content=" a=$var \n ")
|
||||
result = target.render(Kernel(), KernelArguments(foo="bar"))
|
||||
assert result == ""
|
||||
|
||||
|
||||
def test_it_renders_to_variable_value_when_available():
|
||||
target = NamedArgBlock(content=" a=$var \n ")
|
||||
result = target.render(Kernel(), KernelArguments(foo="bar", var="able"))
|
||||
assert result == "able"
|
||||
|
||||
|
||||
def test_it_renders_to_value():
|
||||
target = NamedArgBlock(content=" a='var' \n ")
|
||||
result = target.render(Kernel(), None)
|
||||
assert result == "var"
|
||||
@@ -0,0 +1,95 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.text_block import TextBlock
|
||||
|
||||
|
||||
def test_init():
|
||||
text_block = TextBlock.from_text(text="test text")
|
||||
assert text_block.content == "test text"
|
||||
|
||||
|
||||
def test_init_with_just_start_index():
|
||||
text_block = TextBlock.from_text(text="test text", start_index=2)
|
||||
assert text_block.content == "st text"
|
||||
|
||||
|
||||
def test_init_with_just_stop_index():
|
||||
text_block = TextBlock.from_text(text="test text", stop_index=2)
|
||||
assert text_block.content == "te"
|
||||
|
||||
|
||||
def test_init_with_start_index_greater_than_stop_index():
|
||||
with pytest.raises(ValueError):
|
||||
TextBlock.from_text(text="test text", start_index=2, stop_index=1)
|
||||
|
||||
|
||||
def test_init_with_start_stop_indices():
|
||||
text_block = TextBlock.from_text(text="test text", start_index=0, stop_index=4)
|
||||
assert text_block.content == "test"
|
||||
|
||||
|
||||
def test_init_with_start_index_less_than_zero():
|
||||
with pytest.raises(ValueError):
|
||||
TextBlock.from_text(text="test text", start_index=-1, stop_index=1)
|
||||
|
||||
|
||||
def test_init_with_negative_stop_index():
|
||||
text_block = TextBlock.from_text(text="test text", stop_index=-1)
|
||||
assert text_block.content == "test tex"
|
||||
|
||||
|
||||
def test_type_property():
|
||||
text_block = TextBlock.from_text(text="test text")
|
||||
assert text_block.type == BlockTypes.TEXT
|
||||
|
||||
|
||||
def test_render():
|
||||
text_block = TextBlock.from_text(text="test text")
|
||||
rendered_value = text_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == "test text"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_", "output"),
|
||||
[
|
||||
(None, ""),
|
||||
("", ""),
|
||||
(" ", " "),
|
||||
(" ", " "),
|
||||
(" ", " "),
|
||||
(" \n", " \n"),
|
||||
(" \t", " \t"),
|
||||
(" \r", " \r"),
|
||||
],
|
||||
ids=["None", "empty", "space", "two_spaces", "three_spaces", "space_newline", "space_tab", "space_carriage_return"],
|
||||
)
|
||||
def test_preserves_empty_values(input_, output):
|
||||
assert output == TextBlock.from_text(text=input_).content
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_", "output"),
|
||||
[
|
||||
(None, ""),
|
||||
("", ""),
|
||||
(" ", " "),
|
||||
(" ", " "),
|
||||
(" ", " "),
|
||||
(" \n", " \n"),
|
||||
(" \t", " \t"),
|
||||
(" \r", " \r"),
|
||||
("test", "test"),
|
||||
(" \nabc", " \nabc"),
|
||||
("'x'", "'x'"),
|
||||
('"x"', '"x"'),
|
||||
("\"'x'\"", "\"'x'\""),
|
||||
],
|
||||
)
|
||||
def test_renders_the_content_as_it(input_, output):
|
||||
assert TextBlock.from_text(text=input_).render() == output
|
||||
@@ -0,0 +1,67 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import ValBlockSyntaxError
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.val_block import ValBlock
|
||||
|
||||
|
||||
def test_init_single_quote():
|
||||
val_block = ValBlock(content="'test value'")
|
||||
assert val_block.content == "'test value'"
|
||||
assert val_block.value == "test value"
|
||||
assert val_block.quote == "'"
|
||||
assert val_block.type == BlockTypes.VALUE
|
||||
|
||||
|
||||
def test_init_double_quote():
|
||||
val_block = ValBlock(content='"test value"')
|
||||
assert val_block.content == '"test value"'
|
||||
assert val_block.value == "test value"
|
||||
assert val_block.quote == '"'
|
||||
assert val_block.type == BlockTypes.VALUE
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
"test value",
|
||||
"'test value",
|
||||
"test value'",
|
||||
'"test value',
|
||||
'test value"',
|
||||
"'test value\"",
|
||||
],
|
||||
ids=[
|
||||
"no_quotes",
|
||||
"single_quote_start",
|
||||
"single_quote_end",
|
||||
"double_quote_start",
|
||||
"double_quote_end",
|
||||
"mixed_quote",
|
||||
],
|
||||
)
|
||||
def test_syntax_error(content):
|
||||
with raises(ValBlockSyntaxError, match=rf".*{content}*"):
|
||||
ValBlock(content=content)
|
||||
|
||||
|
||||
def test_render():
|
||||
val_block = ValBlock(content="'test value'")
|
||||
rendered_value = val_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == "test value"
|
||||
|
||||
|
||||
def test_escaping():
|
||||
val_block = ValBlock(content="'f\\'oo'")
|
||||
rendered_value = val_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == "f\\'oo"
|
||||
|
||||
|
||||
def test_escaping2():
|
||||
val_block = ValBlock(content=r"'f\'oo'")
|
||||
rendered_value = val_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == r"f\'oo"
|
||||
@@ -0,0 +1,87 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import VarBlockSyntaxError
|
||||
from semantic_kernel.exceptions.template_engine_exceptions import VarBlockRenderError
|
||||
from semantic_kernel.functions.kernel_arguments import KernelArguments
|
||||
from semantic_kernel.kernel import Kernel
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.blocks.var_block import VarBlock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_init():
|
||||
var_block = VarBlock(content="$test_var")
|
||||
assert var_block.content == "$test_var"
|
||||
assert var_block.name == "test_var"
|
||||
assert var_block.type == BlockTypes.VARIABLE
|
||||
|
||||
|
||||
def test_it_trims_spaces():
|
||||
assert VarBlock(content=" $x ").content == "$x"
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"0",
|
||||
"1",
|
||||
"a",
|
||||
"_",
|
||||
"01",
|
||||
"01a",
|
||||
"a01",
|
||||
"_0",
|
||||
"a01_e",
|
||||
"_a01e",
|
||||
],
|
||||
)
|
||||
def test_valid_syntax(name):
|
||||
target = VarBlock(content=f" ${name} ")
|
||||
result = target.render(Kernel(), KernelArguments(**{name: "value"}))
|
||||
assert target.name == name
|
||||
assert result == "value"
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"content",
|
||||
["$", "$test-var", "test_var", "$a>b", "$."],
|
||||
ids=["prefix_only", "invalid_characters", "no_prefix", "invalid_characters2", "invalid_characters3"],
|
||||
)
|
||||
def test_syntax_errors(content):
|
||||
match = content.replace("$", "\\$") if "$" in content else content
|
||||
with raises(VarBlockSyntaxError, match=rf".*{match}.*"):
|
||||
VarBlock(content=content)
|
||||
|
||||
|
||||
def test_render():
|
||||
var_block = VarBlock(content="$test_var")
|
||||
rendered_value = var_block.render(Kernel(), KernelArguments(test_var="test_value"))
|
||||
assert rendered_value == "test_value"
|
||||
|
||||
|
||||
def test_render_variable_not_found():
|
||||
var_block = VarBlock(content="$test_var")
|
||||
rendered_value = var_block.render(Kernel(), KernelArguments())
|
||||
assert rendered_value == ""
|
||||
|
||||
|
||||
def test_render_no_args():
|
||||
target = VarBlock(content="$var")
|
||||
result = target.render(Kernel())
|
||||
assert result == ""
|
||||
|
||||
|
||||
class MockNonString(str):
|
||||
def __str__(self):
|
||||
raise ValueError("This is not a string")
|
||||
|
||||
|
||||
def test_not_string():
|
||||
target = VarBlock(content="$var")
|
||||
with raises(VarBlockRenderError):
|
||||
target.render(Kernel(), KernelArguments(var=MockNonString("1")))
|
||||
@@ -0,0 +1,130 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import CodeBlockSyntaxError
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer
|
||||
|
||||
|
||||
def test_it_parses_empty_text():
|
||||
target = CodeTokenizer()
|
||||
|
||||
assert not target.tokenize(None)
|
||||
assert not target.tokenize("")
|
||||
assert not target.tokenize(" ")
|
||||
assert not target.tokenize(" \n ")
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template, content",
|
||||
[
|
||||
("$foo", "$foo"),
|
||||
("$foo ", "$foo"),
|
||||
(" $foo", "$foo"),
|
||||
(" $bar ", "$bar"),
|
||||
],
|
||||
)
|
||||
def test_it_parses_var_blocks(template, content):
|
||||
target = CodeTokenizer()
|
||||
blocks = target.tokenize(template)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].content == content
|
||||
assert blocks[0].type == BlockTypes.VARIABLE
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template, content",
|
||||
[
|
||||
("'foo'", "'foo'"),
|
||||
("'foo' ", "'foo'"),
|
||||
(" 'foo'", "'foo'"),
|
||||
(' "bar" ', '"bar"'),
|
||||
],
|
||||
)
|
||||
def test_it_parses_val_blocks(template, content):
|
||||
blocks = CodeTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].content == content
|
||||
assert blocks[0].type == BlockTypes.VALUE
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template, content",
|
||||
[
|
||||
("f", "f"),
|
||||
(" x ", "x"),
|
||||
("foo", "foo"),
|
||||
("fo.o ", "fo.o"),
|
||||
(" f.oo", "f.oo"),
|
||||
(" bar ", "bar"),
|
||||
],
|
||||
)
|
||||
def test_it_parses_function_id_blocks(template, content):
|
||||
blocks = CodeTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].content == content
|
||||
assert blocks[0].type == BlockTypes.FUNCTION_ID
|
||||
|
||||
|
||||
def test_it_parses_function_calls():
|
||||
template1 = "x.y $foo"
|
||||
template2 = "xy $foo"
|
||||
template3 = "xy '$value'"
|
||||
|
||||
blocks1 = CodeTokenizer.tokenize(template1)
|
||||
blocks2 = CodeTokenizer.tokenize(template2)
|
||||
blocks3 = CodeTokenizer.tokenize(template3)
|
||||
|
||||
assert len(blocks1) == 2
|
||||
assert len(blocks2) == 2
|
||||
assert len(blocks3) == 2
|
||||
|
||||
assert blocks1[0].content == "x.y"
|
||||
assert blocks2[0].content == "xy"
|
||||
assert blocks3[0].content == "xy"
|
||||
|
||||
assert blocks1[0].type == BlockTypes.FUNCTION_ID
|
||||
assert blocks2[0].type == BlockTypes.FUNCTION_ID
|
||||
assert blocks3[0].type == BlockTypes.FUNCTION_ID
|
||||
|
||||
assert blocks1[1].content == "$foo"
|
||||
assert blocks2[1].content == "$foo"
|
||||
assert blocks3[1].content == "'$value'"
|
||||
|
||||
assert blocks1[1].type == BlockTypes.VARIABLE
|
||||
assert blocks2[1].type == BlockTypes.VARIABLE
|
||||
assert blocks3[1].type == BlockTypes.VALUE
|
||||
|
||||
|
||||
def test_it_supports_escaping():
|
||||
template = "func 'f\\'oo'"
|
||||
blocks = CodeTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 2
|
||||
assert blocks[0].content == "func"
|
||||
assert blocks[1].content == "'f'oo'"
|
||||
|
||||
|
||||
def test_it_throws_when_separators_are_missing():
|
||||
template1 = "call 'f\\\\'xy'"
|
||||
template2 = "call 'f\\\\'x"
|
||||
|
||||
with raises(CodeBlockSyntaxError):
|
||||
CodeTokenizer.tokenize(template1)
|
||||
|
||||
with raises(CodeBlockSyntaxError):
|
||||
CodeTokenizer.tokenize(template2)
|
||||
|
||||
|
||||
def test_named_args():
|
||||
template = 'plugin.function "direct" arg1=$arg1 arg2="arg2"'
|
||||
blocks = CodeTokenizer.tokenize(template)
|
||||
assert len(blocks) == 4
|
||||
assert blocks[0].content == "plugin.function"
|
||||
assert blocks[1].content == '"direct"'
|
||||
assert blocks[2].content == "arg1=$arg1"
|
||||
assert blocks[3].content == 'arg2="arg2"'
|
||||
@@ -0,0 +1,212 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.exceptions import TemplateSyntaxError
|
||||
from semantic_kernel.template_engine.blocks.block_types import BlockTypes
|
||||
from semantic_kernel.template_engine.template_tokenizer import TemplateTokenizer
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"text, block_type",
|
||||
[
|
||||
(None, BlockTypes.TEXT),
|
||||
("", BlockTypes.TEXT),
|
||||
(" ", BlockTypes.TEXT),
|
||||
(" ", BlockTypes.TEXT),
|
||||
(" {} ", BlockTypes.TEXT),
|
||||
(" {{} ", BlockTypes.TEXT),
|
||||
(" {{ } } } ", BlockTypes.TEXT),
|
||||
(" { { }} }", BlockTypes.TEXT),
|
||||
("{{}}", BlockTypes.TEXT),
|
||||
("{{ }}", BlockTypes.TEXT),
|
||||
("{{ }}", BlockTypes.TEXT),
|
||||
("{{ '}}x", BlockTypes.TEXT),
|
||||
('{{ "}}x', BlockTypes.TEXT),
|
||||
],
|
||||
)
|
||||
def test_it_parses_text_without_code(text, block_type):
|
||||
blocks = TemplateTokenizer.tokenize(text)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].type == block_type
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"text, block_type",
|
||||
[
|
||||
("", BlockTypes.TEXT),
|
||||
(" ", BlockTypes.TEXT),
|
||||
(" ", BlockTypes.TEXT),
|
||||
(" aaa ", BlockTypes.TEXT),
|
||||
("{{$a}}", BlockTypes.VARIABLE),
|
||||
("{{ $a}}", BlockTypes.VARIABLE),
|
||||
("{{ $a }}", BlockTypes.VARIABLE),
|
||||
("{{ $a }}", BlockTypes.VARIABLE),
|
||||
("{{code}}", BlockTypes.CODE),
|
||||
("{{code }}", BlockTypes.CODE),
|
||||
("{{ code }}", BlockTypes.CODE),
|
||||
("{{ code }}", BlockTypes.CODE),
|
||||
("{{ code }}", BlockTypes.CODE),
|
||||
("{{''}}", BlockTypes.VALUE),
|
||||
("{{' '}}", BlockTypes.VALUE),
|
||||
("{{ ' '}}", BlockTypes.VALUE),
|
||||
("{{ ' ' }}", BlockTypes.VALUE),
|
||||
("{{ ' ' }}", BlockTypes.VALUE),
|
||||
("{{ ' ' }}", BlockTypes.VALUE),
|
||||
],
|
||||
)
|
||||
def test_it_parses_basic_blocks(text, block_type):
|
||||
blocks = TemplateTokenizer.tokenize(text)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].type == block_type
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template, block_count",
|
||||
[
|
||||
(None, 1),
|
||||
("", 1),
|
||||
("}}{{a}} {{b}}x", 5),
|
||||
("}}{{ -a\n} } {{b}}x", 3),
|
||||
],
|
||||
)
|
||||
def test_it_tokenizes_the_right_token_count(template, block_count):
|
||||
blocks = TemplateTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == block_count
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template, error",
|
||||
[
|
||||
("}}{{{ {$a}}}} {{b}}x}}", TemplateSyntaxError),
|
||||
("}}{{ -a}} {{b}}x", TemplateSyntaxError),
|
||||
("}}{{ -a\n}} {{b}}x", TemplateSyntaxError),
|
||||
("{{ plugin.func $va-r }}", TemplateSyntaxError),
|
||||
("{{ plugin.func 'val' 'val' }}", TemplateSyntaxError),
|
||||
("{{ arg=$arg }}", TemplateSyntaxError),
|
||||
("{{ plugin.func 'var'arg=$arg }}", TemplateSyntaxError),
|
||||
],
|
||||
ids=[
|
||||
"invalid_function_id",
|
||||
"invalid_function_id_2",
|
||||
"invalid_function_id_3",
|
||||
"invalid_var",
|
||||
"invalid_code_blocks",
|
||||
"invalid_named_arg",
|
||||
"invalid_code_block_syntax",
|
||||
],
|
||||
)
|
||||
def test_invalid_syntax(template, error):
|
||||
with raises(error):
|
||||
TemplateTokenizer.tokenize(template)
|
||||
|
||||
|
||||
def test_it_tokenizes_edge_cases_correctly_1():
|
||||
blocks1 = TemplateTokenizer.tokenize("{{{{a}}")
|
||||
blocks2 = TemplateTokenizer.tokenize("{{'{{a}}")
|
||||
blocks3 = TemplateTokenizer.tokenize("{{'a}}")
|
||||
blocks4 = TemplateTokenizer.tokenize("{{a'}}")
|
||||
|
||||
assert len(blocks1) == 2
|
||||
assert len(blocks2) == 1
|
||||
assert len(blocks3) == 1
|
||||
assert len(blocks4) == 1
|
||||
|
||||
assert blocks1[0].type == BlockTypes.TEXT
|
||||
assert blocks1[1].type == BlockTypes.CODE
|
||||
|
||||
assert blocks1[0].content == "{{"
|
||||
assert blocks1[1].content == "a"
|
||||
|
||||
|
||||
def test_it_tokenizes_edge_cases_correctly_3():
|
||||
template = "}}{{{{$a}}}} {{b}}$x}}"
|
||||
|
||||
blocks = TemplateTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 5
|
||||
|
||||
assert blocks[0].content == "}}{{"
|
||||
assert blocks[0].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[1].content == "$a"
|
||||
assert blocks[1].type == BlockTypes.VARIABLE
|
||||
|
||||
assert blocks[2].content == "}} "
|
||||
assert blocks[2].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[3].content == "b"
|
||||
assert blocks[3].type == BlockTypes.CODE
|
||||
|
||||
assert blocks[4].content == "$x}}"
|
||||
assert blocks[4].type == BlockTypes.TEXT
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"template",
|
||||
[
|
||||
("{{ asis 'f\\'oo' }}"),
|
||||
],
|
||||
)
|
||||
def test_it_tokenizes_edge_cases_correctly_4(template):
|
||||
blocks = TemplateTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 1
|
||||
assert blocks[0].type == BlockTypes.CODE
|
||||
assert blocks[0].content == template[2:-2].strip()
|
||||
|
||||
|
||||
def test_it_tokenizes_a_typical_prompt():
|
||||
template = "this is a {{ $prompt }} with {{$some}} variables and {{function $calls}} {{ and 'values' }}"
|
||||
|
||||
blocks = TemplateTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 8
|
||||
|
||||
assert blocks[0].content == "this is a "
|
||||
assert blocks[0].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[1].content == "$prompt"
|
||||
assert blocks[1].type == BlockTypes.VARIABLE
|
||||
|
||||
assert blocks[2].content == " with "
|
||||
assert blocks[2].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[3].content == "$some"
|
||||
assert blocks[3].type == BlockTypes.VARIABLE
|
||||
|
||||
assert blocks[4].content == " variables and "
|
||||
assert blocks[4].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[5].content == "function $calls"
|
||||
assert blocks[5].type == BlockTypes.CODE
|
||||
|
||||
assert blocks[6].content == " "
|
||||
assert blocks[6].type == BlockTypes.TEXT
|
||||
|
||||
assert blocks[7].content == "and 'values'"
|
||||
assert blocks[7].type == BlockTypes.CODE
|
||||
|
||||
|
||||
def test_it_tokenizes_a_named_args_prompt():
|
||||
template = '{{ plugin.function "direct" arg1=$arg1 arg2="arg2" }}'
|
||||
|
||||
blocks = TemplateTokenizer.tokenize(template)
|
||||
|
||||
assert len(blocks) == 1
|
||||
block = blocks[0]
|
||||
assert block.type == BlockTypes.CODE
|
||||
|
||||
assert len(block.tokens) == 4
|
||||
assert block.tokens[0].type == BlockTypes.FUNCTION_ID
|
||||
assert block.tokens[1].type == BlockTypes.VALUE
|
||||
assert block.tokens[2].type == BlockTypes.NAMED_ARG
|
||||
assert block.tokens[3].type == BlockTypes.NAMED_ARG
|
||||
|
||||
assert block.tokens[2].name == "arg1"
|
||||
assert block.tokens[2].variable.content == "$arg1"
|
||||
assert block.tokens[3].name == "arg2"
|
||||
assert block.tokens[3].value.content == '"arg2"'
|
||||
Reference in New Issue
Block a user