chore: import upstream snapshot with attribution

python/semantic_kernel/template_engine/README.md

@@ -0,0 +1,26 @@
+# Prompt Template Engine
+
+The Semantic Kernel uses the following grammar to parse prompt templates:
+
+```
+# BNF parsed by TemplateTokenizer
+[template]       ::= "" | [block] | [block] [template]
+[block]          ::= [sk-block] | [text-block]
+[sk-block]       ::= "{{" [variable] "}}" | "{{" [value] "}}" | "{{" [function-call] "}}"
+[text-block]     ::= [any-char] | [any-char] [text-block]
+[any-char]       ::= any char
+
+# BNF parsed by CodeTokenizer:
+[template]       ::= "" | [variable] " " [template] | [value] " " [template] | [function-call] " " [template]
+[variable]       ::= "$" [valid-name]
+[value]          ::= "'" [text] "'" | '"' [text] '"'
+[function-call]  ::= [function-id] | [function-id] [parameter]
+[parameter]      ::= [variable] | [value]
+
+# BNF parsed by dedicated blocks
+[function-id]    ::= [valid-name] | [valid-name] "." [valid-name]
+[valid-name]     ::= [valid-symbol] | [valid-symbol] [valid-name]
+[valid-symbol]   ::= [letter] | [digit] | "_"
+[letter]         ::= "a" | "b" ... | "z" | "A" | "B" ... | "Z"
+[digit]          ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+```

python/semantic_kernel/template_engine/blocks/block.py

@@ -0,0 +1,24 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import ClassVar
+
+from pydantic import field_validator
+
+from semantic_kernel.kernel_pydantic import KernelBaseModel
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class Block(KernelBaseModel):
+    """A block."""
+
+    type: ClassVar[BlockTypes] = BlockTypes.UNDEFINED
+    content: str
+
+    @field_validator("content", mode="before")
+    @classmethod
+    def content_strip(cls, content: str):
+        """Strip the content of the block."""
+        return content.strip()

python/semantic_kernel/template_engine/blocks/block_types.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from enum import Enum, auto
+
+
+class BlockTypes(Enum):
+    """Block types."""
+
+    UNDEFINED = auto()
+    TEXT = auto()
+    CODE = auto()
+    VARIABLE = auto()
+    VALUE = auto()
+    FUNCTION_ID = auto()
+    NAMED_ARG = auto()

python/semantic_kernel/template_engine/blocks/code_block.py

@@ -0,0 +1,173 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from copy import copy
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import Field, field_validator, model_validator
+
+from semantic_kernel.exceptions import CodeBlockRenderException, CodeBlockTokenError
+from semantic_kernel.exceptions.kernel_exceptions import KernelFunctionNotFoundError, KernelPluginNotFoundError
+from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
+from semantic_kernel.template_engine.blocks.named_arg_block import NamedArgBlock
+from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+VALID_ARG_TYPES = [BlockTypes.VALUE, BlockTypes.VARIABLE, BlockTypes.NAMED_ARG]
+
+
+class CodeBlock(Block):
+    """Create a code block.
+
+    A code block is a block that usually contains functions to be executed by the kernel.
+    It consists of a list of tokens that can be either a function_id, value, a variable or a named argument.
+
+    If the first token is not a function_id but a variable or value, the rest of the tokens will be ignored.
+    Only the first argument for the function can be a variable or value, the rest of the tokens have be named arguments.
+
+    Args:
+        content: The content of the code block.
+        tokens: The list of tokens that compose the code block, if empty, will be created by the CodeTokenizer.
+
+    Raises:
+        CodeBlockTokenError: If the content does not contain at least one token.
+        CodeBlockTokenError: If the first token is a named argument.
+        CodeBlockTokenError: If the second token is not a value or variable.
+        CodeBlockTokenError: If a token is not a named argument after the second token.
+        CodeBlockRenderError: If the plugin collection is not set in the kernel.
+        CodeBlockRenderError: If the function is not found in the plugin collection.
+        CodeBlockRenderError: If the function does not take any arguments, but it is being
+            called in the template with arguments.
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.CODE
+    tokens: list[Block] = Field(default_factory=list)
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content of the code block and tokenize it.
+
+        If tokens are already present, skip the tokenizing.
+        """
+        if isinstance(fields, Block) or "tokens" in fields:
+            return fields
+        content = fields.get("content", "").strip()
+        fields["tokens"] = CodeTokenizer.tokenize(content)
+        return fields
+
+    @field_validator("tokens", mode="after")
+    def check_tokens(cls, tokens: list[Block]) -> list[Block]:
+        """Check the tokens in the list.
+
+        If the first token is a value or variable, the rest of the tokens will be ignored.
+        If the first token is a function_id, then the next token can be a value,
+            variable or named_arg, the rest have to be named_args.
+
+        Raises:
+            CodeBlockTokenError: If the content does not contain at least one token.
+            CodeBlockTokenError: If the first token is a named argument.
+            CodeBlockTokenError: If the second token is not a value or variable.
+            CodeBlockTokenError: If a token is not a named argument after the second token.
+        """
+        if not tokens:
+            raise CodeBlockTokenError("The content should contain at least one token.")
+        for index, token in enumerate(tokens):
+            if index == 0 and token.type == BlockTypes.NAMED_ARG:
+                raise CodeBlockTokenError(
+                    f"The first token needs to be a function_id, value or variable, got: {token.type}"
+                )
+            if index == 0 and token.type in [BlockTypes.VALUE, BlockTypes.VARIABLE]:
+                if len(tokens) > 1:
+                    logger.warning(
+                        "The first token is a value or variable, but there are more tokens in the content, \
+these will be ignored."
+                    )
+                return [token]
+            if index == 1 and token.type not in VALID_ARG_TYPES:
+                raise CodeBlockTokenError(
+                    f"Unexpected type for the second token type, should be variable, value or named_arg: {token.type}"
+                )
+            if index > 1 and token.type != BlockTypes.NAMED_ARG:
+                raise CodeBlockTokenError(
+                    f"Every argument for the function after the first has to be a named arg, instead: {token.type}"
+                )
+        return tokens
+
+    async def render_code(self, kernel: "Kernel", arguments: "KernelArguments") -> str:
+        """Render the code block.
+
+        If the first token is a function_id, it will call the function from the plugin collection.
+        Otherwise, it is a value or variable and those are then rendered directly.
+        """
+        logger.debug(f"Rendering code: `{self.content}`")
+        if isinstance(self.tokens[0], FunctionIdBlock):
+            return await self._render_function_call(kernel, arguments)
+        # validated that if the first token is not a function_id, it is a value or variable
+        return self.tokens[0].render(kernel, arguments)  # type: ignore
+
+    async def _render_function_call(self, kernel: "Kernel", arguments: "KernelArguments"):
+        if not isinstance(self.tokens[0], FunctionIdBlock):
+            raise CodeBlockRenderException("The first token should be a function_id")
+        function_block: FunctionIdBlock = self.tokens[0]
+        try:
+            function = kernel.get_function(function_block.plugin_name, function_block.function_name)
+        except (KernelFunctionNotFoundError, KernelPluginNotFoundError) as exc:
+            error_msg = f"Function `{function_block.content}` not found"
+            logger.error(error_msg)
+            raise CodeBlockRenderException(error_msg) from exc
+
+        arguments_clone = copy(arguments)
+        if len(self.tokens) > 1:
+            arguments_clone = self._enrich_function_arguments(kernel, arguments_clone, function.metadata)
+        try:
+            result = await function.invoke(kernel, arguments_clone)
+        except Exception as exc:
+            error_msg = f"Error invoking function `{function_block.content}`"
+            logger.error(error_msg)
+            raise CodeBlockRenderException(error_msg) from exc
+        return str(result) if result else ""
+
+    def _enrich_function_arguments(
+        self,
+        kernel: "Kernel",
+        arguments: "KernelArguments",
+        function_metadata: KernelFunctionMetadata,
+    ) -> "KernelArguments":
+        if not function_metadata.parameters:
+            raise CodeBlockRenderException(
+                f"Function {function_metadata.plugin_name}.{function_metadata.name} does not take any arguments "
+                f"but it is being called in the template with {len(self.tokens) - 1} arguments."
+            )
+        for index, token in enumerate(self.tokens[1:], start=1):
+            logger.debug(f"Parsing variable/value: `{self.tokens[1].content}`")
+
+            # For VarBlock, get the raw value to preserve the original type
+            # For other blocks (ValBlock, NamedArgBlock), render to string as usual
+            from semantic_kernel.template_engine.blocks.var_block import VarBlock
+
+            if isinstance(token, VarBlock):
+                rendered_value = token.get_value(arguments)
+            elif isinstance(token, NamedArgBlock):
+                # NamedArgBlock may contain a VarBlock, so check for that
+                if token.variable:
+                    rendered_value = token.variable.get_value(arguments)
+                else:
+                    rendered_value = token.render(kernel, arguments)
+            else:
+                rendered_value = token.render(kernel, arguments)  # type: ignore
+
+            if not isinstance(token, NamedArgBlock) and index == 1:
+                arguments[function_metadata.parameters[0].name] = rendered_value
+                continue
+            arguments[token.name] = rendered_value  # type: ignore
+
+        return arguments

python/semantic_kernel/template_engine/blocks/function_id_block.py

@@ -0,0 +1,67 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import compile
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import FunctionIdBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+FUNCTION_ID_BLOCK_REGEX = r"^((?P<plugin>[0-9A-Za-z_]+)[.])?(?P<function>[0-9A-Za-z_]+)$"
+
+FUNCTION_ID_BLOCK_MATCHER = compile(FUNCTION_ID_BLOCK_REGEX)
+
+
+class FunctionIdBlock(Block):
+    """Block to represent a function id. It can be used to call a function from a plugin.
+
+    The content is parsed using a regex, that returns either a plugin and
+    function name or just a function name, depending on the content.
+
+    Anything other than that and a ValueError is raised.
+
+    Args:
+        content (str): The content of the block.
+        function_name (Optional[str], optional): The function name.
+        plugin_name (Optional[str], optional): The plugin name.
+
+    Raises:
+        ValueError: If the content does not have valid syntax.
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.FUNCTION_ID
+    function_name: str = ""
+    plugin_name: str | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> dict[str, Any]:
+        """Parse the content of the function id block and extract the plugin and function name.
+
+        If both are present in the fields, return the fields as is.
+        Otherwise, use the regex to extract the plugin and function name.
+        """
+        if isinstance(fields, dict):
+            if "plugin_name" in fields and "function_name" in fields:
+                return fields
+            content = fields.get("content", "").strip()
+            matches = FUNCTION_ID_BLOCK_MATCHER.match(content)
+            if not matches:
+                raise FunctionIdBlockSyntaxError(content=content)
+            if plugin := matches.groupdict().get("plugin"):
+                fields["plugin_name"] = plugin
+            fields["function_name"] = matches.group("function")
+        return fields
+
+    def render(self, *_: "Kernel | KernelArguments | None") -> str:
+        """Render the function id block."""
+        return self.content

python/semantic_kernel/template_engine/blocks/named_arg_block.py

@@ -0,0 +1,98 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import compile
+from typing import TYPE_CHECKING, Any, ClassVar, Optional
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import NamedArgBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.val_block import ValBlock
+from semantic_kernel.template_engine.blocks.var_block import VarBlock
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+NAMED_ARG_REGEX = r"^(?P<name>[0-9A-Za-z_]+)[=]{1}(?P<value>[${1}](?P<var_name>[0-9A-Za-z_]+)|(?P<quote>[\"'])(?P<val>.[^\"^']*)(?P=quote))$"  # noqa: E501
+
+NAMED_ARG_MATCHER = compile(NAMED_ARG_REGEX)
+
+
+class NamedArgBlock(Block):
+    """Create a named argument block.
+
+    A named arg block is used to add arguments to a function call.
+    It needs to be combined with a function_id block to be useful.
+    Inside a code block, if the first block is a function_id block,
+    the first block can be a variable or value block, anything else
+    must be a named arg block.
+
+    The value inside the NamedArgBlock can be a ValBlock or a VarBlock.
+
+    Examples:
+        {{ plugin.function arg1=$var }}
+        {{ plugin.function arg1='value' }}
+        {{ plugin.function 'value' arg2=$var }}
+        {{ plugin.function $var arg2='value' }}
+        {{ plugin_function arg1=$var1 arg2=$var2 arg3='value' }}
+
+    Args:
+        content - str : The content of the named argument block,
+            the name and value separated by an equal sign, for instance arg1=$var.
+        name - str: The name of the argument.
+        value - ValBlock: The value of the argument.
+        variable - VarBlock: The variable of the argument.
+            Either the value or variable field is used.
+
+    Raises:
+        NamedArgBlockSyntaxError: If the content does not match the named argument syntax.
+
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.NAMED_ARG
+    name: str | None = None
+    value: ValBlock | None = None
+    variable: VarBlock | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content of the named argument block and extract the name and value.
+
+        If the name and either value or variable is present the parsing is skipped.
+        Otherwise, the content is parsed using a regex to extract the name and value.
+        Those are then turned into Blocks.
+
+        Raises:
+            NamedArgBlockSyntaxError: If the content does not match the named argument syntax.
+        """
+        if isinstance(fields, Block) or ("name" in fields and ("value" in fields or "variable" in fields)):
+            return fields
+        content = fields.get("content", "").strip()
+        matches = NAMED_ARG_MATCHER.match(content)
+        if not matches:
+            raise NamedArgBlockSyntaxError(content=content)
+        matches_dict = matches.groupdict()
+        if name := matches_dict.get("name"):
+            fields["name"] = name
+        if value := matches_dict.get("value"):
+            if matches_dict.get("var_name"):
+                fields["variable"] = VarBlock(content=value, name=matches_dict["var_name"])
+            elif matches_dict.get("val"):
+                fields["value"] = ValBlock(content=value, value=matches_dict["val"], quote=matches_dict["quote"])
+        return fields
+
+    def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"] = None) -> Any:
+        """Render the named argument block."""
+        if self.value:
+            return self.value.render()
+        if arguments is None:
+            return ""
+        if self.variable:
+            return self.variable.render(kernel, arguments)
+        return None

python/semantic_kernel/template_engine/blocks/symbols.py

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft. All rights reserved.
+from enum import Enum
+
+
+class Symbols(str, Enum):
+    """Symbols used in the template engine."""
+
+    BLOCK_STARTER = "{"
+    BLOCK_ENDER = "}"
+
+    VAR_PREFIX = "$"
+
+    DBL_QUOTE = '"'
+    SGL_QUOTE = "'"
+    ESCAPE_CHAR = "\\"
+
+    SPACE = " "
+    TAB = "\t"
+    NEW_LINE = "\n"
+    CARRIAGE_RETURN = "\r"
+
+    NAMED_ARG_BLOCK_SEPARATOR = "="

python/semantic_kernel/template_engine/blocks/text_block.py

@@ -0,0 +1,59 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from typing import TYPE_CHECKING, ClassVar, Optional
+
+from pydantic import field_validator
+
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class TextBlock(Block):
+    """A block with text content."""
+
+    type: ClassVar[BlockTypes] = BlockTypes.TEXT
+
+    @field_validator("content", mode="before")
+    @classmethod
+    def content_strip(cls, content: str):
+        """Strip the content of the text block.
+
+        Overload strip method, text blocks are not stripped.
+        """
+        return content
+
+    @classmethod
+    def from_text(
+        cls,
+        text: str | None = None,
+        start_index: int | None = None,
+        stop_index: int | None = None,
+    ):
+        """Create a text block from a string."""
+        if text is None:
+            return cls(content="")
+        if start_index is not None and stop_index is not None:
+            if start_index > stop_index:
+                raise ValueError(f"start_index ({start_index}) must be less than stop_index ({stop_index})")
+
+            if start_index < 0:
+                raise ValueError(f"start_index ({start_index}) must be greater than 0")
+
+            text = text[start_index:stop_index]
+        elif start_index is not None:
+            text = text[start_index:]
+        elif stop_index is not None:
+            text = text[:stop_index]
+
+        return cls(content=text)
+
+    def render(self, *_: tuple[Optional["Kernel"], Optional["KernelArguments"]]) -> str:
+        """Render the text block."""
+        return self.content

python/semantic_kernel/template_engine/blocks/val_block.py

@@ -0,0 +1,74 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import S, compile
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import ValBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+VAL_BLOCK_REGEX = r"^(?P<quote>[\"'])(?P<value>.*)(?P=quote)$"
+
+VAL_BLOCK_MATCHER = compile(VAL_BLOCK_REGEX, flags=S)
+
+
+class ValBlock(Block):
+    """Create a value block.
+
+    A value block is used to represent a value in a template.
+    It can be used to represent any characters.
+    It needs to start and end with the same quote character,
+    can be both single or double quotes, as long as they are not mixed.
+
+    Examples:
+        'value'
+        "value"
+        'value with "quotes"'
+        "value with 'quotes'"
+
+    Args:
+        content - str : The content of the value block.
+        value - str: The value of the block.
+        quote - str: The quote used to wrap the value.
+
+    Raises:
+        ValBlockSyntaxError: If the content does not match the value block syntax.
+
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.VALUE
+    value: str | None = ""
+    quote: str | None = "'"
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content and extract the value and quote.
+
+        The parsing is based on a regex that returns the value and quote.
+        if the 'value' is already present then the parsing is skipped.
+        """
+        if isinstance(fields, Block) or "value" in fields:
+            return fields
+        content = fields.get("content", "").strip()
+        matches = VAL_BLOCK_MATCHER.match(content)
+        if not matches:
+            raise ValBlockSyntaxError(content=content)
+        if value := matches.groupdict().get("value"):
+            fields["value"] = value
+        if quote := matches.groupdict().get("quote"):
+            fields["quote"] = quote
+        return fields
+
+    def render(self, *_: "Kernel | KernelArguments | None") -> str:
+        """Render the value block."""
+        return self.value or ""

python/semantic_kernel/template_engine/blocks/var_block.py

@@ -0,0 +1,99 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+from re import compile
+from typing import TYPE_CHECKING, Any, ClassVar, Optional
+
+from pydantic import model_validator
+
+from semantic_kernel.exceptions import VarBlockRenderError, VarBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+
+if TYPE_CHECKING:
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+    from semantic_kernel.kernel import Kernel
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+VAR_BLOCK_REGEX = r"^[${1}](?P<name>[0-9A-Za-z_]+)$"
+
+VAR_BLOCK_MATCHER = compile(VAR_BLOCK_REGEX)
+
+
+class VarBlock(Block):
+    """Create a variable block.
+
+    A variable block is used to add a variable to a template.
+    It gets rendered from KernelArguments, if the variable is not found
+    a warning is logged and an empty string is returned.
+    The variable must start with $ and be followed by a valid variable name.
+    A valid variable name is a string of letters, numbers and underscores.
+
+    Examples:
+        $var
+        $test_var
+
+    Args:
+        content - str : The content of the variable block, the name of the variable.
+        name - str: The name of the variable.
+
+    Raises:
+        VarBlockSyntaxError: If the content does not match the variable syntax.
+
+    """
+
+    type: ClassVar[BlockTypes] = BlockTypes.VARIABLE
+    name: str = ""
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_content(cls, fields: Any) -> Any:
+        """Parse the content and extract the name.
+
+        The parsing is based on a regex that returns the name.
+        if the 'name' is already present then the parsing is skipped.
+        """
+        if isinstance(fields, Block) or "name" in fields:
+            return fields
+        content = fields.get("content", "").strip()
+        matches = VAR_BLOCK_MATCHER.match(content)
+        if not matches:
+            raise VarBlockSyntaxError(content=content)
+        if name := matches.groupdict().get("name"):
+            fields["name"] = name
+        return fields
+
+    def render(self, _: "Kernel", arguments: Optional["KernelArguments"] = None) -> str:
+        """Render the variable block with the given arguments.
+
+        If the variable is not found in the arguments, return an empty string.
+        """
+        if arguments is None:
+            return ""
+        value = arguments.get(self.name, None)
+        if value is None:
+            logger.warning(f"Variable `{Symbols.VAR_PREFIX}: {self.name}` not found in the KernelArguments")
+            return ""
+        try:
+            return str(value)
+        except Exception as e:
+            raise VarBlockRenderError(
+                f"Block {self.name} failed to be parsed to a string, type is {type(value)}"
+            ) from e
+
+    def get_value(self, arguments: "KernelArguments | None" = None) -> Any:
+        """Get the raw value of the variable from arguments without converting to string.
+
+        This is used when passing arguments to functions to preserve their original types.
+
+        Args:
+            arguments: The KernelArguments to get the value from.
+
+        Returns:
+            The raw value from the arguments, or None if not found.
+        """
+        if arguments is None:
+            return None
+        return arguments.get(self.name, None)

python/semantic_kernel/template_engine/code_tokenizer.py

@@ -0,0 +1,157 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+
+from semantic_kernel.exceptions import CodeBlockSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.function_id_block import FunctionIdBlock
+from semantic_kernel.template_engine.blocks.named_arg_block import NamedArgBlock
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+from semantic_kernel.template_engine.blocks.val_block import ValBlock
+from semantic_kernel.template_engine.blocks.var_block import VarBlock
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+# BNF parsed by CodeTokenizer:
+# [template]       ::= "" | [variable] " " [template]
+#                         | [value] " " [template]
+#                         | [function-call] " " [template]
+# [variable]       ::= "$" [valid-name]
+# [value]          ::= "'" [text] "'" | '"' [text] '"'
+# [function-call]  ::= [function-id] | [function-id] [parameter]
+# [parameter]      ::= [variable] | [value]
+class CodeTokenizer:
+    """Tokenize the code text into blocks."""
+
+    @staticmethod
+    def tokenize(text: str) -> list[Block]:
+        """Tokenize the code text into blocks."""
+        # Remove spaces, which are ignored anyway
+        text = text.strip() if text else ""
+        # Render None/empty to []
+        if not text:
+            return []
+        # 1 char only edge case, var and val blocks are invalid with one char, so it must be a function id block
+        if len(text) == 1:
+            return [FunctionIdBlock(content=text)]
+
+        # Track what type of token we're reading
+        current_token_type = None
+
+        # Track the content of the current token
+        current_token_content: list[str] = []
+
+        # Other state we need to track
+        text_value_delimiter = None
+        space_separator_found = False
+        skip_next_char = False
+        next_char = ""
+        blocks: list[Block] = []
+
+        for index, current_char in enumerate(text[:-1]):
+            next_char = text[index + 1]
+
+            if skip_next_char:
+                skip_next_char = False
+                continue
+
+            # First char is easy
+            if index == 0:
+                if current_char == Symbols.VAR_PREFIX:
+                    current_token_type = BlockTypes.VARIABLE
+                elif current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                    current_token_type = BlockTypes.VALUE
+                    text_value_delimiter = current_char
+                else:
+                    current_token_type = BlockTypes.FUNCTION_ID
+
+                current_token_content.append(current_char)
+                continue
+
+            # While reading values between quotes
+            if current_token_type in (BlockTypes.VALUE, BlockTypes.NAMED_ARG):
+                # If the current char is escaping the next special char we:
+                #  - skip the current char (escape char)
+                #  - add the next char (special char)
+                #  - jump to the one after (to handle "\\" properly)
+                if current_char == Symbols.ESCAPE_CHAR and next_char in (
+                    Symbols.DBL_QUOTE,
+                    Symbols.SGL_QUOTE,
+                    Symbols.ESCAPE_CHAR,
+                ):
+                    current_token_content.append(next_char)
+                    skip_next_char = True
+                    continue
+
+                current_token_content.append(current_char)
+
+                # When we reach the end of the value, we add the block
+                if current_char == text_value_delimiter:
+                    blocks.append(ValBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+                    current_token_type = None
+                    space_separator_found = False
+
+                continue
+
+            # If we're not between quotes, a space signals the end of the current token
+            # Note: there might be multiple consecutive spaces
+            if current_char in (
+                Symbols.SPACE,
+                Symbols.NEW_LINE,
+                Symbols.CARRIAGE_RETURN,
+                Symbols.TAB,
+            ):
+                if current_token_type == BlockTypes.VARIABLE:
+                    blocks.append(VarBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+                elif current_token_type == BlockTypes.FUNCTION_ID:
+                    if Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content:
+                        blocks.append(NamedArgBlock(content="".join(current_token_content)))
+                    else:
+                        blocks.append(FunctionIdBlock(content="".join(current_token_content)))
+                    current_token_content.clear()
+
+                space_separator_found = True
+                current_token_type = None
+
+                continue
+
+            # If we're not inside a quoted value, and we're not processing a space
+            current_token_content.append(current_char)
+
+            if current_token_type is None:
+                if not space_separator_found:
+                    raise CodeBlockSyntaxError("Tokens must be separated by one space least")
+
+                if current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                    # A quoted value starts here
+                    current_token_type = BlockTypes.VALUE
+                    text_value_delimiter = current_char
+                elif current_char == Symbols.VAR_PREFIX:
+                    # A variable starts here
+                    current_token_type = BlockTypes.VARIABLE
+                else:
+                    # A function id starts here
+                    current_token_type = BlockTypes.FUNCTION_ID
+
+        # end of main for loop
+
+        # Capture last token
+        current_token_content.append(next_char)
+
+        if current_token_type == BlockTypes.VALUE:
+            blocks.append(ValBlock(content="".join(current_token_content)))
+        elif current_token_type == BlockTypes.VARIABLE:
+            blocks.append(VarBlock(content="".join(current_token_content)))
+        elif current_token_type == BlockTypes.FUNCTION_ID:
+            if Symbols.NAMED_ARG_BLOCK_SEPARATOR.value in current_token_content:
+                blocks.append(NamedArgBlock(content="".join(current_token_content)))
+            else:
+                blocks.append(FunctionIdBlock(content="".join(current_token_content)))
+        else:
+            raise CodeBlockSyntaxError("Tokens must be separated by one space least")
+
+        return blocks

python/semantic_kernel/template_engine/protocols/code_renderer.py

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from semantic_kernel import Kernel
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+
+
+@runtime_checkable
+class CodeRenderer(Protocol):
+    """Protocol for dynamic code blocks that need async IO to be rendered."""
+
+    @abstractmethod
+    async def render_code(self, kernel: "Kernel", arguments: "KernelArguments") -> str:
+        """Render the block using the given context.
+
+        :param context: kernel execution context
+        :return: Rendered content
+        """

python/semantic_kernel/template_engine/protocols/text_renderer.py

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Optional, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from semantic_kernel import Kernel
+    from semantic_kernel.functions.kernel_arguments import KernelArguments
+
+
+@runtime_checkable
+class TextRenderer(Protocol):
+    """Protocol for static (text) blocks that don't need async rendering."""
+
+    @abstractmethod
+    def render(self, kernel: "Kernel", arguments: Optional["KernelArguments"] = None) -> str:
+        """Render the block using only the given variables.
+
+        :param variables: Optional variables used to render the block
+        :return: Rendered content
+        """

python/semantic_kernel/template_engine/template_tokenizer.py

@@ -0,0 +1,161 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import logging
+
+from semantic_kernel.exceptions import BlockSyntaxError, CodeBlockTokenError, TemplateSyntaxError
+from semantic_kernel.template_engine.blocks.block import Block
+from semantic_kernel.template_engine.blocks.block_types import BlockTypes
+from semantic_kernel.template_engine.blocks.code_block import CodeBlock
+from semantic_kernel.template_engine.blocks.symbols import Symbols
+from semantic_kernel.template_engine.blocks.text_block import TextBlock
+from semantic_kernel.template_engine.code_tokenizer import CodeTokenizer
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+# BNF parsed by TemplateTokenizer:
+# [template]       ::= "" | [block] | [block] [template]
+# [block]          ::= [sk-block] | [text-block]
+# [sk-block]       ::= "{{" [variable] "}}"
+#                      | "{{" [value] "}}"
+#                      | "{{" [function-call] "}}"
+# [text-block]     ::= [any-char] | [any-char] [text-block]
+# [any-char]       ::= any char
+class TemplateTokenizer:
+    """Tokenize the template text into blocks."""
+
+    @staticmethod
+    def tokenize(text: str) -> list[Block]:
+        """Tokenize the template text into blocks."""
+        code_tokenizer = CodeTokenizer()
+        # An empty block consists of 4 chars: "{{}}"
+        EMPTY_CODE_BLOCK_LENGTH = 4
+        # A block shorter than 5 chars is either empty or
+        # invalid, e.g. "{{ }}" and "{{$}}"
+        MIN_CODE_BLOCK_LENGTH = EMPTY_CODE_BLOCK_LENGTH + 1
+
+        text = text or ""
+
+        # Render None/empty to ""
+        if not text:
+            return [TextBlock.from_text("")]
+
+        # If the template is "empty" return it as a text block
+        if len(text) < MIN_CODE_BLOCK_LENGTH:
+            return [TextBlock.from_text(text)]
+
+        blocks: list[Block] = []
+        end_of_last_block = 0
+        block_start_pos = 0
+        block_start_found = False
+        inside_text_value = False
+        text_value_delimiter = None
+        skip_next_char = False
+
+        for current_char_pos, current_char in enumerate(text[:-1]):
+            next_char_pos = current_char_pos + 1
+            next_char = text[next_char_pos]
+
+            if skip_next_char:
+                skip_next_char = False
+                continue
+
+            # When "{{" is found outside a value
+            # Note: "{{ {{x}}" => ["{{ ", "{{x}}"]
+            if not inside_text_value and current_char == Symbols.BLOCK_STARTER and next_char == Symbols.BLOCK_STARTER:
+                # A block starts at the first "{"
+                block_start_pos = current_char_pos
+                block_start_found = True
+
+            if not block_start_found:
+                continue
+            # After having found "{{"
+            if inside_text_value:
+                # While inside a text value, when the end quote is found
+                # If the current char is escaping the next special char we skip
+                if current_char == Symbols.ESCAPE_CHAR and next_char in (
+                    Symbols.DBL_QUOTE,
+                    Symbols.SGL_QUOTE,
+                    Symbols.ESCAPE_CHAR,
+                ):
+                    skip_next_char = True
+                    continue
+
+                if current_char == text_value_delimiter:
+                    inside_text_value = False
+                continue
+
+            # A value starts here
+            if current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE):
+                inside_text_value = True
+                text_value_delimiter = current_char
+                continue
+            # If the block ends here
+            if current_char == Symbols.BLOCK_ENDER and next_char == Symbols.BLOCK_ENDER:
+                blocks.extend(
+                    TemplateTokenizer._extract_blocks(
+                        text, code_tokenizer, block_start_pos, end_of_last_block, next_char_pos
+                    )
+                )
+                end_of_last_block = next_char_pos + 1
+                block_start_found = False
+
+        # If there is something left after the last block, capture it as a TextBlock
+        if end_of_last_block < len(text):
+            blocks.append(TextBlock.from_text(text, end_of_last_block, len(text)))
+
+        return blocks
+
+    @staticmethod
+    def _extract_blocks(
+        text: str, code_tokenizer: CodeTokenizer, block_start_pos: int, end_of_last_block: int, next_char_pos: int
+    ) -> list[Block]:
+        """Extract the blocks from the found code.
+
+        If there is text before the current block, create a TextBlock from that.
+
+        If the block is empty, return a TextBlock with the delimiters.
+
+        If the block is not empty, tokenize it and return the result.
+        If there is only a variable or value in the code block,
+        return just that, instead of the CodeBlock.
+        """
+        new_blocks: list[Block] = []
+        if block_start_pos > end_of_last_block:
+            new_blocks.append(
+                TextBlock.from_text(
+                    text,
+                    end_of_last_block,
+                    block_start_pos,
+                )
+            )
+
+        content_with_delimiters = text[block_start_pos : next_char_pos + 1]
+        content_without_delimiters = content_with_delimiters[2:-2].strip()
+
+        if len(content_without_delimiters) == 0:
+            # If what is left is empty (only {{}}), consider the raw block
+            # a TextBlock
+            new_blocks.append(TextBlock.from_text(content_with_delimiters))
+            return new_blocks
+
+        try:
+            code_blocks = code_tokenizer.tokenize(content_without_delimiters)
+        except BlockSyntaxError as e:
+            msg = f"Failed to tokenize code block: {content_without_delimiters}. {e}"
+            logger.warning(msg)
+            raise TemplateSyntaxError(msg) from e
+
+        if code_blocks[0].type in (
+            BlockTypes.VALUE,
+            BlockTypes.VARIABLE,
+        ):
+            new_blocks.append(code_blocks[0])
+            return new_blocks
+        try:
+            new_blocks.append(CodeBlock(content=content_without_delimiters, tokens=code_blocks))
+            return new_blocks
+        except CodeBlockTokenError as e:
+            msg = f"Failed to tokenize code block: {content_without_delimiters}. {e}"
+            logger.warning(msg)
+            raise TemplateSyntaxError(msg) from e