chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,313 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import ast
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import AsyncIterable, Callable
|
||||
from inspect import getsource
|
||||
from typing import Any, ClassVar, Final, Literal
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from httpx import AsyncClient, HTTPStatusError, RequestError
|
||||
from pydantic import Field, SecretStr, ValidationError
|
||||
|
||||
from semantic_kernel.connectors._search_shared import SearchLambdaVisitor
|
||||
from semantic_kernel.data.text_search import (
|
||||
KernelSearchResults,
|
||||
SearchOptions,
|
||||
TextSearch,
|
||||
TextSearchResult,
|
||||
TSearchResult,
|
||||
)
|
||||
from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidRequestError
|
||||
from semantic_kernel.kernel_pydantic import KernelBaseModel, KernelBaseSettings
|
||||
from semantic_kernel.kernel_types import OptionalOneOrList
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental
|
||||
from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
else:
|
||||
from typing_extensions import override
|
||||
|
||||
logger: logging.Logger = logging.getLogger(__name__)
|
||||
|
||||
CUSTOM_SEARCH_URL: Final[str] = "https://www.googleapis.com/customsearch/v1"
|
||||
# For more info on this list: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list
|
||||
QUERY_PARAMETERS: Final[list[str]] = [
|
||||
# Country, Restricts search results to documents originating in a particular country.
|
||||
# You may use Boolean operators in the cr parameter's value.
|
||||
"cr",
|
||||
# Date Restrict, Restricts results to URLs based on date. Supported values include:
|
||||
# d[number]: requests results from the specified number of past days.
|
||||
# w[number]: requests results from the specified number of past weeks.
|
||||
# m[number]: requests results from the specified number of past months.
|
||||
# y[number]: requests results from the specified number of past years.
|
||||
"dateRestrict",
|
||||
# exactTerms, Identifies a phrase that all documents in the search results must contain.
|
||||
"exactTerms",
|
||||
# excludeTerms, Identifies a word or phrase that should not appear in any documents in the search results.
|
||||
"excludeTerms",
|
||||
# fileType, Restricts results to files of a specified extension. A list of file types indexable by Google
|
||||
# can be found in Search Console Help Center: https://support.google.com/webmasters/answer/35287
|
||||
"fileType",
|
||||
# filter, Controls turning on or off the duplicate content filter.
|
||||
"filter",
|
||||
# gl, Geolocation of end user. The gl parameter value is a two-letter country code. The gl parameter boosts search
|
||||
# results whose country of origin matches the parameter value.
|
||||
# See the Country Codes page for a list of valid values.
|
||||
"gl",
|
||||
# highRange, Specifies the ending value for a search range.
|
||||
"highRange",
|
||||
# hl, Sets the user interface language.
|
||||
"hl",
|
||||
# linkSite, Specifies that all search results should contain a link to a particular URL.
|
||||
"linkSite",
|
||||
# Language of the result. Restricts the search to documents written in a particular language (e.g., lr=lang_ja).
|
||||
"lr",
|
||||
# or Terms, Provides additional search terms to check for in a document, where each document in the search results
|
||||
# must contain at least one of the additional search terms.
|
||||
"orTerms",
|
||||
# rights, Filters based on licensing. Supported values include:
|
||||
# cc_publicdomain, cc_attribute, cc_sharealike, cc_noncommercial, cc_nonderived
|
||||
"rights",
|
||||
# siteSearch, Specifies all search results should be pages from a given site.
|
||||
"siteSearch",
|
||||
# siteSearchFilter, Controls whether to include or exclude results from the site named in the siteSearch parameter.
|
||||
"siteSearchFilter",
|
||||
]
|
||||
|
||||
|
||||
@experimental
|
||||
class GoogleSearchResult(KernelBaseModel):
|
||||
"""A Google web page."""
|
||||
|
||||
kind: str = ""
|
||||
title: str = ""
|
||||
html_title: str = Field(default="", alias="htmlTitle")
|
||||
link: str = ""
|
||||
display_link: str = Field(default="", alias="displayLink")
|
||||
snippet: str = ""
|
||||
html_snippet: str = Field(default="", alias="htmlSnippet")
|
||||
cache_id: str = Field(default="", alias="cacheId")
|
||||
formatted_url: str = Field(default="", alias="formattedUrl")
|
||||
html_formatted_url: str = Field(default="", alias="htmlFormattedUrl")
|
||||
pagemap: dict[str, Any] = Field(default_factory=dict)
|
||||
mime: str = ""
|
||||
file_format: str = Field(default="", alias="fileFormat")
|
||||
image: dict[str, Any] = Field(default_factory=dict)
|
||||
labels: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
@experimental
|
||||
class GoogleSearchInformation(KernelBaseModel):
|
||||
"""Information about the search."""
|
||||
|
||||
search_time: float = Field(alias="searchTime")
|
||||
formatted_search_time: str = Field(alias="formattedSearchTime")
|
||||
total_results: str = Field(alias="totalResults")
|
||||
formatted_total_results: str = Field(alias="formattedTotalResults")
|
||||
|
||||
|
||||
@experimental
|
||||
class GoogleSearchResponse(KernelBaseModel):
|
||||
"""The response from a Google search."""
|
||||
|
||||
kind: str = ""
|
||||
url: dict[str, str] = Field(default_factory=dict)
|
||||
queries: dict[str, list[dict[str, str | int]]] = Field(default_factory=dict)
|
||||
context: dict[str, Any] = Field(default_factory=dict)
|
||||
search_information: GoogleSearchInformation | None = None
|
||||
spelling: dict[str, Any] = Field(default_factory=dict)
|
||||
promotions: list[dict[str, Any]] = Field(default_factory=list)
|
||||
items: list[GoogleSearchResult] | None = Field(None)
|
||||
|
||||
|
||||
class GoogleSearchSettings(KernelBaseSettings):
|
||||
"""Google Search Connector settings.
|
||||
|
||||
The settings are first loaded from environment variables with the prefix 'GOOGLE_'. If the
|
||||
environment variables are not found, the settings can be loaded from a .env file with the
|
||||
encoding 'utf-8'. If the settings are not found in the .env file, the settings are ignored;
|
||||
however, validation will fail alerting that the settings are missing.
|
||||
|
||||
Required settings for prefix 'GOOGLE_SEARCH_' are:
|
||||
- api_key: SecretStr - The Google Search API key (Env var GOOGLE_SEARCH_API_KEY)
|
||||
|
||||
Optional settings for prefix 'GOOGLE_SEARCH_' are:
|
||||
- engine_id: str - The Google search engine ID (Env var GOOGLE_SEARCH_ENGINE_ID)
|
||||
- env_file_path: str | None - if provided, the .env settings are read from this file path location
|
||||
- env_file_encoding: str - if provided, the .env file encoding used. Defaults to "utf-8".
|
||||
"""
|
||||
|
||||
env_prefix: ClassVar[str] = "GOOGLE_SEARCH_"
|
||||
|
||||
api_key: SecretStr
|
||||
engine_id: str | None = None
|
||||
|
||||
|
||||
@experimental
|
||||
class GoogleSearch(KernelBaseModel, TextSearch):
|
||||
"""A search engine connector that uses the Google Search API to perform a web search."""
|
||||
|
||||
settings: GoogleSearchSettings
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
search_engine_id: str | None = None,
|
||||
env_file_path: str | None = None,
|
||||
env_file_encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Initializes a new instance of the Google Search class.
|
||||
|
||||
Args:
|
||||
api_key: The Google Search API key. If provided, will override
|
||||
the value in the env vars or .env file.
|
||||
search_engine_id: The Google search engine ID.
|
||||
If provided, will override the value in the env vars or .env file.
|
||||
env_file_path: The optional path to the .env file. If provided,
|
||||
the settings are read from this file path location.
|
||||
env_file_encoding: The optional encoding of the .env file.
|
||||
"""
|
||||
try:
|
||||
settings = GoogleSearchSettings(
|
||||
api_key=api_key,
|
||||
engine_id=search_engine_id,
|
||||
env_file_path=env_file_path,
|
||||
env_file_encoding=env_file_encoding,
|
||||
)
|
||||
except ValidationError as ex:
|
||||
raise ServiceInitializationError("Failed to create Google settings.") from ex
|
||||
|
||||
super().__init__(settings=settings) # type: ignore[call-arg]
|
||||
|
||||
@override
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
output_type: type[str] | type[TSearchResult] | Literal["Any"] = str,
|
||||
*,
|
||||
filter: OptionalOneOrList[Callable | str] = None,
|
||||
skip: int = 0,
|
||||
top: int = 5,
|
||||
include_total_count: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> "KernelSearchResults[TSearchResult]":
|
||||
options = SearchOptions(filter=filter, skip=skip, top=top, include_total_count=include_total_count, **kwargs)
|
||||
results = await self._inner_search(query=query, options=options)
|
||||
return KernelSearchResults(
|
||||
results=self._get_result_strings(results)
|
||||
if output_type is str
|
||||
else self._get_text_search_results(results)
|
||||
if output_type is TextSearchResult
|
||||
else self._get_google_search_results(results),
|
||||
total_count=self._get_total_count(results, options),
|
||||
metadata=self._get_metadata(results),
|
||||
)
|
||||
|
||||
async def _get_result_strings(self, response: GoogleSearchResponse) -> AsyncIterable[str]:
|
||||
if response.items is None:
|
||||
return
|
||||
for item in response.items:
|
||||
yield item.snippet or ""
|
||||
|
||||
async def _get_text_search_results(self, response: GoogleSearchResponse) -> AsyncIterable[TextSearchResult]:
|
||||
if response.items is None:
|
||||
return
|
||||
for item in response.items:
|
||||
yield TextSearchResult(
|
||||
name=item.title,
|
||||
value=item.snippet,
|
||||
link=item.link,
|
||||
)
|
||||
|
||||
async def _get_google_search_results(self, response: GoogleSearchResponse) -> AsyncIterable[GoogleSearchResult]:
|
||||
if response.items is None:
|
||||
return
|
||||
for val in response.items:
|
||||
yield val
|
||||
|
||||
def _get_metadata(self, response: GoogleSearchResponse) -> dict[str, Any]:
|
||||
return {
|
||||
"search_time": response.search_information.search_time if response.search_information else 0,
|
||||
}
|
||||
|
||||
def _get_total_count(self, response: GoogleSearchResponse, options: SearchOptions) -> int | None:
|
||||
total_results = (
|
||||
None
|
||||
if not options.include_total_count
|
||||
else response.search_information.total_results or None
|
||||
if response.search_information
|
||||
else None
|
||||
)
|
||||
if total_results is not None:
|
||||
return int(total_results)
|
||||
return None
|
||||
|
||||
def _get_options(self, options: "SearchOptions | None", **kwargs: Any) -> SearchOptions:
|
||||
if options is not None and isinstance(options, SearchOptions):
|
||||
return options
|
||||
try:
|
||||
return SearchOptions(**kwargs)
|
||||
except ValidationError:
|
||||
return SearchOptions()
|
||||
|
||||
async def _inner_search(self, query: str, options: SearchOptions) -> GoogleSearchResponse:
|
||||
self._validate_options(options)
|
||||
|
||||
logger.info(
|
||||
f"Received request for google web search with \
|
||||
params:\nnum_results: {options.top}\noffset: {options.skip}"
|
||||
)
|
||||
|
||||
full_url = f"{CUSTOM_SEARCH_URL}{self._build_query(query, options)}"
|
||||
headers = {"user_agent": SEMANTIC_KERNEL_USER_AGENT}
|
||||
try:
|
||||
async with AsyncClient(timeout=5) as client:
|
||||
response = await client.get(full_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
return GoogleSearchResponse.model_validate_json(response.text)
|
||||
except HTTPStatusError as ex:
|
||||
logger.error(f"Failed to get search results: {ex}")
|
||||
raise ServiceInvalidRequestError("Failed to get search results.") from ex
|
||||
except RequestError as ex:
|
||||
logger.error(f"Client error occurred: {ex}")
|
||||
raise ServiceInvalidRequestError("A client error occurred while getting search results.") from ex
|
||||
except Exception as ex:
|
||||
logger.error(f"An unexpected error occurred: {ex}")
|
||||
raise ServiceInvalidRequestError("An unexpected error occurred while getting search results.") from ex
|
||||
|
||||
def _validate_options(self, options: SearchOptions) -> None:
|
||||
if options.top > 10:
|
||||
raise ServiceInvalidRequestError("count value must be less than or equal to 10.")
|
||||
|
||||
def _parse_filter_lambda(self, filter_lambda: Callable | str) -> list[dict[str, str]]:
|
||||
"""Parse a string lambda or string expression into a list of {field: value} dicts using AST."""
|
||||
expr = filter_lambda if isinstance(filter_lambda, str) else getsource(filter_lambda).strip()
|
||||
tree = ast.parse(expr, mode="eval")
|
||||
node = tree.body
|
||||
visitor = SearchLambdaVisitor(valid_parameters=QUERY_PARAMETERS)
|
||||
visitor.visit(node)
|
||||
return visitor.filters
|
||||
|
||||
def _build_query(self, query: str, options: SearchOptions) -> str:
|
||||
params = {
|
||||
"key": self.settings.api_key.get_secret_value(),
|
||||
"cx": self.settings.engine_id,
|
||||
"num": options.top,
|
||||
"start": options.skip,
|
||||
}
|
||||
# parse the filter lambdas to query parameters
|
||||
if options.filter:
|
||||
filters = options.filter
|
||||
if not isinstance(filters, list):
|
||||
filters = [filters]
|
||||
for f in filters:
|
||||
try:
|
||||
for d in self._parse_filter_lambda(f):
|
||||
params.update(d)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to parse filter lambda: {f}, ignoring this filter. Error: {exc}")
|
||||
continue
|
||||
return f"?q={quote_plus(query)}&{'&'.join(f'{k}={v}' for k, v in params.items())}"
|
||||
Reference in New Issue
Block a user