wehub-resource-sync
115 lines
4.2 KiB
YAML
115 lines
4.2 KiB
YAML
# This YAML configuration file is used to set up and configure the Adaptive RAG template.
|
|
# It defines various components such as data sources, language models, embedders, splitters, parsers, and retrievers.
|
|
# Each section is configured to specify how the template should process and handle data for generating responses.
|
|
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
|
|
|
|
|
|
|
|
# $sources defines the data sources used to read the data which will be indexed in the RAG.
|
|
# You can learn more how to configure data sources here:
|
|
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
|
|
|
|
$sources:
|
|
# File System connector, reading data locally.
|
|
- !pw.io.fs.read
|
|
path: data
|
|
format: binary
|
|
with_metadata: true
|
|
|
|
# Uncomment to use the SharePoint connector
|
|
# - !pw.xpacks.connectors.sharepoint.read
|
|
# url: $SHAREPOINT_URL
|
|
# tenant: $SHAREPOINT_TENANT
|
|
# client_id: $SHAREPOINT_CLIENT_ID
|
|
# cert_path: sharepointcert.pem
|
|
# thumbprint: $SHAREPOINT_THUMBPRINT
|
|
# root_path: $SHAREPOINT_ROOT
|
|
# with_metadata: true
|
|
# refresh_interval: 30
|
|
|
|
# Uncomment to use the Google Drive connector
|
|
# - !pw.io.gdrive.read
|
|
# object_id: $DRIVE_ID
|
|
# service_user_credentials_file: gdrive_indexer.json
|
|
# file_name_pattern:
|
|
# - "*.pdf"
|
|
# - "*.pptx"
|
|
# object_size_limit: null
|
|
# with_metadata: true
|
|
# refresh_interval: 30
|
|
|
|
|
|
|
|
# Configures the LLM model settings for generating responses.
|
|
# The list of available Pathway LLM wrappers is available here:
|
|
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
|
|
# You can learn more about those in our documentation:
|
|
# https://pathway.com/developers/templates/rag-customization/llm-chats
|
|
|
|
$llm: !pw.xpacks.llm.llms.OpenAIChat
|
|
model: "gpt-4.1-mini"
|
|
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
|
|
max_retries: 6
|
|
cache_strategy: !pw.udfs.DefaultCache {}
|
|
temperature: 0
|
|
capacity: 8
|
|
|
|
# Specifies the embedder model for converting text into embeddings.
|
|
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
|
|
model: "text-embedding-3-small"
|
|
cache_strategy: !pw.udfs.DefaultCache {}
|
|
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
|
|
|
|
# Defines the splitter settings for dividing text into smaller chunks.
|
|
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
|
|
max_tokens: 400
|
|
|
|
# Configures the parser for processing and extracting information from documents.
|
|
$parser: !pw.xpacks.llm.parsers.DoclingParser
|
|
async_mode: "fully_async"
|
|
chunk: false
|
|
cache_strategy: !pw.udfs.DefaultCache {}
|
|
|
|
# Sets up the retriever factory for indexing and retrieving documents.
|
|
$retriever_factory: !pw.indexing.UsearchKnnFactory
|
|
reserved_space: 1000
|
|
embedder: $embedder
|
|
metric: !pw.indexing.USearchMetricKind.COS
|
|
|
|
# Manages the storage and retrieval of documents for the RAG template.
|
|
$document_store: !pw.xpacks.llm.document_store.DocumentStore
|
|
docs: $sources
|
|
parser: $parser
|
|
splitter: $splitter
|
|
retriever_factory: $retriever_factory
|
|
|
|
# Configures the question-answering component using the RAG approach.
|
|
# The component builds a RAG over an index.
|
|
# You can interact with obtained RAG using a REST API.
|
|
# You can learn more about the available operations here:
|
|
# https://pathway.com/developers/templates/rag-customization/rest-api
|
|
question_answerer: !pw.xpacks.llm.question_answering.AdaptiveRAGQuestionAnswerer
|
|
llm: $llm
|
|
indexer: $document_store
|
|
n_starting_documents: 2
|
|
factor: 2
|
|
max_iterations: 4
|
|
|
|
# Change host and port of the webserver by uncommenting these lines
|
|
# host: "0.0.0.0"
|
|
# port: 8000
|
|
|
|
# By default, caching is enabled for UDFs with cache_strategy set.
|
|
# You can disable it by uncommenting the following line.
|
|
# persistence_mode: null
|
|
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
|
|
# across restarts.
|
|
# By default, when enabled, Cache is stored in .Cache directory.
|
|
# You can customize the location by uncommenting and modifying the following lines:
|
|
# persistence_backend: !pw.persistence.Backend.filesystem
|
|
# path: ".Cache"
|
|
|
|
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
|
|
# Defaults to false, uncomment the following line if you want to set it to true
|
|
# terminate_on_error: true
|