chore: import upstream snapshot with attribution

pydoc/README.md

@@ -0,0 +1,15 @@
+# :ledger: Looking for the docs?
+
+You can find Haystack's documentation at https://docs.haystack.deepset.ai/.
+
+# API Reference
+
+We use [haystack-pydoc-tools](https://github.com/deepset-ai/haystack-pydoc-tools) to create Markdown files from the docstrings in our code. There is a [Github workflow](https://github.com/deepset-ai/haystack/blob/main/.github/workflows/docusaurus_sync.yml) that regenerates the API reference when code changes.
+
+If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how haystack-pydoc-tools will generate the page and commit it to main.
+
+All the updates to API reference live in unstable docs version and are promoted to stable docs version when a new version is released.
+
+### Configuration
+
+For configuration details, see the [haystack-pydoc-tools documentation](https://github.com/deepset-ai/haystack-pydoc-tools/blob/main/README.md#configuration) or inspect the existing `.yml` files in `pydoc`.

pydoc/agents_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/agents]
+    modules: ["agent", "state/state"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Agents
+  id: agents-api
+  description: Tool-using agents with provider-agnostic chat model support.
+  filename: agents_api.md

pydoc/builders_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/builders]
+    modules: ["answer_builder", "prompt_builder", "chat_prompt_builder"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Builders
+  id: builders-api
+  description: Extract the output of a Generator to an Answer format, and build prompts.
+  filename: builders_api.md

pydoc/caching_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/components/caching]
+    modules: ["cache_checker"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Caching
+  id: caching-api
+  description: Checks if any document coming from the given URL is already present
+    in the store.
+  filename: cachings_api.md

pydoc/converters_api.yml

@@ -0,0 +1,14 @@
+loaders:
+  - search_path: [../haystack/components/converters]
+    modules: ["csv", "docx", "file_to_file_content", "html", "json", "markdown", "msg", "multi_file_converter",
+      "output_adapter", "pdfminer", "pptx", "pypdf", "txt", "xlsx",
+      "image/document_to_image", "image/file_to_document", "image/file_to_image", "image/pdf_to_image"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Converters
+  id: converters-api
+  description: Various converters to transform data from one format to another.
+  filename: converters_api.md

pydoc/data_classes_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/dataclasses]
+    modules: ["answer", "breakpoints", "byte_stream", "chat_message", "document",
+      "file_content", "image_content", "skill_info", "sparse_embedding", "streaming_chunk"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Data Classes
+  id: data-classes-api
+  description: Core classes that carry data through the system.
+  filename: data_classes_api.md

pydoc/document_stores_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/document_stores/in_memory]
+    modules: ["document_store"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Document Stores
+  id: document-stores-api
+  description: Stores your texts and meta data and provides them to the Retriever
+    at query time.
+  filename: document_stores_api.md

pydoc/document_writers_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/writers]
+    modules: ["document_writer"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Document Writers
+  id: document-writers-api
+  description: Writes Documents to a DocumentStore.
+  filename: document_writers_api.md

pydoc/embedders_api.yml

@@ -0,0 +1,20 @@
+loaders:
+  - search_path: [../haystack/components/embedders]
+    modules:
+      [
+        "azure_document_embedder",
+        "azure_text_embedder",
+        "mock_document_embedder",
+        "mock_text_embedder",
+        "openai_document_embedder",
+        "openai_text_embedder",
+      ]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Embedders
+  id: embedders-api
+  description: Transforms queries into vectors to look for similar or relevant Documents.
+  filename: embedders_api.md

pydoc/evaluation_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/evaluation]
+    modules: ["eval_run_result"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Evaluation
+  id: evaluation-api
+  description: Represents the results of evaluation.
+  filename: evaluation_api.md

pydoc/evaluators_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/components/evaluators]
+    modules: ["answer_exact_match", "context_relevance", "document_map", "document_mrr",
+      "document_ndcg", "document_recall", "faithfulness", "llm_evaluator", "sas_evaluator"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Evaluators
+  id: evaluators-api
+  description: Evaluate your pipelines or individual components.
+  filename: evaluators_api.md

pydoc/extractors_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/components/extractors]
+    modules: ["llm_metadata_extractor", "image/llm_document_content_extractor",
+      "regex_text_extractor"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Extractors
+  id: extractors-api
+  description: Components to extract specific elements from textual data.
+  filename: extractors_api.md

pydoc/fetchers_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/components/fetchers]
+    modules: ["link_content"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Fetchers
+  id: fetchers-api
+  description: Fetches content from a list of URLs and returns a list of extracted
+    content streams.
+  filename: fetchers_api.md

pydoc/generators_api.yml

@@ -0,0 +1,23 @@
+loaders:
+  - search_path: [../haystack/components/generators]
+    modules:
+      [
+        "chat/azure",
+        "chat/azure_responses",
+        "chat/fallback",
+        "chat/llm",
+        "chat/mock",
+        "chat/openai",
+        "chat/openai_responses",
+        "openai_image_generator",
+        "utils",
+      ]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Generators
+  id: generators-api
+  description: Enables text generation using LLMs.
+  filename: generators_api.md

pydoc/hooks_api.yml

@@ -0,0 +1,14 @@
+loaders:
+  - search_path: [../haystack/hooks]
+    modules: ["protocol", "from_function", "tool_result_offloading/hooks", "tool_result_offloading/policies",
+      "tool_result_offloading/stores", "tool_result_offloading/types/protocol"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Hooks
+  id: hooks-api
+  description: Hooks that run at points in the Agent's run loop and influence it by mutating State, including
+    built-in tool result offloading.
+  filename: hooks_api.md

pydoc/human_in_the_loop_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/human_in_the_loop]
+    modules: ["dataclasses", "hooks", "policies", "strategies", "user_interfaces"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Human-in-the-Loop
+  id: human-in-the-loop-api
+  description: Abstractions for integrating human feedback and interaction into Agent
+    workflows.
+  filename: human_in_the_loop_api.md

pydoc/image_converters_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/converters/image]
+    modules: ["document_to_image", "file_to_document", "file_to_image", "pdf_to_image"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Image Converters
+  id: image-converters-api
+  description: Various converters to transform image data from one format to another.
+  filename: image_converters_api.md

pydoc/joiners_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/joiners]
+    modules: ["answer_joiner", "branch", "document_joiner", "list_joiner", "string_joiner"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Joiners
+  id: joiners-api
+  description: Components that join list of different objects
+  filename: joiners_api.md

pydoc/pipeline_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/core/pipeline]
+    modules: ["pipeline"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Pipeline
+  id: pipeline-api
+  description: Arranges components and integrations in flow.
+  filename: pipeline_api.md

pydoc/preprocessors_api.yml

@@ -0,0 +1,15 @@
+loaders:
+  - search_path: [../haystack/components/preprocessors]
+    modules: ["csv_document_cleaner", "csv_document_splitter", "document_cleaner",
+      "document_preprocessor", "document_splitter", "embedding_based_document_splitter",
+      "hierarchical_document_splitter", "markdown_header_splitter", "recursive_splitter",
+      "text_cleaner"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: PreProcessors
+  id: preprocessors-api
+  description: Preprocess your Documents and texts. Clean, split, and more.
+  filename: preprocessors_api.md

pydoc/query_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/query]
+    modules: ["query_expander"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Query
+  id: query-api
+  description: Components for query processing and expansion.
+  filename: query_api.md

pydoc/rankers_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/rankers]
+    modules: ["llm_ranker", "lost_in_the_middle", "meta_field", "meta_field_grouping_ranker"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Rankers
+  id: rankers-api
+  description: Reorders a set of Documents based on their relevance to the query.
+  filename: rankers_api.md

pydoc/retrievers_api.yml

@@ -0,0 +1,15 @@
+loaders:
+  - search_path: [../haystack/components/retrievers]
+    modules: ["auto_merging_retriever", "filter_retriever", "in_memory/bm25_retriever",
+      "in_memory/embedding_retriever", "multi_query_embedding_retriever", "multi_query_text_retriever",
+      "multi_retriever", "text_embedding_retriever", "sentence_window_retriever"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Retrievers
+  id: retrievers-api
+  description: Sweeps through a Document Store and returns a set of candidate Documents
+    that are relevant to the query.
+  filename: retrievers_api.md

pydoc/routers_api.yml

@@ -0,0 +1,14 @@
+loaders:
+  - search_path: [../haystack/components/routers]
+    modules: ["conditional_router", "document_length_router", "document_type_router",
+      "file_type_router", "llm_messages_router", "metadata_router"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Routers
+  id: routers-api
+  description: Routers is a group of components that route queries or Documents to
+    other components that can handle them best.
+  filename: routers_api.md

pydoc/samplers_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/samplers]
+    modules: ["top_p"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Samplers
+  id: samplers-api
+  description: Filters documents based on their similarity scores using top-p sampling.
+  filename: samplers_api.md

pydoc/skill_stores_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/skill_stores]
+    modules: ["types/protocol", "file_system/skill_store"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Skill Stores
+  id: skill-stores-api
+  description: Storage layers that discover skills and serve their content on demand.
+  filename: skill_stores_api.md

pydoc/tools_api.yml

@@ -0,0 +1,13 @@
+loaders:
+  - search_path: [../haystack/tools]
+    modules: ["tool", "from_function", "component_tool", "pipeline_tool", "searchable_toolset", "toolset",
+      "skills/skill_toolset"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Tools
+  id: tools-api
+  description: Unified abstractions to represent tools across the framework.
+  filename: tools_api.md

pydoc/utils_api.yml

@@ -0,0 +1,29 @@
+loaders:
+  - search_path: [../haystack/utils]
+    modules:
+      [
+        "auth",
+        "azure",
+        "base_serialization",
+        "callable_serialization",
+        "deserialization",
+        "device",
+        "filters",
+        "http_client",
+        "jinja2_chat_extension",
+        "jinja2_extensions",
+        "jupyter",
+        "misc",
+        "requests_utils",
+        "type_serialization",
+        "url_validation",
+      ]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Utils
+  id: utils-api
+  description: Utility functions and classes used across the library.
+  filename: utils_api.md

pydoc/validators_api.yml

@@ -0,0 +1,12 @@
+loaders:
+  - search_path: [../haystack/components/validators]
+    modules: ["json_schema"]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  title: Validators
+  id: validators-api
+  description: Validators validate LLM outputs
+  filename: validators_api.md