site_name: Ragas site_description: Evaluation framework for your AI Application site_url: !ENV READTHEDOCS_CANONICAL_URL repo_name: vibrantlabsai/ragas repo_url: https://github.com/vibrantlabsai/ragas watch: - src # Navigation nav: - "": index.md - 🚀 Get Started: - getstarted/index.md - Installation: getstarted/install.md - Quick Start: getstarted/quickstart.md - Tutorials: - Evaluate a prompt: tutorials/prompt.md - Evaluate a simple RAG system: tutorials/rag.md - Evaluate an AI Workflow: tutorials/workflow.md - Evaluate an AI Agent: tutorials/agent.md - 📚 Core Concepts: - concepts/index.md - Experimentation: concepts/experimentation.md - Datasets: concepts/datasets.md - Metrics: - concepts/metrics/index.md - Overview: concepts/metrics/overview/index.md - Available Metrics: - concepts/metrics/available_metrics/index.md - Retrieval Augmented Generation: - Context Precision: concepts/metrics/available_metrics/context_precision.md - Context Recall: concepts/metrics/available_metrics/context_recall.md - Context Entities Recall: concepts/metrics/available_metrics/context_entities_recall.md - Noise Sensitivity: concepts/metrics/available_metrics/noise_sensitivity.md - Response Relevancy: concepts/metrics/available_metrics/answer_relevance.md - Faithfulness: concepts/metrics/available_metrics/faithfulness.md - Nvidia Metrics: - Answer Accuracy: concepts/metrics/available_metrics/nvidia_metrics/#answer-accuracy - Context Relevance: concepts/metrics/available_metrics/nvidia_metrics/#context-relevance - Response Groundedness: concepts/metrics/available_metrics/nvidia_metrics/#response-groundedness - Agents or Tool Use Cases: - concepts/metrics/available_metrics/agents.md - Topic Adherence: concepts/metrics/available_metrics/agents/#topic-adherence - Tool Call Accuracy: concepts/metrics/available_metrics/agents/#tool-call-accuracy - Tool Call F1: concepts/metrics/available_metrics/agents/#tool-call-f1 - Agent Goal Accuracy: concepts/metrics/available_metrics/agents/#agent-goal-accuracy - Natural Language Comparison: - Factual Correctness: concepts/metrics/available_metrics/factual_correctness.md - Semantic Similarity: concepts/metrics/available_metrics/semantic_similarity.md - Traditional non LLM metrics: - concepts/metrics/available_metrics/traditional.md - Non LLM String Similarity: concepts/metrics/available_metrics/traditional/#non-llm-string-similarity - BLEU Score: concepts/metrics/available_metrics/traditional/#bleu-score - CHRF Score: concepts/metrics/available_metrics/traditional/#chrf-score - ROUGE Score: concepts/metrics/available_metrics/traditional/#rouge-score - String Presence: concepts/metrics/available_metrics/traditional/#string-presence - Exact Match: concepts/metrics/available_metrics/traditional/#exact-match - SQL: - concepts/metrics/available_metrics/sql.md - Execution based Datacompy Score: concepts/metrics/available_metrics/sql/#execution-based-metrics - SQL Query Equivalence: concepts/metrics/available_metrics/sql/#sql-query-semantic-equivalence - General Purpose: - concepts/metrics/available_metrics/general_purpose.md - Aspect Critic: concepts/metrics/available_metrics/general_purpose/#aspect-critic - Simple Criteria Scoring: concepts/metrics/available_metrics/general_purpose/#simple-criteria-scoring - Rubrics Based Scoring: concepts/metrics/available_metrics/general_purpose/#rubrics-based-criteria-scoring - Instance Specific Rubrics Scoring: concepts/metrics/available_metrics/general_purpose/#instance-specific-rubrics-criteria-scoring - Other Tasks: - Summarization: concepts/metrics/available_metrics/summarization_score.md - Test Data Generation: - concepts/test_data_generation/index.md - RAG: - concepts/test_data_generation/rag.md - KG Building: concepts/test_data_generation/rag/#knowledge-graph-creation - Scenario Generation: concepts/test_data_generation/rag/#scenario-generation - Agents or tool use: - concepts/test_data_generation/agents.md - Components: - concepts/components/index.md - General: - Prompt: concepts/components/prompt.md - Evaluation: - Evaluation Sample: concepts/components/eval_sample.md - Evaluation Dataset: concepts/components/eval_dataset.md - 🛠️ How-to Guides: - howtos/index.md - Customizations: - howtos/customizations/index.md - General: - Customise models: howtos/customizations/customize_models.md - Run Config: howtos/customizations/run_config.md - Caching: howtos/customizations/_caching.md - Cancelling Tasks: howtos/customizations/cancellation.md - LLM Adapters: howtos/llm-adapters.md - Metrics: - Modify Prompts: howtos/customizations/metrics/modifying-prompts-metrics.md - Adapt Metrics to Languages: howtos/customizations/metrics/_metrics_language_adaptation.md - Train and Align Metrics: howtos/customizations/metrics/train_your_own_metric.md - Testset Generation: - Non-English Testset Generation: howtos/customizations/testgenerator/_language_adaptation.md - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md - Custom Single-hop Query: howtos/customizations/testgenerator/_testgen-custom-single-hop.md - Custom Multi-hop Query: howtos/customizations/testgenerator/_testgen-customisation.md - Using Pre-chunked Data: howtos/customizations/testgenerator/prechunked_data.md - Optimizers: - DSPy Optimizer: howtos/customizations/optimizers/index.md - Applications: - howtos/applications/index.md - Prompt Evaluation: - Iterate and Improve Prompts: howtos/applications/iterate_prompt.md - Systematic Prompt Optimization: howtos/applications/prompt_optimization.md - Metrics: - Cost Analysis: howtos/applications/_cost.md - Evaluating Multi-turn Conversations: howtos/applications/evaluating_multi_turn_conversations.md - Evaluations with Vertex AI models: howtos/applications/vertexai_x_ragas.md - Testset Generation: - Single-hop Query Testset: howtos/applications/singlehop_testset_gen.md - Benchmarking: - Evaluate a New LLM: howtos/applications/benchmark_llm.md - Agent Evaluation: - Evaluate a Text-to-SQL Agent: howtos/applications/text2sql.md - Align an LLM as a Judge: howtos/applications/align-llm-as-judge.md - RAG Evaluation: - Evaluate and Improve a RAG App: howtos/applications/evaluate-and-improve-rag.md - CLI: - howtos/cli/index.md - RAG Evaluation: howtos/cli/rag_eval.md - Improve RAG: howtos/cli/improve_rag.md - Integrations: - howtos/integrations/index.md - Observability: - Arize: howtos/integrations/_arize.md - LangSmith: howtos/integrations/langsmith.md - LLM Providers: - Amazon Bedrock: howtos/integrations/amazon_bedrock.md - Google Gemini: howtos/integrations/gemini.md - OCI Gen AI: howtos/integrations/oci_genai.md - Frameworks: - AG-UI: howtos/integrations/ag_ui.md - Griptape: howtos/integrations/griptape.md - Haystack: howtos/integrations/haystack.md - LangChain: howtos/integrations/langchain.md - LangGraph: howtos/integrations/_langgraph_agent_evaluation.md - LlamaIndex: howtos/integrations/_llamaindex.md - LlamaIndex Agents: howtos/integrations/llamaindex_agents.md - LlamaStack: howtos/integrations/llama_stack.md - R2R: howtos/integrations/r2r.md - Swarm: howtos/integrations/swarm_agent_evaluation.md - Migrations: - From v0.1 to v0.2: howtos/migrations/migrate_from_v01_to_v02.md - From v0.3 to v0.4: howtos/migrations/migrate_from_v03_to_v04.md - 📖 References: - references/index.md - Core: - Prompt: references/prompt.md - LLMs: references/llms.md - Embeddings: references/embeddings.md - Tokenizers: references/tokenizers.md - RunConfig: references/run_config.md - Executor: references/executor.md - Cache: references/cache.md - Optimizers: references/optimizers.md - Evaluation: - Schemas: references/evaluation_schema.md - Metrics: references/metrics.md - evaluate(): references/evaluate.md - aevaluate(): references/aevaluate.md - Testset Generation: - Schemas: references/testset_schema.md - Graph: references/graph.md - Transforms: references/transforms.md - Synthesizers: references/synthesizers.md - Generation: references/generate.md - Integrations: references/integrations.md - ❤️ Community: community/index.md # https://www.mkdocs.org/user-guide/configuration/#validation validation: omitted_files: warn absolute_links: warn unrecognized_links: warn # Material-Docs Theme theme: name: material custom_dir: docs/extra/overrides logo: _static/imgs/ragas-logo.png favicon: _static/favicon.ico features: - announce.dismiss - content.tabs.link - content.code.annotate - content.code.copy - announce.dismiss - navigation.tabs - navigation.path - navigation.instant - navigation.instant.prefetch - navigation.instant.preview - navigation.sections - navigation.top - navigation.tracking - navigation.indexes - navigation.footer - search.suggest - search.highlight palette: - media: "(prefers-color-scheme)" toggle: icon: material/brightness-auto name: Switch to light mode - media: "(prefers-color-scheme: light)" scheme: default primary: "#bd8526" accent: "#bd8526" toggle: icon: material/brightness-7 name: Switch to dark mode - media: "(prefers-color-scheme: dark)" scheme: slate primary: "#bd8526" accent: "#bd8526" toggle: icon: material/brightness-4 name: Switch to system preference markdown_extensions: - pymdownx.highlight: anchor_linenums: true line_spans: __span pygments_lang_class: true - admonition - pymdownx.inlinehilite - pymdownx.details - pymdownx.tabbed: alternate_style: true - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg - attr_list - md_in_html - pymdownx.arithmatex: generic: true - pymdownx.superfences: custom_fences: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.snippets: base_path: ["./docs/extra/components/"] # Extra CSS extra_css: - extra/ragas-modern.css # Plugins extra: version: provider: mike analytics: provider: google property: !ENV GOOGLE_ANALYTICS_KEY plugins: - search - social: enabled: !ENV [MKDOCS_CI, true] - copy-to-llm: repo_url: "https://raw.githubusercontent.com/vibrantlabsai/ragas/main/docs" buttons: copy_page: true copy_markdown_link: false # Disabled until plugin bug is fixed view_as_markdown: false # Disabled until plugin bug is fixed open_in_chatgpt: true open_in_claude: true - llmstxt: markdown_description: | Ragas is an open-source evaluation framework for LLM applications including RAG pipelines, AI agents, and workflows. It provides objective metrics for evaluation, test data generation capabilities, and integrations with popular LLM frameworks like LangChain and LlamaIndex. full_output: llms-full.txt sections: Getting Started: - getstarted/*.md Tutorials: - tutorials/*.md Core Concepts: - concepts/*.md - concepts/components/*.md Metrics: - concepts/metrics/overview/*.md - concepts/metrics/available_metrics/*.md Test Data Generation: - concepts/test_data_generation/*.md Customization Guides: - howtos/customizations/*.md - howtos/customizations/metrics/*.md - howtos/customizations/testgenerator/*.md - howtos/customizations/optimizers/*.md Application Guides: - howtos/applications/*.md CLI: - howtos/cli/*.md Integrations: - howtos/integrations/*.md API Reference: - references/*.md - git-revision-date-localized: enabled: !ENV [MKDOCS_CI, false] enable_creation_date: true - git-committers: enabled: !ENV [MKDOCS_CI, false] repository: vibrantlabsai/ragas branch: main - mkdocstrings: handlers: python: paths: [src] options: docstring_style: numpy members_order: source separate_signature: true filters: ["!^_"] docstring_options: ignore_init_summary: true merge_init_into_class: true show_signature_annotations: true signature_crossrefs: true - glightbox # - gen-files: # scripts: # - docs/ipynb_to_md.py extra_javascript: - _static/js/mathjax.js - _static/js/header_border.js - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js - _static/js/toggle.js - https://cdn.octolane.com/tag.js?pk=c7c9b2b863bf7eaf4e2a # octolane for analytics - _static/js/commonroom.js # commonroom analytics