chore: import upstream snapshot with attribution

2026-07-13 13:35:10 +08:00
commit e4f55014ae
695 changed files with 121471 additions and 0 deletions
@@ -0,0 +1,53 @@
+"""
+Text-to-SQL Agent Evaluation Framework
+
+This module provides a comprehensive framework for evaluating Text-to-SQL agents using Ragas.
+It includes dataset preparation, agent implementation, evaluation metrics, and error analysis tools.
+
+Key Components:
+- Text2SQLAgent: Core agent implementation with OpenAI integration
+- Dataset utilities for BookSQL and custom datasets
+- Database interface for SQLite query execution
+- Ragas-based evaluation framework with custom metrics
+- Error analysis and validation tools
+
+Usage:
+    import asyncio
+    from openai import AsyncOpenAI
+    from ragas_examples.text2sql import Text2SQLAgent, execute_sql, text2sql_experiment, load_dataset
+    
+    # Create and use agent
+    client = AsyncOpenAI(api_key="your-api-key")
+    agent = Text2SQLAgent(client=client, model_name="gpt-5-mini")
+    result = await agent.query("What is the total revenue?")
+    
+    # Execute SQL queries
+    success, data = execute_sql(result['sql'])
+    
+    # Run evaluation
+    async def evaluate():
+        dataset = load_dataset()
+        results = await text2sql_experiment.arun(
+            dataset,
+            name="my_evaluation",
+            model="gpt-5-mini",
+            prompt_file=None,
+        )
+        return results
+"""
+
+from .data_utils import create_sample_dataset, download_booksql_dataset
+from .db_utils import SQLiteDB, execute_sql
+from .text2sql_agent import Text2SQLAgent
+from .evals import load_dataset, text2sql_experiment, execution_accuracy
+
+__all__ = [
+    "Text2SQLAgent",
+    "execute_sql",
+    "SQLiteDB",
+    "download_booksql_dataset",
+    "create_sample_dataset",
+    "load_dataset", 
+    "text2sql_experiment",
+    "execution_accuracy",
+]
@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""
+Error Analysis Script for Text2SQL Evaluation Results
+
+Analyzes CSV files containing text2sql evaluation results and adds error analysis
+for rows where execution_accuracy is incorrect using OpenAI's GPT model.
+"""
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+import dotenv
+import pandas as pd
+from openai import OpenAI
+
+dotenv.load_dotenv("../../../.env")
+
+ERROR_TAXONOMY = [
+    "AGGR_DISTINCT_MISSING",
+    "WRONG_FILTER_COLUMN", 
+    "WRONG_SOURCE_TABLE_OR_COLUMN",
+    "EXTRA_TRANSFORMATION_OR_CONDITION",
+    "OUTPUT_COLUMN_ALIAS_MISMATCH",
+    "NULL_OR_EMPTY_RESULT",
+    "GENERIC_VALUE_MISMATCH",
+    "OTHER"
+]
+
+
+def get_error_analysis(client: OpenAI, row: Dict[str, Any]) -> Dict[str, Any]:
+    """Get error analysis from OpenAI for a single row."""
+    
+    prompt = f"""You are analyzing why a Text2SQL prediction failed. Given the following information, identify the error codes and provide a brief analysis.
+
+Available error codes:
+- AGGR_DISTINCT_MISSING: Used COUNT/SUM without DISTINCT or deduplication
+- WRONG_FILTER_COLUMN: Filtered on the wrong column 
+- WRONG_SOURCE_TABLE_OR_COLUMN: Selected metric from the wrong table/column
+- EXTRA_TRANSFORMATION_OR_CONDITION: Added ABS(), extra filters that change results
+- OUTPUT_COLUMN_ALIAS_MISMATCH: Output column names don't match
+- NULL_OR_EMPTY_RESULT: Result is None/empty due to wrong filters or source
+- GENERIC_VALUE_MISMATCH: Aggregation computed but numeric value differs for unclear reasons
+- OTHER: Fallback
+
+Query: {row['query']}
+Expected SQL: {row['expected_sql']}
+Predicted SQL: {row['predicted_sql']}
+SQL Validity: {row['sql_validity']}
+Execution Accuracy: {row['execution_accuracy']}
+Validity Reason: {row['validity_reason']}
+Accuracy Reason: {row['accuracy_reason']}
+
+Respond with JSON containing:
+- error_codes: array of applicable error codes (1 or more)
+- error_analysis: brief 1-3 sentence explanation of what went wrong"""
+
+    response = client.chat.completions.create(
+        model="gpt-5",
+        messages=[{"role": "user", "content": prompt}],
+        response_format={"type": "json_object"},
+    )
+    
+    content = response.choices[0].message.content
+    if content is None:
+        return {"error_codes": ["OTHER"], "error_analysis": "No response from model"}
+    
+    return json.loads(content)
+
+
+
+
+def analyze_errors(input_file: str, output_file: str) -> None:
+    """Analyze errors in the CSV file and add error analysis columns."""
+    
+    # Check for OpenAI API key
+    if not os.getenv("OPENAI_API_KEY"):
+        print("Error: OPENAI_API_KEY environment variable not set")
+        sys.exit(1)
+    
+    client = OpenAI()
+    
+    # Read the CSV file
+    df = pd.read_csv(input_file)
+    
+    # Initialize new columns
+    df['error_analysis'] = ''
+    df['error_codes'] = ''
+    
+    # Process rows with incorrect execution accuracy
+    incorrect_mask = df['execution_accuracy'].str.lower() == 'incorrect'
+    incorrect_rows = df[incorrect_mask]
+    
+    print(f"Found {len(incorrect_rows)} rows with incorrect execution accuracy")
+    
+    # Process rows sequentially
+    total_rows = len(incorrect_rows)
+    for i, (idx, row) in enumerate(incorrect_rows.iterrows(), 1):
+        print(f"Processing row {i}/{total_rows} (ID: {row.get('id', 'unknown')})")
+        
+        try:
+            result = get_error_analysis(client, row.to_dict())
+            df.at[idx, 'error_analysis'] = result.get('error_analysis', 'Analysis not available')
+            df.at[idx, 'error_codes'] = json.dumps(result.get('error_codes', ['OTHER']))
+            print(f"  ✓ Completed: {result.get('error_codes', ['OTHER'])}")
+        except Exception as e:
+            print(f"  ✗ Error processing row {idx}: {e}")
+            df.at[idx, 'error_analysis'] = f"Error during analysis: {str(e)}"
+            df.at[idx, 'error_codes'] = json.dumps(["OTHER"])
+    
+    # Write the output CSV
+    df.to_csv(output_file, index=False)
+    print(f"Analysis complete. Output written to: {output_file}")
+    
+    # Print error code summary
+    print("\n" + "="*50)
+    print("ERROR CODE SUMMARY")
+    print("="*50)
+    
+    error_counts = {}
+    for _, row in df[incorrect_mask].iterrows():
+        try:
+            error_codes_str = str(row['error_codes']).strip()
+            if error_codes_str and error_codes_str != 'nan':
+                codes = json.loads(error_codes_str)
+                for code in codes:
+                    error_counts[code] = error_counts.get(code, 0) + 1
+        except (json.JSONDecodeError, TypeError, KeyError, ValueError):
+            error_counts['OTHER'] = error_counts.get('OTHER', 0) + 1
+    
+    if error_counts:
+        for code, count in sorted(error_counts.items(), key=lambda x: x[1], reverse=True):
+            print(f"{code:<35} {count:>3}")
+    else:
+        print("No error codes found.")
+    print("="*50)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Analyze errors in Text2SQL evaluation results")
+    parser.add_argument("--input", required=True, help="Input CSV file path")
+    parser.add_argument("--output", help="Output CSV file path (default: <input>_annotated.csv)")
+    
+    args = parser.parse_args()
+    
+    input_path = Path(args.input)
+    if not input_path.exists():
+        print(f"Error: Input file {args.input} does not exist")
+        sys.exit(1)
+    
+    if args.output:
+        output_path = args.output
+    else:
+        output_path = input_path.parent / f"{input_path.stem}_annotated.csv"
+    
+    analyze_errors(args.input, str(output_path))
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,467 @@
+#!/usr/bin/env python3
+"""
+Data utilities for Text-to-SQL evaluation with Ragas.
+
+This module provides CLI tools to download and prepare datasets for 
+text-to-SQL evaluation workflows.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict, List
+
+# Load environment variables from ragas root
+try:
+    from dotenv import load_dotenv
+    # Load .env from ragas root directory (3 levels up from this file)
+    ragas_root = Path(__file__).parent.parent.parent.parent
+    env_path = ragas_root / ".env"
+    load_dotenv(env_path)
+except ImportError:
+    # dotenv is optional, continue without it
+    pass
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(levelname)s: %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+try:
+    from huggingface_hub import snapshot_download
+    from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
+except ImportError:
+    logger.error("huggingface_hub is required. Install with: pip install huggingface_hub")
+    sys.exit(1)
+
+try:
+    import pandas as pd
+    from pandas import DataFrame
+except ImportError:
+    logger.error("pandas is required. Install with: pip install pandas")
+    sys.exit(1)
+
+# Import validation functions from validate_sql_dataset.py
+try:
+    from .validate_sql_dataset import execute_and_validate_query
+except ImportError:
+    logger.error("validate_sql_dataset.py not found in the same directory")
+    sys.exit(1)
+
+
+def download_booksql_dataset() -> bool:
+    """
+    Download the BookSQL dataset from Hugging Face Hub to ./BookSQL-files directory.
+        
+    Returns:
+        bool: True if download successful, False otherwise
+        
+    Note:
+        This dataset is gated and requires accepting terms on the Hugging Face Hub.
+        You need to:
+        1. Visit https://huggingface.co/datasets/Exploration-Lab/BookSQL
+        2. Accept the terms and conditions
+        3. Authenticate with: huggingface-cli login
+    """
+    repo_id = "Exploration-Lab/BookSQL"
+    local_dir = "BookSQL-files"
+    
+    # Create local directory if it doesn't exist
+    Path(local_dir).mkdir(parents=True, exist_ok=True)
+    
+    logger.info(f"Downloading BookSQL dataset to {local_dir}")
+    logger.info(f"Repository: {repo_id}")
+    
+    try:
+        # Download the entire repository
+        downloaded_path = snapshot_download(
+            repo_id=repo_id,
+            repo_type="dataset",
+            local_dir=local_dir,
+            local_dir_use_symlinks=False  # Create actual files, not symlinks
+        )
+        
+        logger.info(f"Successfully downloaded dataset to: {downloaded_path}")
+        
+        # List downloaded files
+        dataset_path = Path(local_dir)
+        files = list(dataset_path.rglob("*"))
+        logger.info(f"Downloaded {len(files)} files")
+        for file in sorted(files)[:5]:  # Show first 5 files
+            if file.is_file():
+                logger.info(f"  {file.relative_to(dataset_path)}")
+        if len(files) > 5:
+            logger.info(f"  ... and {len(files) - 5} more files")
+            
+        return True
+        
+    except GatedRepoError:
+        logger.error("This dataset is gated and requires authentication")
+        logger.error("Please follow these steps:")
+        logger.error("1. Visit: https://huggingface.co/datasets/Exploration-Lab/BookSQL")
+        logger.error("2. Accept the terms and conditions")
+        logger.error("3. Run: huggingface-cli login")
+        logger.error("4. Try downloading again")
+        return False
+        
+    except RepositoryNotFoundError:
+        logger.error(f"Repository '{repo_id}' not found")
+        return False
+        
+    except Exception as e:
+        logger.error(f"Error downloading dataset: {e}")
+        return False
+
+
+def validate_query_data(query_data: Dict[str, Any], require_data: bool = False) -> bool:
+    """
+    Validate a single query by executing it against the database.
+    
+    Args:
+        query_data: Dictionary containing query information (query, sql, level, split)
+        require_data: If True, only accept queries that return actual data
+        
+    Returns:
+        bool: True if query is valid (and optionally returns data), False otherwise
+    """
+    try:
+        result = execute_and_validate_query(query_data)
+        
+        if not result['execution_success']:
+            return False
+            
+        if require_data:
+            # Only accept queries that return actual data (not empty or null values)
+            return result.get('result_type') == 'has_data'
+        else:
+            # Accept any successful query execution
+            return True
+            
+    except Exception as e:
+        logger.warning(f"Error validating query: {e}")
+        return False
+
+
+def load_and_clean_data(input_file: str) -> DataFrame:
+    """
+    Load JSON data and remove duplicates.
+    
+    Args:
+        input_file: Path to the BookSQL train.json file
+        
+    Returns:
+        DataFrame: Cleaned train data with duplicates removed
+        
+    Raises:
+        FileNotFoundError: If input file doesn't exist
+        json.JSONDecodeError: If JSON is invalid
+    """
+    input_path = Path(input_file)
+    
+    if not input_path.exists():
+        raise FileNotFoundError(f"Input file '{input_file}' not found")
+    
+    logger.info(f"Loading data from {input_file}")
+    
+    # Load JSON data
+    with open(input_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    logger.info(f"Loaded {len(data)} total records")
+    
+    # Convert to DataFrame and filter for train split
+    df = pd.DataFrame(data)
+    train_df = df[df['split'] == 'train'].copy()
+    logger.info(f"Found {len(train_df)} train records")
+    
+    # Remove duplicates based on Query + SQL combination
+    original_count = len(train_df)
+    train_df = train_df.drop_duplicates(subset=['Query', 'SQL'], keep='first')
+    duplicate_count = original_count - len(train_df)
+    
+    if duplicate_count > 0:
+        logger.info(f"Removed {duplicate_count} duplicate records")
+    logger.info(f"{len(train_df)} unique records remaining")
+    
+    # Show difficulty distribution
+    level_counts = train_df['Levels'].value_counts()
+    logger.info("Difficulty distribution after deduplication:")
+    for level, count in level_counts.items():
+        logger.info(f"  {level}: {count} records")
+    
+    return train_df
+
+
+def sample_by_difficulty(data: DataFrame, level: str, samples_per_level: int, random_seed: int) -> DataFrame:
+    """
+    Sample data for a specific difficulty level.
+    
+    Args:
+        data: DataFrame containing the data
+        level: Difficulty level ('easy', 'medium', 'hard')
+        samples_per_level: Number of samples to take
+        random_seed: Random seed for reproducible sampling
+        
+    Returns:
+        DataFrame: Sampled data for the specified level
+    """
+    level_data = data[data['Levels'] == level]
+    
+    if len(level_data) == 0:
+        logger.warning(f"No '{level}' records found, skipping")
+        return pd.DataFrame()
+    
+    if len(level_data) < samples_per_level:
+        logger.warning(f"Only {len(level_data)} '{level}' records available, using all")
+        return level_data
+    else:
+        sampled = level_data.sample(n=samples_per_level, random_state=random_seed)
+        logger.info(f"Sampled {len(sampled)} '{level}' records")
+        return sampled
+
+
+def validate_samples(data: DataFrame, level: str, samples_per_level: int, 
+                    random_seed: int, require_data: bool = False) -> DataFrame:
+    """
+    Sample and validate data for a specific difficulty level.
+    
+    Args:
+        data: DataFrame containing the data
+        level: Difficulty level ('easy', 'medium', 'hard')
+        samples_per_level: Number of samples to find
+        random_seed: Random seed for reproducible sampling
+        require_data: If True, only include queries that return data
+        
+    Returns:
+        DataFrame: Validated samples for the specified level
+    """
+    level_data = data[data['Levels'] == level]
+    
+    if len(level_data) == 0:
+        logger.warning(f"No '{level}' records found, skipping")
+        return pd.DataFrame()
+    
+    logger.info(f"Validating '{level}' queries to find {samples_per_level} valid samples")
+    
+    # Shuffle data for random sampling during validation
+    shuffled_data = level_data.sample(frac=1, random_state=random_seed).reset_index(drop=True)
+    
+    valid_samples = []
+    checked_count = 0
+    
+    for idx, row in shuffled_data.iterrows():
+        checked_count += 1
+        
+        # Prepare query data for validation
+        query_data = {
+            'index': idx,
+            'query': row['Query'],
+            'sql': row['SQL'],
+            'level': row['Levels'],
+            'split': row['split']
+        }
+        
+        if validate_query_data(query_data, require_data):
+            valid_samples.append(row)
+            
+            # Stop if we have enough samples
+            if len(valid_samples) >= samples_per_level:
+                break
+    
+    if len(valid_samples) == 0:
+        logger.warning(f"No valid '{level}' queries found, skipping this level")
+        return pd.DataFrame()
+    elif len(valid_samples) < samples_per_level:
+        logger.warning(f"Only found {len(valid_samples)} valid '{level}' queries out of {samples_per_level} requested")
+    else:
+        logger.info(f"Found {len(valid_samples)} valid '{level}' queries")
+    
+    return pd.DataFrame(valid_samples) if valid_samples else pd.DataFrame()
+
+
+def save_results(data: DataFrame, output_dir: str, output_filename: str, random_seed: int) -> bool:
+    """
+    Save final dataset to CSV.
+    
+    Args:
+        data: Final dataset to save
+        output_dir: Directory to save the output CSV
+        output_filename: Name of the output CSV file
+        random_seed: Random seed for final shuffle
+        
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    if data.empty:
+        logger.error("No data to save")
+        return False
+    
+    # Create output directory
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+    
+    # Final duplicate check
+    pre_final_count = len(data)
+    data = data.drop_duplicates(subset=['Query', 'SQL'], keep='first')
+    final_duplicate_count = pre_final_count - len(data)
+    
+    if final_duplicate_count > 0:
+        logger.warning(f"Removed {final_duplicate_count} duplicates from final sample")
+    
+    # Shuffle the final dataset
+    data = data.sample(frac=1, random_state=random_seed).reset_index(drop=True)
+    
+    # Save to CSV
+    output_file_path = output_path / output_filename
+    data.to_csv(output_file_path, index=False)
+    
+    logger.info(f"Saved {len(data)} records to {output_file_path}")
+    logger.info("Final distribution:")
+    for level, count in data['Levels'].value_counts().items():
+        logger.info(f"  {level}: {count} records")
+    
+    return True
+
+
+def create_sample_dataset(
+    input_file: str = "BookSQL-files/BookSQL/train.json",
+    output_dir: str = "datasets",
+    output_filename: str = "booksql_sample.csv",
+    samples_per_level: int = 10,
+    random_seed: int = 42,
+    validate_queries: bool = False,
+    require_data: bool = False
+) -> bool:
+    """
+    Create a balanced sample dataset from BookSQL train.json.
+    
+    This function orchestrates the data loading, sampling, validation, and saving process.
+    
+    Args:
+        input_file: Path to the BookSQL train.json file
+        output_dir: Directory to save the output CSV
+        output_filename: Name of the output CSV file
+        samples_per_level: Number of samples per difficulty level (easy, medium, hard)
+        random_seed: Random seed for reproducible sampling
+        validate_queries: If True, validate SQL queries before including them
+        require_data: If True (and validate_queries=True), only include queries that return data
+        
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    try:
+        # Step 1: Load and clean data
+        train_df = load_and_clean_data(input_file)
+        
+        # Step 2: Sample data for each difficulty level
+        sampled_dfs = []
+        
+        if validate_queries:
+            logger.info("Validation enabled - testing SQL queries before including them in sample")
+            if require_data:
+                logger.info("Only including queries that return actual data")
+        
+        for level in ['easy', 'medium', 'hard']:
+            if validate_queries:
+                sampled = validate_samples(train_df, level, samples_per_level, random_seed, require_data)
+            else:
+                sampled = sample_by_difficulty(train_df, level, samples_per_level, random_seed)
+            
+            if not sampled.empty:
+                sampled_dfs.append(sampled)
+        
+        if not sampled_dfs:
+            logger.error("No data could be sampled")
+            return False
+        
+        # Step 3: Combine all sampled data
+        final_df = pd.concat(sampled_dfs, ignore_index=True)
+        
+        # Step 4: Save results
+        return save_results(final_df, output_dir, output_filename, random_seed)
+        
+    except FileNotFoundError:
+        logger.error(f"Input file '{input_file}' not found")
+        logger.error("Tip: Run with --download-data first to download the BookSQL dataset")
+        return False
+    except json.JSONDecodeError as e:
+        logger.error(f"Invalid JSON in {input_file}: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Error processing data: {e}")
+        return False
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Data utilities for Text-to-SQL evaluation",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s --download-data                          # Download BookSQL dataset
+  %(prog)s --create-sample                          # Create sample CSV (15 per level)
+  %(prog)s --create-sample --samples 5              # Create sample with 5 per level
+  %(prog)s --create-sample --validate               # Create sample with SQL validation
+  %(prog)s --create-sample --validate --require-data # Only queries that return data
+        """
+    )
+    
+    parser.add_argument(
+        "--download-data",
+        action="store_true",
+        help="Download the BookSQL dataset to ./BookSQL-files directory"
+    )
+    
+    parser.add_argument(
+        "--create-sample",
+        action="store_true",
+        help="Create a balanced sample CSV from BookSQL train.json"
+    )
+    
+    parser.add_argument(
+        "--samples",
+        type=int,
+        default=15,
+        help="Number of samples per difficulty level (default: 15)"
+    )
+    
+    parser.add_argument(
+        "--validate",
+        action="store_true",
+        help="Validate SQL queries before including them in the sample"
+    )
+    
+    parser.add_argument(
+        "--require-data",
+        action="store_true",
+        help="Only include queries that return actual data (requires --validate)"
+    )
+    
+    args = parser.parse_args()
+    
+    if args.download_data:
+        success = download_booksql_dataset()
+        sys.exit(0 if success else 1)
+    elif args.create_sample:
+        # Validate argument combinations
+        if args.require_data and not args.validate:
+            logger.error("--require-data requires --validate to be enabled")
+            sys.exit(1)
+            
+        success = create_sample_dataset(
+            samples_per_level=args.samples,
+            validate_queries=args.validate,
+            require_data=args.require_data
+        )
+        sys.exit(0 if success else 1)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,100 @@
+Query,SQL,Levels,split
+What is the balance due from Richard Aguirre?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Richard Aguirre"" ) ",medium,train
+What is the balance due from Sarah Oconnor?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Sarah Oconnor"" ) ",medium,train
+What is my average invoice from Jeffrey Moore?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Jeffrey Moore"" and transaction_type = 'invoice')",hard,train
+How much open credit does customer Andrew Bennett?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Andrew Bennett"" ) ",easy,train
+What is my average invoice from Jeremy Strong?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Jeremy Strong"" and transaction_type = 'invoice')",hard,train
+What is my average invoice from Lisa Mitchell?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Lisa Mitchell"" and transaction_type = 'invoice')",hard,train
+Justin Estes has received how many invoices?,"select count(distinct transaction_id) from master_txn_table where customers = ""Justin Estes"" and transaction_type = 'invoice'",medium,train
+Display the total number of transactions with Jonathan Barton,"select count(distinct transaction_id) from master_txn_table where customers = ""Jonathan Barton""",medium,train
+How much open credit does customer Tracy Bean?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Tracy Bean"" ) ",easy,train
+How much open credit does customer Wanda Welch?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Wanda Welch"" ) ",easy,train
+How much open credit does customer Kathleen George?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Kathleen George"" ) ",easy,train
+How much we received from Providing independent operation of railroad terminals?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Providing independent operation of railroad terminals"")",hard,train
+What was the most recent invoice for Leslie Beck?,"select transaction_id from master_txn_table where transaction_type = 'invoice' and customers = ""Leslie Beck"" order by transaction_date desc limit 1",medium,train
+How much open credit does customer Sylvia Williams?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Sylvia Williams"" ) ",easy,train
+Display all transactions involving Crystal Todd,"select distinct transaction_id from master_txn_table where customers = ""Crystal Todd""",medium,train
+How much open credit does customer Robert Bowers?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Robert Bowers"" ) ",easy,train
+How much open credit does customer Andrew Vaughan?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Andrew Vaughan"" ) ",easy,train
+How much open credit does customer Karen Bonilla?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Karen Bonilla"" ) ",easy,train
+How much has Colleen Ward been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Colleen Ward"" group by date(transaction_date, 'start of month')",hard,train
+What are my total sales by Duplexes?,"select sum(credit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income') and product_service = ""Duplexes""",hard,train
+What was the total amount earned in Intravenous Therapy This fiscal year to date?,"select sum(credit) from master_txn_table where transaction_date BETWEEN date(current_date, '-3 months', 'start of year', '+3 months') AND date(current_date, '-3 months', 'start of year','+1 year', '+3 months', '-1 day')  and product_service = 'Intravenous Therapy' and transaction_type in ('invoice', 'sales recept')",medium,train
+What is my average invoice from Nicholas Kim?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Nicholas Kim"" and transaction_type = 'invoice')",hard,train
+How much has Tracy Rojas been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Tracy Rojas"" group by date(transaction_date, 'start of month')",hard,train
+How much open credit does customer Suzanne Hayes?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Suzanne Hayes"" ) ",easy,train
+What are the invoice dates for customers with the customer name Natasha Lin?,"SELECT transaction_date from (select distinct transaction_id, transaction_date from master_txn_table where customers=""Natasha Lin"" and transaction_type = 'invoice') ",medium,train
+When was the last time we billed for Loading and unloading,"select transaction_date from master_txn_table where product_service = ""Loading and unloading"" order by transaction_date desc limit 1; ",medium,train
+How much open credit does customer Robert Roberts?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Robert Roberts"" ) ",easy,train
+"In the This fiscal year, what has been my total revenue from Catherine Lindsey?","select sum(credit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income') and customers = ""Catherine Lindsey"" and transaction_date BETWEEN date(current_date, '-3 months', 'start of year', '+3 months') AND date(current_date) ",hard,train
+How much open credit does customer Jacob Melendez?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Jacob Melendez"" ) ",easy,train
+Display all transactions involving Julie Randall,"select distinct transaction_id from master_txn_table where customers = ""Julie Randall""",medium,train
+How much has Shannon Hernandez been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Shannon Hernandez"" group by date(transaction_date, 'start of month')",hard,train
+How much open credit does customer Miguel Villarreal?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Miguel Villarreal"" ) ",easy,train
+How much open credit does customer Brian Wheeler?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Brian Wheeler"" ) ",easy,train
+How many credit card transactions occurred This year?,"select count(distinct transaction_id) from master_txn_table as T1 join payment_method as T2 on T1.payment_method = T2.payment_method where T2.credit_card = ""yes"" and T1.transaction_date BETWEEN date(current_date, 'start of year') AND date(current_date) ",hard,train
+How much open credit does customer Tonya Lee?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Tonya Lee"" ) ",easy,train
+Show all transactions with Mr Andrea Smith,select distinct transaction_id from master_txn_table where customers = 'Andrea Smith' or vendor = 'Andrea Smith',medium,train
+How much has Samantha Aguilar been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Samantha Aguilar"" group by date(transaction_date, 'start of month')",hard,train
+Show number of transactions with Carol Smith,select count(distinct transaction_id) from master_txn_table where customers = 'Carol Smith' or vendor = 'Carol Smith',medium,train
+How much open credit does customer Natalie Myers?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Natalie Myers"" ) ",easy,train
+How much we received from Fuel?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Fuel"")",hard,train
+"As of This month to date, how many invoices for Brent Rodriguez were still outstanding?","select count(distinct transaction_id) from master_txn_table where customers = ""Brent Rodriguez"" and transaction_type = 'invoice' and open_balance >0 and transaction_date BETWEEN date( current_date, ""start of month"") AND date( current_date) ",medium,train
+How much open credit does customer Melissa Weaver?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Melissa Weaver"" ) ",easy,train
+Show all transactions with Mr Corey Durham,select distinct transaction_id from master_txn_table where customers = 'Corey Durham' or vendor = 'Corey Durham',medium,train
+How much open credit does customer Karen Brown?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Karen Brown"" ) ",easy,train
+How much open credit does customer Julie Flynn MD?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Julie Flynn MD"" ) ",easy,train
+What are my total sales by Oil and gas wells?,"select sum(credit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income') and product_service = ""Oil and gas wells""",hard,train
+How much open credit does customer Robert Hammond?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Robert Hammond"" ) ",easy,train
+What is my last invoice from Vicki Page?,"select distinct transaction_id, amount, transaction_date from master_txn_table where customers = ""Vicki Page"" and transaction_type = 'invoice' order by transaction_date desc limit 1 ",medium,train
+How much open credit does customer Casey King?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Casey King"" ) ",easy,train
+How much open credit does customer Gail Hoover?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Gail Hoover"" ) ",easy,train
+How much open credit does customer Jeremy Benson?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Jeremy Benson"" ) ",easy,train
+How much open credit does customer Susan Williamson?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Susan Williamson"" ) ",easy,train
+What was the mean invoice amount for Barbara Scott?,"select avg(credit) from master_txn_table where transaction_type = 'invoice' and customers = ""Barbara Scott"" ",medium,train
+How much open credit does customer Jerry Nunez?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Jerry Nunez"" ) ",easy,train
+What is my average invoice from Robert Edwards?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Robert Edwards"" and transaction_type = 'invoice')",hard,train
+How much open credit does customer Sabrina Newton?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Sabrina Newton"" ) ",easy,train
+What is my average invoice from Anna Martin?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Anna Martin"" and transaction_type = 'invoice')",hard,train
+How many invoices have we sent to Nathaniel Montgomery?,"select count(distinct transaction_id) from master_txn_table where customers = ""Nathaniel Montgomery"" and transaction_type = 'invoice'",medium,train
+What's the profit Last 12 months?,"select sum(credit - debit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income','Expense','Other Expense') and  transaction_date BETWEEN date( current_date, ""-12 months"", ""start of month"") AND date( current_date, 'start of month', '-1 day') ",hard,train
+Show all of Andrea Martinez's transactions,"select distinct transaction_id from master_txn_table where customers = ""Andrea Martinez""",medium,train
+How much has Monica Valentine been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Monica Valentine"" group by date(transaction_date, 'start of month')",hard,train
+What is my total bill for Tammy Johnson?,"select sum(credit) from master_txn_table where transaction_type = 'bill' and vendor = ""Tammy Johnson""",medium,train
+How many invoices have we sent to Nathan Pineda?,"select count(distinct transaction_id) from master_txn_table where customers = ""Nathan Pineda"" and transaction_type = 'invoice'",medium,train
+Show all transactions with Mr John Copeland,select distinct transaction_id from master_txn_table where customers = 'John Copeland' or vendor = 'John Copeland',medium,train
+How much we received from Manufacturing other Natural oils?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Manufacturing other Natural oils"")",hard,train
+Display the total number of transactions with Raymond Brown,"select count(distinct transaction_id) from master_txn_table where customers = ""Raymond Brown""",medium,train
+How much we received from Other Services?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Other Services"")",hard,train
+What is my total bill for Sydney Gonzalez?,"select sum(credit) from master_txn_table where transaction_type = 'bill' and vendor = ""Sydney Gonzalez""",medium,train
+What is my average invoice from Jordan Schmidt?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Jordan Schmidt"" and transaction_type = 'invoice')",hard,train
+How much we received from Acidizing and chemically treating wells?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Acidizing and chemically treating wells"")",hard,train
+"As of in q3 last year, how many invoices for Crystal Anthony were still outstanding?","select count(distinct transaction_id) from master_txn_table where customers = ""Crystal Anthony"" and transaction_type = 'invoice' and open_balance >0 and transaction_date BETWEEN date(current_date, '-1 year', 'start of year', '+6 month') AND date(current_date, '-1 year', 'start of year', '+9 month', '-1 day') ",medium,train
+What is my last invoice from Jody Sanchez?,"select distinct transaction_id, amount, transaction_date from master_txn_table where customers = ""Jody Sanchez"" and transaction_type = 'invoice' order by transaction_date desc limit 1 ",medium,train
+Number of invoices created for Loan Payable?,"select count(distinct transaction_id) from master_txn_table where transaction_type = 'invoice' and instr(account,""Loan Payable"")",medium,train
+What is my average invoice from Ashley Thompson?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Ashley Thompson"" and transaction_type = 'invoice')",hard,train
+Show number of transactions with Terri Bowman,select count(distinct transaction_id) from master_txn_table where customers = 'Terri Bowman' or vendor = 'Terri Bowman',medium,train
+How much we received from Wholesaling aircraft?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Wholesaling aircraft"")",hard,train
+How much open credit does customer Kiara Pearson?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Kiara Pearson"" ) ",easy,train
+What is my average invoice from Heather Haas?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Heather Haas"" and transaction_type = 'invoice')",hard,train
+What was the most recent invoice for Roberta Shaw?,"select transaction_id from master_txn_table where transaction_type = 'invoice' and customers = ""Roberta Shaw"" order by transaction_date desc limit 1",medium,train
+What are the invoice dates for customers with the customer name Bryan Garcia?,"SELECT transaction_date from (select distinct transaction_id, transaction_date from master_txn_table where customers=""Bryan Garcia"" and transaction_type = 'invoice') ",medium,train
+How much has Dawn Roman been paying us every month,"select date(transaction_date, 'start of month'), sum(credit) from master_txn_table where customers = ""Dawn Roman"" group by date(transaction_date, 'start of month')",hard,train
+Number of invoices created for Installation?,"select count(distinct transaction_id) from master_txn_table where transaction_type = 'invoice' and instr(account,""Installation"")",medium,train
+How much open credit does customer Eric Smith II?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Eric Smith II"" ) ",easy,train
+How much open credit does customer Andre Stevens?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Andre Stevens"" ) ",easy,train
+What was the min invoice value for Photocopying services?,"select min(credit) from master_txn_table where transaction_type = 'invoice' and instr(account,""Photocopying services"")",medium,train
+How much open credit does customer Helen Patrick?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Helen Patrick"" ) ",easy,train
+How much open credit does customer Jonathan Bradley?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Jonathan Bradley"" ) ",easy,train
+How much open credit does customer Anthony Olson?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Anthony Olson"" ) ",easy,train
+What is my average invoice from Kathleen Brown?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Kathleen Brown"" and transaction_type = 'invoice')",hard,train
+What is my average invoice from Erik Mckenzie?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Erik Mckenzie"" and transaction_type = 'invoice')",hard,train
+How much we received from Data entry services?,"select sum(credit) from master_txn_table as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income', 'Other Income') and instr(account,""Data entry services"")",hard,train
+What is my average invoice from William Hendricks?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""William Hendricks"" and transaction_type = 'invoice')",hard,train
+What is my average invoice from Anthony Armstrong?,"select avg(amount) from (select distinct transaction_id, amount from master_txn_table where customers = ""Anthony Armstrong"" and transaction_type = 'invoice')",hard,train
+How much open credit does customer Harold Neal?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Harold Neal"" ) ",easy,train
+Display the total number of transactions with Margaret Alvarez,"select count(distinct transaction_id) from master_txn_table where customers = ""Margaret Alvarez""",medium,train
+What are my total sales by Ships?,"select sum(credit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income') and product_service = ""Ships""",hard,train
+How much open credit does customer Samuel Turner?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Samuel Turner"" ) ",easy,train
+What are my total sales by Miscellaneous?,"select sum(credit) from master_txn_table  as T1 join chart_of_accounts as T2 on T1.account = T2.account_name where account_type in ('Income','Other Income') and product_service = ""Miscellaneous""",hard,train
+How much money does Joshua Hensley still owe?,"select sum(open_balance) from ( select distinct transaction_id, open_balance from master_txn_table where customers = ""Joshua Hensley"")",medium,train
@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""
+Simple database utilities for Text-to-SQL evaluation.
+
+This module helps you execute SQL queries against SQLite databases 
+and get results as pandas DataFrames for easy comparison in evaluations.
+"""
+
+import argparse
+import re
+import sqlite3
+import sys
+from pathlib import Path
+from typing import Optional, Tuple, Union
+
+try:
+    import pandas as pd
+except ImportError:
+    raise ImportError("pandas is required. Install with: pip install pandas")
+
+
+class SQLiteDB:
+    """
+    Simple SQLite database interface for text-to-SQL evaluation.
+    
+    This class makes it easy to:
+    - Connect to SQLite databases  
+    - Execute SQL queries
+    - Get results as pandas DataFrames
+    - Handle errors gracefully
+    """
+    
+    def __init__(self, db_path: Optional[str] = None):
+        """
+        Create a new database connection.
+        
+        Args:
+            db_path: Path to SQLite database file.
+                    If None, uses BookSQL dataset: "BookSQL-files/BookSQL/accounting.sqlite"
+        """
+        if db_path is None:
+            self.db_path = Path("BookSQL-files/BookSQL/accounting.sqlite")
+        else:
+            self.db_path = Path(db_path)
+            
+        self._connection = None
+        
+    def connect(self) -> Tuple[bool, str]:
+        """
+        Connect to the database.
+        
+        Returns:
+            (success: bool, message: str)
+        """
+        try:
+            if not self.db_path.exists():
+                return False, f"Database file not found: {self.db_path}"
+                
+            self._connection = sqlite3.connect(str(self.db_path), timeout=1.0)
+            self._connection.row_factory = sqlite3.Row
+            return True, "Connected successfully"
+            
+        except Exception as e:
+            return False, f"Database connection error: {e}"
+    
+    def disconnect(self) -> None:
+        """Close the database connection."""
+        if self._connection:
+            self._connection.close()
+            self._connection = None
+    
+    def execute_query(self, sql: str, replace_current_date: bool = True, case_insensitive: bool = True) -> Tuple[bool, Union[pd.DataFrame, str]]:
+        """
+        Execute a SQL query and return results as a DataFrame.
+        
+        Args:
+            sql: SQL SELECT query to execute
+            replace_current_date: Replace date functions with fixed date for historical data
+            case_insensitive: Make string comparisons case-insensitive
+            
+        Returns:
+            (success: bool, result: DataFrame or error_message: str)
+            
+        Example:
+            success, result = db.execute_query("SELECT COUNT(*) FROM customers")
+            if success:
+                print(f"Found {result.iloc[0, 0]} customers")
+            else:
+                print(f"Query failed: {result}")
+        """
+        # Connect if needed
+        if not self._connection:
+            success, message = self.connect()
+            if not success:
+                return False, f"Connection failed: {message}"
+        
+        # Security check - only allow SELECT queries
+        if not sql.strip().upper().startswith('SELECT'):
+            return False, "Only SELECT queries are supported"
+        
+        # Clean up the SQL query
+        sql = self._normalize_sql(sql, replace_current_date, case_insensitive)
+            
+        try:
+            # Execute query and convert to DataFrame
+            df = pd.read_sql_query(sql, self._connection)
+            return True, df
+            
+        except Exception as e:
+            return False, f"SQL execution error: {e}"
+    
+    def _normalize_sql(self, sql: str, replace_current_date: bool, case_insensitive: bool) -> str:
+        """
+        Clean up SQL query for better compatibility.
+        
+        This method:
+        - Fixes quote marks (double → single)
+        - Cleans up whitespace
+        - Replaces date functions with fixed dates 
+        - Makes text case-insensitive if requested
+        """
+        # Fix quotes: double → single
+        sql = sql.replace('"', "'")
+        
+        # Clean up whitespace
+        sql = re.sub(r'\s+', ' ', sql.strip())
+        
+        # Replace date functions with fixed date for historical data
+        if replace_current_date:
+            sql = sql.replace('current_date', "'2022-06-01'")
+            sql = sql.replace(', now', ", '2022-06-01'")
+            sql = sql.replace("'now'", "'2022-06-01'")
+            sql = sql.replace('%y', "%Y")
+        
+        # Make case-insensitive if requested
+        if case_insensitive:
+            sql = sql.lower()
+            
+        return sql
+    
+    def get_schema_info(self) -> Tuple[bool, Union[pd.DataFrame, str]]:
+        """
+        Get information about all tables and views in the database.
+        
+        Returns:
+            (success: bool, schema_info: DataFrame or error_message: str)
+            DataFrame contains: name, type, sql (CREATE statements)
+        """
+        schema_query = """
+        SELECT name, type, sql
+        FROM sqlite_master
+        WHERE type IN ('table', 'view')
+          AND name NOT LIKE 'sqlite_%'
+        ORDER BY type, name
+        """
+        return self.execute_query(schema_query, replace_current_date=False, case_insensitive=False)
+    
+    def get_table_names(self) -> Tuple[bool, Union[list, str]]:
+        """
+        Get a list of all table names in the database.
+        
+        Returns:
+            (success: bool, table_names: list or error_message: str)
+        """
+        tables_query = """
+        SELECT name FROM sqlite_master 
+        WHERE type='table' AND name NOT LIKE 'sqlite_%'
+        ORDER BY name
+        """
+        success, result = self.execute_query(tables_query, replace_current_date=False, case_insensitive=False)
+        
+        if success and isinstance(result, pd.DataFrame):
+            return True, result['name'].tolist()
+        else:
+            return False, str(result)
+
+
+# Convenience functions for quick usage
+
+def execute_sql(sql: str, db_path: Optional[str] = None, replace_current_date: bool = True, case_insensitive: bool = True) -> Tuple[bool, Union[pd.DataFrame, str]]:
+    """
+    Execute a SQL query with automatic connection management.
+    
+    This is the main function you'll use for running SQL queries in evaluations.
+    
+    Args:
+        sql: SQL SELECT query to execute
+        db_path: Path to database file (uses BookSQL default if None)
+        replace_current_date: Replace date functions with fixed date
+        case_insensitive: Make string comparisons case-insensitive
+        
+    Returns:
+        (success: bool, result: DataFrame or error_message: str)
+        
+    Example:
+        success, data = execute_sql("SELECT COUNT(*) FROM customers")
+        if success:
+            print(f"Query returned {len(data)} rows")
+        else:
+            print(f"Error: {data}")
+    """
+    db = SQLiteDB(db_path)
+    try:
+        return db.execute_query(sql, replace_current_date, case_insensitive)
+    finally:
+        db.disconnect()
+
+
+def get_database_schema(db_path: Optional[str] = None) -> Tuple[bool, Union[pd.DataFrame, str]]:
+    """
+    Get database schema information with automatic connection management.
+    
+    Args:
+        db_path: Path to database file (uses BookSQL default if None)
+        
+    Returns:
+        (success: bool, schema_info: DataFrame or error_message: str)
+    """
+    db = SQLiteDB(db_path)
+    try:
+        return db.get_schema_info()
+    finally:
+        db.disconnect()
+
+
+def main():
+    """Simple command-line interface for testing queries."""
+    parser = argparse.ArgumentParser(
+        description="Execute SQL queries against SQLite database",
+        epilog="""
+Examples:
+  python db_utils.py --query "SELECT COUNT(*) FROM master_txn_table"
+  python db_utils.py --schema
+  python db_utils.py --tables
+        """
+    )
+    
+    parser.add_argument("--query", "-q", help="SQL query to execute")
+    parser.add_argument("--db", "-d", help="Database file path")
+    parser.add_argument("--schema", "-s", action="store_true", help="Show database schema")
+    parser.add_argument("--tables", "-t", action="store_true", help="List all tables")
+    
+    args = parser.parse_args()
+    
+    # Must specify at least one action
+    if not any([args.query, args.schema, args.tables]):
+        parser.print_help()
+        print("\nError: Specify --query, --schema, or --tables")
+        sys.exit(1)
+    
+    try:
+        db = SQLiteDB(args.db)
+        
+        # Show schema
+        if args.schema:
+            print("=== Database Schema ===")
+            success, result = db.get_schema_info()
+            if success:
+                print(result.to_string(index=False))
+            else:
+                print(f"Error: {result}")
+                sys.exit(1)
+        
+        # List tables
+        if args.tables:
+            print("=== Tables ===")
+            success, tables = db.get_table_names()
+            if success:
+                for table in tables:
+                    print(f"  {table}")
+            else:
+                print(f"Error: {tables}")
+                sys.exit(1)
+        
+        # Execute query
+        if args.query:
+            print("=== Query Results ===")
+            print(f"Query: {args.query}")
+            print()
+            
+            success, result = db.execute_query(args.query)
+            if success:
+                if len(result) == 0:
+                    print("No rows returned.")
+                else:
+                    print(result.to_string(index=False))
+                    print(f"\nRows: {len(result)}")
+            else:
+                print(f"Error: {result}")
+                sys.exit(1)
+                
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    finally:
+        if 'db' in locals():
+            db.disconnect()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,232 @@
+import asyncio
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from ragas import Dataset, experiment
+from ragas.metrics.discrete import discrete_metric
+from ragas.metrics.result import MetricResult
+
+import datacompy
+
+from .db_utils import execute_sql
+from .text2sql_agent import Text2SQLAgent
+
+# Load environment variables
+load_dotenv(".env")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+logger = logging.getLogger(__name__)
+
+# Suppress HTTP request logs from OpenAI/httpx
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("openai._base_client").setLevel(logging.WARNING)
+
+@discrete_metric(name="execution_accuracy", allowed_values=["correct", "incorrect"])
+def execution_accuracy(expected_sql: str, predicted_success: bool, predicted_result):
+    """Compare execution results of predicted vs expected SQL using datacompy."""
+    try:
+        # Execute expected SQL
+        expected_success, expected_result = execute_sql(expected_sql)
+        
+        # If expected SQL fails, it's incorrect
+        if not expected_success:
+            return MetricResult(
+                value="incorrect",
+                reason=f"Expected SQL failed to execute: {expected_result}"
+            )
+        
+        # If predicted SQL fails, it's incorrect
+        if not predicted_success:
+            return MetricResult(
+                value="incorrect",
+                reason=f"Predicted SQL failed to execute: {predicted_result}"
+            )
+        
+        # Both queries succeeded - compare DataFrames using datacompy
+        if isinstance(expected_result, pd.DataFrame) and isinstance(predicted_result, pd.DataFrame):
+            
+            # Handle empty DataFrames
+            if expected_result.empty and predicted_result.empty:
+                return MetricResult(
+                    value="correct",
+                    reason="Both queries returned empty results"
+                )
+            
+            # If one is empty and the other isn't, they're different
+            if expected_result.empty != predicted_result.empty:
+                return MetricResult(
+                    value="incorrect",
+                    reason=f"Expected returned {len(expected_result)} rows, predicted returned {len(predicted_result)} rows"
+                )
+            
+            # Guard for very large results to avoid pathological comparisons
+            if len(expected_result) > 10000 or len(predicted_result) > 10000:
+                return MetricResult(
+                    value="incorrect",
+                    reason=(
+                        f"Result too large to compare (expected_rows={len(expected_result)}, "
+                        f"predicted_rows={len(predicted_result)}, max_rows=10000)"
+                    ),
+                )
+
+            # Use datacompy to compare DataFrames
+            try:
+                # Reset index to ensure clean comparison
+                expected_clean = expected_result.reset_index(drop=True)
+                predicted_clean = predicted_result.reset_index(drop=True)
+                
+                # Compare using datacompy with index-based comparison
+                comparison = datacompy.Compare(
+                    expected_clean, 
+                    predicted_clean,
+                    on_index=True,  # Compare row-by-row by index position
+                    abs_tol=1e-10,  # Very small tolerance for floating point comparison
+                    rel_tol=1e-10,
+                    df1_name='expected',
+                    df2_name='predicted'
+                )
+                
+                if comparison.matches():
+                    return MetricResult(
+                        value="correct",
+                        reason=f"DataFrames match exactly ({len(expected_result)} rows, {len(expected_result.columns)} columns)"
+                    )
+                else:
+                    return MetricResult(
+                        value="incorrect",
+                        reason=f"DataFrames do not match. {comparison.report()}\nExpected: \n{expected_result}\nPredicted: \n{predicted_result}"
+                    )
+                    
+            except Exception as comparison_error:
+                # If datacompy fails, report it as incorrect
+                return MetricResult(
+                    value="incorrect",
+                    reason=f"DataFrame comparison failed with datacompy: {str(comparison_error)}"
+                )
+        else:
+            return MetricResult(
+                value="incorrect",
+                reason="One or both query results are not DataFrames"
+            )
+            
+    except Exception as e:
+        return MetricResult(
+            value="incorrect",
+            reason=f"Execution accuracy evaluation failed: {str(e)}"
+        )
+
+
+@experiment()
+async def text2sql_experiment(
+    row,
+    model: str,
+    prompt_file: Optional[str],
+):
+    """Experiment function for text-to-SQL evaluation."""
+    # Create text-to-SQL agent
+    openai_client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])
+    agent = Text2SQLAgent(
+        client=openai_client,
+        model_name=model,
+        prompt_file=prompt_file
+    )
+    
+    # Generate SQL from natural language query
+    result = await agent.query(row["Query"])
+
+    # Execute predicted SQL
+    try:
+        predicted_success, predicted_result = execute_sql(result["sql"])
+    except Exception as e:
+        predicted_success, predicted_result = False, f"SQL execution failed: {str(e)}"
+
+    # Score the response using execution accuracy
+    accuracy_score = await execution_accuracy.ascore(
+        expected_sql=row["SQL"],
+        predicted_success=predicted_success,
+        predicted_result=predicted_result,
+    )
+
+    return {
+        "query": row["Query"],
+        "expected_sql": row["SQL"],
+        "predicted_sql": result["sql"],
+        "level": row["Levels"],
+        "execution_accuracy": accuracy_score.value,
+        "accuracy_reason": accuracy_score.reason,
+    }
+
+
+def load_dataset(limit: Optional[int] = None):
+    """Load the text-to-SQL dataset from CSV file."""
+    dataset_path = Path(__file__).parent / "datasets" / "booksql_sample.csv"
+    
+    # Read CSV
+    df = pd.read_csv(dataset_path)
+    
+    # Limit dataset size if requested
+    if limit is not None and limit > 0:
+        df = df.head(limit)
+    
+    # Create Ragas Dataset
+    dataset = Dataset(name="text2sql_booksql", backend="local/csv", root_dir=".")
+    
+    for _, row in df.iterrows():
+        dataset.append({
+            "Query": row["Query"],
+            "SQL": row["SQL"], 
+            "Levels": row["Levels"],
+            "split": row["split"],
+        })
+    
+    return dataset
+
+
+async def main():
+    """Simple demo script to run text-to-SQL evaluation."""
+    logger.info("TEXT-TO-SQL EVALUATION DEMO")
+    logger.info("=" * 40)
+    
+    # Configuration
+    model = "gpt-5-mini"
+    prompt_file = None
+    name = "demo_evaluation"
+    limit = 5  # Only evaluate 5 samples for demo
+    
+    # Validate API key is available
+    if not os.environ.get("OPENAI_API_KEY"):
+        logger.error("❌ Error: OPENAI_API_KEY environment variable is not set")
+        return
+
+    # Load dataset
+    logger.info("Loading dataset...")
+    dataset = load_dataset(limit=limit)
+    logger.info(f"Dataset loaded with {len(dataset)} samples")
+
+    logger.info(f"Running text-to-SQL evaluation with model: {model}")
+        
+    # Run the experiment
+    results = await text2sql_experiment.arun(
+        dataset, 
+        name=name,
+        model=model,
+        prompt_file=prompt_file,
+    )
+    
+    # Report results
+    logger.info(f"✅ {name}: {len(results)} cases evaluated")
+
+    # Calculate and display accuracy
+    accuracy_rate = sum(1 for r in results if r["execution_accuracy"] == "correct") / max(1, len(results))
+    logger.info(f"{name} Execution Accuracy: {accuracy_rate:.2%}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
@@ -0,0 +1,111 @@
+You are a SQL query generator for a business accounting database. Convert natural language queries to SQL queries.
+
+DATABASE CONTEXT:
+This is an accounting database (accounting.sqlite) containing business transaction and entity data.
+
+TABLES AND THEIR PURPOSE:
+- master_txn_table: Main transaction records for all business transactions
+- chart_of_accounts: Account names and their types for all businesses  
+- products_service: Products/services and their types used by businesses
+- customers: Customer records with billing/shipping details
+- vendors: Vendor records with billing address details
+- payment_method: Payment methods used by businesses
+- employees: Employee details including name, ID, hire date
+
+DATABASE SCHEMA (DDL):
+
+CREATE TABLE chart_of_accounts(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Account_name TEXT NOT NULL,
+  Account_type TEXT NOT NULL,
+  PRIMARY KEY(id,businessID,Account_name)
+);
+
+CREATE TABLE customers(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  customer_name TEXT NOT NULL,
+  customer_full_name TEXT,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Shipping_address TEXT,
+  Shipping_city TEXT,
+  Shipping_state TEXT,
+  Shipping_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Customer_name)
+);
+
+CREATE TABLE employees(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Employee_name TEXT NOT NULL,
+  Employee_ID TEXT,
+  Hire_date DATE,
+  Billing_rate DOUBLE,
+  Deleted TEXT,
+  PRIMARY KEY(id,businessID,Employee_name)
+);
+
+CREATE TABLE master_txn_table(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Transaction_ID INTEGER NOT NULL,
+  Transaction_DATE DATE NOT NULL,
+  Transaction_TYPE TEXT NOT NULL,
+  Amount DOUBLE NOT NULL,
+  CreatedDATE DATE NOT NULL,
+  CreatedUSER TEXT NOT NULL,
+  Account TEXT NOT NULL,
+  AR_paid TEXT,
+  AP_paid TEXT,
+  Due_DATE DATE,
+  Open_balance DOUBLE,
+  Customers TEXT,
+  Vendor TEXT,
+  Product_Service TEXT,
+  Quantity INTEGER,
+  Rate DOUBLE,
+  Credit DOUBLE,
+  Debit DOUBLE,
+  payment_method TEXT,
+  Misc TEXT,
+  FOREIGN KEY(businessID,Account) REFERENCES chart_of_accounts(businessID,Account_name),
+  FOREIGN KEY(businessID,Customers) REFERENCES customers(businessID,customer_name),
+  FOREIGN KEY(businessID,Vendor) REFERENCES vendors(businessID,Vendor_name),
+  FOREIGN KEY(businessID,Product_Service) REFERENCES products(businessID,Product_Service)
+);
+
+CREATE TABLE payment_method(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Payment_method TEXT,
+  Credit_card TEXT,
+  PRIMARY KEY(id,businessID,Payment_method)
+);
+
+CREATE TABLE products(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Product_Service TEXT NOT NULL,
+  Product_Service_type TEXT,
+  PRIMARY KEY(id,businessID,Product_Service)
+);
+
+CREATE TABLE vendors(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Vendor_name TEXT NOT NULL,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Vendor_name)
+);
+
+INSTRUCTIONS:
+Convert the user's natural language query into a valid SQL SELECT query. Return only the SQL query, no explanations or formatting.
@@ -0,0 +1,135 @@
+You are a SQL query generator for a business accounting database. Convert natural language queries to SQL queries.
+
+DATABASE CONTEXT:
+This is an accounting database (accounting.sqlite) containing business transaction and entity data.
+
+TABLES AND THEIR PURPOSE:
+- master_txn_table: Main transaction records for all business transactions
+- chart_of_accounts: Account names and their types for all businesses  
+- products_service: Products/services and their types used by businesses
+- customers: Customer records with billing/shipping details
+- vendors: Vendor records with billing address details
+- payment_method: Payment methods used by businesses
+- employees: Employee details including name, ID, hire date
+
+DATABASE SCHEMA (DDL):
+
+CREATE TABLE chart_of_accounts(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Account_name TEXT NOT NULL,
+  Account_type TEXT NOT NULL,
+  PRIMARY KEY(id,businessID,Account_name)
+);
+
+CREATE TABLE customers(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  customer_name TEXT NOT NULL,
+  customer_full_name TEXT,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Shipping_address TEXT,
+  Shipping_city TEXT,
+  Shipping_state TEXT,
+  Shipping_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Customer_name)
+);
+
+CREATE TABLE employees(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Employee_name TEXT NOT NULL,
+  Employee_ID TEXT,
+  Hire_date DATE,
+  Billing_rate DOUBLE,
+  Deleted TEXT,
+  PRIMARY KEY(id,businessID,Employee_name)
+);
+
+CREATE TABLE master_txn_table(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Transaction_ID INTEGER NOT NULL,
+  Transaction_DATE DATE NOT NULL,
+  Transaction_TYPE TEXT NOT NULL,
+  Amount DOUBLE NOT NULL,
+  CreatedDATE DATE NOT NULL,
+  CreatedUSER TEXT NOT NULL,
+  Account TEXT NOT NULL,
+  AR_paid TEXT,
+  AP_paid TEXT,
+  Due_DATE DATE,
+  Open_balance DOUBLE,
+  Customers TEXT,
+  Vendor TEXT,
+  Product_Service TEXT,
+  Quantity INTEGER,
+  Rate DOUBLE,
+  Credit DOUBLE,
+  Debit DOUBLE,
+  payment_method TEXT,
+  Misc TEXT,
+  FOREIGN KEY(businessID,Account) REFERENCES chart_of_accounts(businessID,Account_name),
+  FOREIGN KEY(businessID,Customers) REFERENCES customers(businessID,customer_name),
+  FOREIGN KEY(businessID,Vendor) REFERENCES vendors(businessID,Vendor_name),
+  FOREIGN KEY(businessID,Product_Service) REFERENCES products(businessID,Product_Service)
+);
+
+CREATE TABLE payment_method(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Payment_method TEXT,
+  Credit_card TEXT,
+  PRIMARY KEY(id,businessID,Payment_method)
+);
+
+CREATE TABLE products(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Product_Service TEXT NOT NULL,
+  Product_Service_type TEXT,
+  PRIMARY KEY(id,businessID,Product_Service)
+);
+
+CREATE TABLE vendors(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Vendor_name TEXT NOT NULL,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Vendor_name)
+);
+
+INSTRUCTIONS:
+Convert the user's natural language query into a valid SQL SELECT query. Return only the SQL query, no explanations or formatting.
+
+Do not add any Alias for final column names. 
+
+GENERATION GUIDELINES:
+- Use exact table and column names from the DATABASE SCHEMA. Do not invent columns.
+- Prefer master_txn_table for transaction-related questions (counts, sums, averages, invoices, balances). Use entity tables (customers, vendors, employees, etc.) only for static attributes (addresses, IDs, names).
+- Map parties correctly:
+  - Customer-focused questions -> filter on Customers
+  - Vendor-focused questions -> filter on Vendor
+- Use Transaction_TYPE to disambiguate business events:
+  - Invoices: Transaction_TYPE = 'invoice'
+  - Bills/vendor expenses: use the appropriate Transaction_TYPE if explicitly asked
+- Avoid double-counting: when aggregating per transaction, deduplicate by Transaction_ID.
+  - Counting transactions/invoices: use COUNT(DISTINCT Transaction_ID)
+  - Aggregating amounts (Amount, Open_balance): aggregate over a deduplicated set, e.g.
+    select sum(x) from (
+      select distinct Transaction_ID, x
+      from master_txn_table
+      where ...
+    )
+- For "average invoice" style questions, compute AVG(Amount) for rows where Transaction_TYPE = 'invoice' and apply deduplication by (Transaction_ID, Amount) to avoid repeated line items.
+- For "open credit/balance due" per customer, aggregate Open_balance from master_txn_table filtered by Customers = '<name>' with deduplication by Transaction_ID.
+- Do not add extra functions or filters (e.g., ABS(), x < 0) unless explicitly requested in the question.
+- Keep the query to a single SELECT statement without comments, CTEs, or aliases unless clearly required by the question.
@@ -0,0 +1,144 @@
+You are a SQL query generator for a business accounting database. Convert natural language queries to SQL queries.
+
+DATABASE CONTEXT:
+This is an accounting database (accounting.sqlite) containing business transaction and entity data.
+
+TABLES AND THEIR PURPOSE:
+- master_txn_table: Main transaction records for all business transactions
+- chart_of_accounts: Account names and their types for all businesses  
+- products_service: Products/services and their types used by businesses
+- customers: Customer records with billing/shipping details
+- vendors: Vendor records with billing address details
+- payment_method: Payment methods used by businesses
+- employees: Employee details including name, ID, hire date
+
+DATABASE SCHEMA (DDL):
+
+CREATE TABLE chart_of_accounts(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Account_name TEXT NOT NULL,
+  Account_type TEXT NOT NULL,
+  PRIMARY KEY(id,businessID,Account_name)
+);
+
+CREATE TABLE customers(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  customer_name TEXT NOT NULL,
+  customer_full_name TEXT,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Shipping_address TEXT,
+  Shipping_city TEXT,
+  Shipping_state TEXT,
+  Shipping_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Customer_name)
+);
+
+CREATE TABLE employees(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Employee_name TEXT NOT NULL,
+  Employee_ID TEXT,
+  Hire_date DATE,
+  Billing_rate DOUBLE,
+  Deleted TEXT,
+  PRIMARY KEY(id,businessID,Employee_name)
+);
+
+CREATE TABLE master_txn_table(
+  id INTEGER,
+  businessID INTEGER NOT NULL,
+  Transaction_ID INTEGER NOT NULL,
+  Transaction_DATE DATE NOT NULL,
+  Transaction_TYPE TEXT NOT NULL,
+  Amount DOUBLE NOT NULL,
+  CreatedDATE DATE NOT NULL,
+  CreatedUSER TEXT NOT NULL,
+  Account TEXT NOT NULL,
+  AR_paid TEXT,
+  AP_paid TEXT,
+  Due_DATE DATE,
+  Open_balance DOUBLE,
+  Customers TEXT,
+  Vendor TEXT,
+  Product_Service TEXT,
+  Quantity INTEGER,
+  Rate DOUBLE,
+  Credit DOUBLE,
+  Debit DOUBLE,
+  payment_method TEXT,
+  Misc TEXT,
+  FOREIGN KEY(businessID,Account) REFERENCES chart_of_accounts(businessID,Account_name),
+  FOREIGN KEY(businessID,Customers) REFERENCES customers(businessID,customer_name),
+  FOREIGN KEY(businessID,Vendor) REFERENCES vendors(businessID,Vendor_name),
+  FOREIGN KEY(businessID,Product_Service) REFERENCES products(businessID,Product_Service)
+);
+
+CREATE TABLE payment_method(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Payment_method TEXT,
+  Credit_card TEXT,
+  PRIMARY KEY(id,businessID,Payment_method)
+);
+
+CREATE TABLE products(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Product_Service TEXT NOT NULL,
+  Product_Service_type TEXT,
+  PRIMARY KEY(id,businessID,Product_Service)
+);
+
+CREATE TABLE vendors(
+  id INTEGER,
+  businessID TEXT NOT NULL,
+  Vendor_name TEXT NOT NULL,
+  Billing_address TEXT,
+  Billing_city TEXT,
+  Billing_state TEXT,
+  Billing_ZIP_code INTEGER,
+  Balance DOUBLE,
+  PRIMARY KEY(id,businessID,Vendor_name)
+);
+
+INSTRUCTIONS:
+Convert the user's natural language query into a valid SQL SELECT query. Return only the SQL query, no explanations or formatting.
+Do not add any Alias for final column names. The output column name must match what is expected. For example, `SELECT MAX(Transaction_DATE)` produces a column named `MAX(Transaction_DATE)`, while `SELECT Transaction_DATE ... ORDER BY Transaction_DATE DESC LIMIT 1` produces a column named `Transaction_DATE`.
+
+---
+
+### CORE QUERY GENERATION GUIDELINES
+
+1.  **Use Correct Schema**: Use exact table and column names from the DATABASE SCHEMA. Do not invent columns.
+2.  **Simplicity First**: Keep the query as simple as possible. Avoid subqueries or extra transformations unless absolutely necessary to prevent incorrect aggregation. Do not add filters that are not explicitly requested.
+3.  **Primary Table**: Prefer `master_txn_table` for all transaction-related questions (counts, sums, averages, invoices, balances). Use other tables like `customers` or `vendors` only for static attributes if a JOIN is needed.
+4.  **Deduplication**: When aggregating, be careful to avoid double-counting. A single transaction can have multiple rows.
+    -   Counting distinct transactions/invoices: `COUNT(DISTINCT Transaction_ID)`.
+    -   Aggregating financial values (e.g., `SUM`, `AVG`): Perform the aggregation over a deduplicated set of transactions if necessary. E.g., `SELECT SUM(Open_balance) FROM (SELECT DISTINCT Transaction_ID, Open_balance FROM master_txn_table WHERE ...)`
+
+### ADVANCED QUERY PATTERNS
+
+5.  **Financial Queries (Revenue, Sales, Expenses)**:
+    -   **Metric Selection**:
+        -   For revenue, income, sales, or money **received**: aggregate the `Credit` column.
+        -   For expenses, bills, or money **spent**: aggregate the `Debit` column.
+        -   Use the `Amount` column only when the query specifically asks for the "amount" of an invoice or transaction line item.
+    -   **Categorical Financial Queries**: For questions involving financial categories (e.g., "sales by X", "revenue from Y"), you **MUST** `JOIN` `master_txn_table` with `chart_of_accounts` on `master_txn_table.Account = chart_of_accounts.Account_name` and filter on `chart_of_accounts.Account_type` (e.g., 'Income', 'Other Income', 'Expense').
+
+6.  **Filtering Logic**:
+    -   **Ambiguous Parties**: For questions about transactions "with" or "involving" a person or company, you **MUST** check both `Customers` and `Vendor` columns. E.g., `WHERE Customers = 'Name' OR Vendor = 'Name'`.
+    -   **Avoid Extra Filters**: Do not add implicit filters. For example, do not assume all sales queries should be filtered by `Transaction_TYPE = 'invoice'`; other types like 'sales receipt' might be relevant.
+
+7.  **Column Selection and Naming**:
+    -   **Avoid `SELECT *`**: When asked to "show all transactions", return only `DISTINCT Transaction_ID` to avoid returning multiple rows for a single transaction. Do NOT use `SELECT *`.
+    -   **"Most Recent" / "Last" Queries**: To get the 'most recent' or 'last' record, use `ORDER BY Transaction_DATE DESC LIMIT 1`. This preserves the original column names in the output. Avoid using `MAX()` on a column if you need to return other columns from that same row.
+
+8.  **Specific Query Types**:
+    -   **Average Invoice**: Compute `AVG(Amount)` for `Transaction_TYPE = 'invoice'`. Apply deduplication by `(Transaction_ID, Amount)`.
+    -   **Open Balance**: Aggregate `SUM(Open_balance)` from `master_txn_table`, filtered by `Customers`, with deduplication by `Transaction_ID`.
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+Text-to-SQL Agent using OpenAI API.
+
+This agent converts natural language queries to SQL queries for database evaluation.
+"""
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+import dotenv
+from openai import AsyncOpenAI
+
+dotenv.load_dotenv(".env")
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+
+class Text2SQLAgent:
+    """
+    Text-to-SQL agent that converts natural language to SQL queries.
+
+    Features:
+    - Schema-aware query generation
+    - Configurable system prompts
+    """
+
+    def __init__(
+        self,
+        client,
+        model_name: str = "gpt-5-mini",
+        prompt_file: Optional[str] = None,
+    ):
+        """
+        Initialize the Text-to-SQL agent.
+
+        Args:
+            client: AsyncOpenAI client instance
+            model_name: Name of the model to use (default: gpt-5-mini)
+            prompt_file: Path to prompt file (default: prompt.txt)
+        """
+        self.client = client
+        self.model_name = model_name
+
+        # Load prompt
+        if prompt_file is None:
+            prompt_path = Path(__file__).parent / "prompt.txt"
+        else:
+            prompt_path = Path(prompt_file)
+
+        with open(prompt_path, "r", encoding="utf-8") as f:
+            self.system_prompt = f.read().strip()
+
+    async def query(self, question: str) -> Dict[str, Any]:
+        """
+        Generate SQL query from natural language input.
+
+        Args:
+            question: Natural language query to convert
+
+        Returns:
+            Dict with query, sql, and metadata
+        """
+        logger.info(f"Generating SQL for query: {question}")
+
+        try:
+            # Prepare messages
+            messages = [
+                {"role": "system", "content": self.system_prompt},
+                {"role": "user", "content": question},
+            ]
+
+            # Call OpenAI API
+            response = await self.client.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+            )
+
+            # Extract and clean generated SQL
+            generated_sql = response.choices[0].message.content.strip()
+            
+            # Remove markdown code blocks
+            generated_sql = generated_sql.replace("```sql", "").replace("```", "").strip()
+
+            logger.info(f"Successfully generated SQL ({len(generated_sql)} chars)")
+            return {
+                "query": question,
+                "sql": generated_sql
+            }
+
+        except Exception as e:
+            error_msg = f"Error: {e}"
+            logger.error(error_msg)
+            return {
+                "query": question,
+                "sql": f"-- ERROR: {error_msg}"
+            }
+
+
+# Demo
+async def main():
+    import os
+    from dotenv import load_dotenv
+    
+    # Load .env from root
+    load_dotenv(".env")
+    
+    # Configure logging for demo
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
+    
+    # Test query
+    test_query = "How much open credit does customer Andrew Bennett?"
+    
+    logger.info("TEXT-TO-SQL AGENT DEMO")
+    logger.info("=" * 40)
+    
+    # Create agent
+    logger.info("Creating Text-to-SQL agent...")
+    openai_client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])
+    agent = Text2SQLAgent(client=openai_client, model_name="gpt-5-mini")
+    
+    # Generate SQL
+    logger.info(f"Query: {test_query}")
+    result = await agent.query(test_query)
+    
+    logger.info(f"Generated SQL: {result['sql']}")
+
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+"""
+SQL Dataset Validation Script
+
+This script validates the Text-to-SQL dataset by executing each SQL query
+against the database and capturing results for manual verification.
+
+Usage:
+    python validate_sql_dataset.py
+    
+Output:
+    - validation_results.json: Detailed results for each query
+    - validation_summary.json: Summary statistics
+"""
+
+import csv
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List
+
+import pandas as pd
+
+# Import our database utilities
+from .db_utils import SQLiteDB, execute_sql
+
+
+def load_dataset(csv_path: str = "datasets/booksql_sample.csv") -> List[Dict[str, Any]]:
+    """
+    Load the SQL dataset from CSV file.
+    
+    Args:
+        csv_path: Path to the CSV file containing queries
+        
+    Returns:
+        List of dictionaries containing query data
+    """
+    dataset = []
+    csv_file = Path(csv_path)
+    
+    if not csv_file.exists():
+        raise FileNotFoundError(f"Dataset file not found: {csv_path}")
+    
+    with open(csv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f)
+        for i, row in enumerate(reader):
+            dataset.append({
+                'index': i,
+                'query': row['Query'].strip(),
+                'sql': row['SQL'].strip(),
+                'level': row['Levels'].strip(),
+                'split': row['split'].strip()
+            })
+    
+    return dataset
+
+
+def execute_and_validate_query(query_data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Execute a single SQL query and capture results.
+    
+    Args:
+        query_data: Dictionary containing query information
+        
+    Returns:
+        Dictionary with execution results
+    """
+    result = {
+        'index': query_data['index'],
+        'natural_language_query': query_data['query'],
+        'sql_query': query_data['sql'],
+        'difficulty_level': query_data['level'],
+        'dataset_split': query_data['split'],
+        'execution_success': False,
+        'execution_time': None,
+        'error_message': None,
+        'result_data': None,
+        'result_shape': None,
+        'result_columns': None
+    }
+    
+    # Record execution time
+    start_time = datetime.now()
+    
+    try:
+        # Execute the SQL query with case-insensitive string matching
+        success, query_result = execute_sql(query_data['sql'], case_insensitive=True)
+        
+        end_time = datetime.now()
+        result['execution_time'] = (end_time - start_time).total_seconds()
+        
+        if success and isinstance(query_result, pd.DataFrame):
+            result['execution_success'] = True
+            result['result_shape'] = list(query_result.shape)  # [rows, columns]
+            result['result_columns'] = list(query_result.columns)
+            
+            # Convert DataFrame to list of dictionaries for JSON serialization
+            # Limit to first 10 rows to keep output manageable
+            if len(query_result) > 10:
+                sample_data = query_result.head(10)
+                result['result_data'] = sample_data.to_dict('records')
+                result['result_truncated'] = True
+                result['total_rows'] = len(query_result)
+            else:
+                result['result_data'] = query_result.to_dict('records')
+                result['result_truncated'] = False
+                result['total_rows'] = len(query_result)
+            
+            # Classify result type for better reporting
+            if len(query_result) == 0:
+                result['result_type'] = 'empty'
+            elif len(query_result) > 0:
+                first_row = query_result.iloc[0]
+                # Check if all values in the first row are null/None
+                if all(pd.isna(value) or value is None for value in first_row):
+                    result['result_type'] = 'null_values'
+                else:
+                    result['result_type'] = 'has_data'
+            else:
+                result['result_type'] = 'has_data'
+        else:
+            result['execution_success'] = False
+            result['error_message'] = str(query_result)
+            result['result_type'] = 'failed'
+            
+    except Exception as e:
+        end_time = datetime.now()
+        result['execution_time'] = (end_time - start_time).total_seconds()
+        result['execution_success'] = False
+        result['error_message'] = f"Unexpected error: {str(e)}"
+        result['result_type'] = 'failed'
+    
+    return result
+
+
+def generate_summary_statistics(results: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Generate summary statistics from validation results.
+    
+    Args:
+        results: List of validation results
+        
+    Returns:
+        Dictionary containing summary statistics
+    """
+    total_queries = len(results)
+    successful_queries = sum(1 for r in results if r['execution_success'])
+    failed_queries = total_queries - successful_queries
+    
+    # Count by result type
+    result_type_counts = {
+        'has_data': sum(1 for r in results if r.get('result_type') == 'has_data'),
+        'null_values': sum(1 for r in results if r.get('result_type') == 'null_values'),
+        'empty': sum(1 for r in results if r.get('result_type') == 'empty'),
+        'failed': sum(1 for r in results if r.get('result_type') == 'failed')
+    }
+    
+    # Group by difficulty level
+    level_stats = {}
+    for result in results:
+        level = result['difficulty_level']
+        if level not in level_stats:
+            level_stats[level] = {
+                'total': 0, 'successful': 0, 'failed': 0,
+                'has_data': 0, 'null_values': 0, 'empty': 0
+            }
+        
+        level_stats[level]['total'] += 1
+        if result['execution_success']:
+            level_stats[level]['successful'] += 1
+        else:
+            level_stats[level]['failed'] += 1
+        
+        # Count by result type for this level
+        result_type = result.get('result_type', 'unknown')
+        if result_type in level_stats[level]:
+            level_stats[level][result_type] += 1
+    
+    # Calculate success rates
+    for level in level_stats:
+        total = level_stats[level]['total']
+        successful = level_stats[level]['successful']
+        level_stats[level]['success_rate'] = successful / total if total > 0 else 0
+    
+    # Common error types
+    error_types = {}
+    for result in results:
+        if not result['execution_success'] and result['error_message']:
+            # Extract first part of error message as error type
+            error_type = result['error_message'].split(':')[0]
+            error_types[error_type] = error_types.get(error_type, 0) + 1
+    
+    # Average execution time
+    execution_times = [r['execution_time'] for r in results if r['execution_time'] is not None]
+    avg_execution_time = sum(execution_times) / len(execution_times) if execution_times else 0
+    
+    summary = {
+        'validation_timestamp': datetime.now().isoformat(),
+        'total_queries': total_queries,
+        'successful_queries': successful_queries,
+        'failed_queries': failed_queries,
+        'overall_success_rate': successful_queries / total_queries if total_queries > 0 else 0,
+        'average_execution_time_seconds': avg_execution_time,
+        'result_type_counts': result_type_counts,
+        'statistics_by_difficulty': level_stats,
+        'common_error_types': error_types,
+        'sample_successful_queries': [
+            r['index'] for r in results if r['execution_success']
+        ][:5],  # First 5 successful queries
+        'sample_failed_queries': [
+            r['index'] for r in results if not r['execution_success']
+        ][:5]   # First 5 failed queries
+    }
+    
+    return summary
+
+
+def main():
+    """Main validation script."""
+    print("🔍 Starting SQL Dataset Validation...")
+    print("=" * 50)
+    
+    # Load dataset
+    try:
+        dataset = load_dataset("datasets/booksql_sample.csv")
+        print(f"📊 Loaded {len(dataset)} queries from dataset")
+    except FileNotFoundError as e:
+        print(f"❌ Error: {e}")
+        return
+    except Exception as e:
+        print(f"❌ Unexpected error loading dataset: {e}")
+        return
+    
+    # Validate database connection
+    print("🔗 Testing database connection...")
+    db = SQLiteDB()
+    success, message = db.connect()
+    if not success:
+        print(f"❌ Database connection failed: {message}")
+        print("💡 Make sure the BookSQL database is available at: BookSQL-files/BookSQL/accounting.sqlite")
+        return
+    
+    # Get database info
+    success, tables = db.get_table_names()
+    if success:
+        print(f"✅ Database connected. Found tables: {tables}")
+    db.disconnect()
+    
+    # Execute all queries
+    print(f"\n🚀 Executing {len(dataset)} SQL queries...")
+    results = []
+    
+    for i, query_data in enumerate(dataset):
+        print(f"Processing query {i+1}/{len(dataset)}: {query_data['level']} level", end=" ... ")
+        
+        result = execute_and_validate_query(query_data)
+        results.append(result)
+        
+        if result['execution_success']:
+            print("✅")
+        else:
+            print("❌")
+    
+    # Generate summary
+    print("\n📈 Generating summary statistics...")
+    summary = generate_summary_statistics(results)
+    
+    # Save results
+    print("💾 Saving validation results...")
+    
+    # Save detailed results
+    with open('validation_results.json', 'w', encoding='utf-8') as f:
+        json.dump(results, f, indent=2, ensure_ascii=False)
+    
+    # Save summary
+    with open('validation_summary.json', 'w', encoding='utf-8') as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+    
+    # Print summary to console
+    print("\n" + "=" * 50)
+    print("📊 VALIDATION SUMMARY")
+    print("=" * 50)
+    print(f"Total Queries: {summary['total_queries']}")
+    print(f"Successful: {summary['successful_queries']} ({summary['overall_success_rate']:.1%})")
+    print(f"Failed: {summary['failed_queries']}")
+    print(f"Average Execution Time: {summary['average_execution_time_seconds']:.3f}s")
+    
+    print("\n📈 Result Type Distribution:")
+    result_counts = summary['result_type_counts']
+    total = summary['total_queries']
+    print(f"  ✅ Has Data: {result_counts['has_data']}/{total} ({result_counts['has_data']/total:.1%})")
+    print(f"  🔍 NULL Values: {result_counts['null_values']}/{total} ({result_counts['null_values']/total:.1%})")
+    print(f"  📭 Empty Results: {result_counts['empty']}/{total} ({result_counts['empty']/total:.1%})")
+    print(f"  ❌ Failed: {result_counts['failed']}/{total} ({result_counts['failed']/total:.1%})")
+    
+    print("\n📈 Success Rate by Difficulty:")
+    for level, stats in summary['statistics_by_difficulty'].items():
+        print(f"  {level.capitalize()}: {stats['successful']}/{stats['total']} ({stats['success_rate']:.1%})")
+        print(f"    ✅ Data: {stats['has_data']}, 🔍 NULL: {stats['null_values']}, 📭 Empty: {stats['empty']}, ❌ Failed: {stats['failed']}")
+    
+    if summary['common_error_types']:
+        print("\n⚠️  Common Error Types:")
+        for error_type, count in sorted(summary['common_error_types'].items(), 
+                                       key=lambda x: x[1], reverse=True)[:5]:
+            print(f"  {error_type}: {count} occurrences")
+    
+    print("\n💾 Detailed results saved to:")
+    print("  - validation_results.json (detailed results)")
+    print("  - validation_summary.json (summary statistics)")
+    
+    if summary['failed_queries'] > 0:
+        print("\n🔍 Review failed queries in validation_results.json")
+        print("💡 Check if database schema matches expected tables/columns")
+
+
+if __name__ == "__main__":
+    main()