chore: import upstream snapshot with attribution

examples/eval-image-classification/README.md

@@ -0,0 +1,50 @@
+# eval-image-classification (Image Classification Example with Promptfoo)
+
+You can run this example with:
+
+```bash
+npx promptfoo@latest init --example eval-image-classification
+cd eval-image-classification
+```
+
+This example demonstrates how to use Promptfoo for image classification tasks using the Fashion MNIST dataset. The example uses GPT-4o and GPT-4o-mini with a structured json schema to analyze images, including classification, color analysis, and additional attributes.
+
+## Getting Started
+
+1. Set up your OpenAI API key:
+
+   ```sh
+   export OPENAI_API_KEY='your-api-key'
+   ```
+
+2. Run the evaluation:
+
+   ```sh
+   npx promptfoo@latest eval
+   ```
+
+3. View the results:
+
+   ```sh
+   npx promptfoo@latest view
+   ```
+
+4. Optionally, re-generate or update the dataset:
+
+   ```sh
+   python dataset_gen.py
+   ```
+
+   Note: You may need to install dependencies with:
+
+   ```sh
+   pip install -r requirements.txt
+   ```
+
+   This script creates a CSV file with 100 random images from the Fashion MNIST dataset and their labels. A CSV with 10 sample images is included so you can skip this step if preferred.
+
+5. Experiment with the configuration:
+   - Modify the JSON schema in `promptfooconfig.yaml` to add or adjust required fields
+   - Try different models such as llama3.2 or Claude Sonnet 4.6 by changing the provider in the config
+   - Adjust the system prompt to improve classification accuracy
+   - Add additional assertions to validate model outputs

examples/eval-image-classification/dataset_gen.py

@@ -0,0 +1,117 @@
+"""
+This script downloads the Fashion MNIST dataset, processes a specified number of samples,
+and saves them to a CSV file. Each row in the CSV file contains the original dataset index,
+the class label name, and the image encoded as a base64 string.
+
+The Fashion MNIST dataset is a collection of 70,000 grayscale images of 28x28 pixels,
+each depicting one of 10 types of clothing. For more information on the dataset, see:
+https://www.tensorflow.org/api_docs/python/tf/keras/datasets/fashion_mnist/load_data
+
+Usage:
+    python save_fashion_mnist_to_csv.py --num_samples <num_samples> --filename <filename>
+
+Arguments:
+    --num_samples: Number of samples to save (default: 100)
+    --filename: Output CSV file name (default: fashion_mnist_sample_base64.csv)
+"""
+
+import base64
+import io
+from typing import List
+
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from PIL import Image
+
+# set np seed for reproducibility
+np.random.seed(0)
+
+
+def get_class_names() -> dict[int, str]:
+    """Retrieves the class names for the Fashion MNIST dataset.
+
+    Returns:
+        A dictionary mapping class indices to class names.
+    """
+    return {
+        0: "T-shirt/top",
+        1: "Trouser",
+        2: "Pullover",
+        3: "Dress",
+        4: "Coat",
+        5: "Sandal",
+        6: "Shirt",
+        7: "Sneaker",
+        8: "Bag",
+        9: "Ankle boot",
+    }
+
+
+def image_to_base64(image: np.ndarray) -> str:
+    """Converts an image to a base64 encoded string.
+
+    Args:
+        image: A numpy array representing the image.
+
+    Returns:
+        A base64 encoded string of the image.
+    """
+    buffered = io.BytesIO()
+    pil_image = Image.fromarray(image)
+    # NOTE: For a dataset with large images, you can resize it here to save
+    # costs on the inference side.
+    # pil_image = pil_image.resize((32, 32))
+    pil_image.save(buffered, format="jpeg")
+    return base64.b64encode(buffered.getvalue()).decode("utf-8")
+
+
+def save_fashion_mnist_sample_to_csv(num_samples: int, filename: str) -> None:
+    """Saves a sample of the Fashion MNIST dataset to a CSV file.
+
+    Args:
+        num_samples: The number of samples to save.
+        filename: The name of the output CSV file.
+    """
+    # Load the Fashion MNIST dataset
+    fashion_mnist = tf.keras.datasets.fashion_mnist
+    (train_images, train_labels), _ = fashion_mnist.load_data()
+
+    class_names = get_class_names()
+
+    # Randomly sample indices without replacement
+    sample_indices = np.random.choice(len(train_images), num_samples, replace=False)
+
+    # Convert images to base64 and combine with labels and indices
+    data: List[List] = []
+    for sample_index in sample_indices:
+        base64_image = image_to_base64(train_images[sample_index])
+        label_index = train_labels[sample_index]
+        label_name = class_names[label_index]
+        data.append([sample_index, label_name, base64_image])
+
+    pd.DataFrame(data, columns=["index", "label", "image_base64"]).sort_values(
+        by=["label", "index"]
+    ).to_csv(filename, index=False)
+    print(f"CSV file '{filename}' created successfully.")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Save Fashion MNIST samples to a CSV file."
+    )
+    parser.add_argument(
+        "--num_samples", type=int, default=100, help="Number of samples to save"
+    )
+    parser.add_argument(
+        "--filename",
+        type=str,
+        default="fashion_mnist_sample_base64.csv",
+        help="Output CSV file name",
+    )
+
+    args = parser.parse_args()
+
+    save_fashion_mnist_sample_to_csv(args.num_samples, args.filename)

examples/eval-image-classification/fashion_mnist_sample_base64.csv

@@ -0,0 +1,11 @@
+index,label,image_base64
+56864,Ankle boot,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+iunuvh54ptrG1vU0me6t7mNJEa0XzSNy7sMq/Mpx6jHoTXP3dldWEwhvLeSCXaG8uVSrAHpkHkVBiuy8GfDrWPFV5byvaz2+ksw8y7ZdoZe4jz949uMgd6+oreyRHUlESJIxGkZH3QOpz17Ba+dPjTp0qfEGSeK3kK3FtFJvCk7sZTP/AI7j8K6/wz44+HUNjaBkh02WBVVY7mxMhUjHzBkVucjOeCTzXcR/ETwcuc+IdNOe++QHH/fNK3xF8JspA8S6cM453N0Hb7tch4m8deB7jUYmlubW/ZYtvmCKRsDc3Gdv4/jXz7RRRX//2Q==
+19563,Pullover,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/AOE+Fnhaw1qW+1DUbYXMdptEUTn5C3JJYd8ADjpzzXsDWOnS26W7aRYywrjbEbVdqgHIxjBAz6YpdStrTWYRb6lY2t2pG1VlgU7B/snGU/4Divm/xPpiaN4n1LTo/wDVQXDLGM5wucj9CK9a+Dwjg8JXEsxCxy3UwLH08tB/M13NvKsO5JoVkX6cg4I6/wBKS3YzXykhVBbO3G0Y9BXgvxPj8v4iarxgMYn+uYkNdx4Nf7L8K1dfvS3QUfV544/5V6FKh82TA43tz+JqhqDPbrZSqxUrqNlnnsbhFI/I1498WVx46mf+/bxH8lx/Ssm18YalZ+G4dIgW3WCC5W4WTaS5YNuAOTjGQOMdq6AfGLxD1NlpbHOSTFIM/k9Rz/FHXNX8qylttPhiknhLGGJg3ySK4wSxxyoqn8T7p7rxrMXVV2RIgC56cnufev/Z
+21860,Pullover,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APENItY555HljEiRoW2EkZOCe3sDXWWeualpkRg0zULmwiZtzx2kpiVjjGSBxnHfrVS/8vVBJJdxie5fLPcNnzWb1LZ5P1zXGkYOD1roPDEcJu43uP8Aj3MoWbv8m1t36HP4VM0bB9pOSMAn3A5q5aqfs7kdTGw/Jgf5Vyl8oTULlB0WVh+prd0ALFp1xckA7UuAc/8AXAgfqRWi9qRcYb7ytyffNWbaFkQpgciRcf8AbJj/AOy1xepf8hS7/wCuz/8AoRrQ0didM1RD90Qlh9cH/AV0sV615KJpIowzDeducZ6+tTpcGXUbK3McYV53YkA54hcY69OTXBaic6ndk95n/ma//9k=
+31488,Pullover,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APL/AIZ+GrbxT4yhsr1PMtIonnljyRvC4AXI5AyRX0jb6bYWVqlnb6bZR20f3YRbIVU+oBHX1PU1jeLfBml+K9KuI57GEXyQN9nukjCuhAO0ZGMrn+E8c8Yr5Xr1n9n6383xpqEuOE08qP8AeaWPH8jXvMrK0shQ5QuxU+ozxVmxVXkcNjOU/LPP9K+J5omhnkib7yMVP1BxXrfwIm+yXPiO9xzb2av/AN87n/8AZa9r04SS6VZSbSd9tE2cHnKKabLcSWev6UjJhbhbiLJyPmCBxx6YjNfI2tp5evain926lH/jxruvhNqc+nR+JVhWM+Zp7E7wTghWAI5/2jXBDVNQVFRb65CqoVQJmwABgAc12Hww1i9i+IOnyvK058uZcTszAAxnPf2FcjrEzXGuX87gB5LmR2x0yWJr/9k=
+40228,Pullover,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APBrKzuNQvoLO0iaa4nkWOKNerMTgD86+t/BHhe20Hwlp9pcaVp8V6LdftZESMzSd9zD73pnJHpXjXxx8Ky6d4gi1mz02GDS54kjeSBFRfPy5IKjuQAc45/CvJq774MWpufilpTBQwhWaUgjPSNgP1Ir6itriK5t4ri3ffDMgkjb1UjIP5Vw/wAXFjuvhrrtsrebcWwt5m3DlQZVIPH+zur5Xr0v4Fv5XxAkmxkxafO4+oxX0NoVm1j4e0mzYYa3soIm+qxqD+ua5f4g27LoHjCVjmOfR7Yr7MjzA/oUr5WrtPhffy6d4uaaEKWNnOpDZxjb7EelaEXxr8YQwJEJbE7VC7jark4/Sr0XxA13xL4Q8Vf2k9u+LWKJfLi2bQXJPTr+Oa8tr//Z
+845,Sandal,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+iiitjSdDF5NO2oSy2VrbwmZ28ks8nIVUjUkbmJYd+BkngGtzTvBml63ZEafr6warj5LDUIRAZD6K5bB/DPviuYv9J1DTL6SzvLSWK4jOGQrn+XBHuKrta3CY3QSrkZGUIq5Fr+swKFi1a+RV4AW4YAfTmpT4o15lKtq96ynqGmJB/OlTxVr8aBI9XvEQdFSUgfpVWfV9SuWDTahdSMBtBaVjgdcdfc1Sooor/9k=
+8870,Sandal,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+iiirml6dJquoJZxSwxMyu2+ZtqKFUscnB7A10R8GiwngfUNQto4CUd0aWNZHTqcRlvMGRnGUGePWumv/AA94cuo4r2CSbUVh84y3CwNaxyIF3DzMgMGVTncF+cgj6eY3cKW15NBHcRXCRuUWaLOyQA43LkA4PXkCu20z4pX1qsK3+kabfmFdqyPGUk6YySOM49q05PjLNFJLPp/hrTLa6mcvNO5ZmkY9SSu0k/Umra/HbUBCFbRoS+3DYuZAp/4D6e2a5nVPifrup3Qn8uzgwu0LHESOpOTknnmuLooor//Z
+25770,Sandal,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+pIYJbh9kMTyOAW2opJwBknjsBzUdPlhlt5TFNG8ci9UdSCPwNXtI0LVNeuvs2l2M91KOojXhfdj0Ue5Iru/+Faa7odsb/R9YSTW7NRLNa2hIZUbONjfx5APGAGwQCSMHqPDNlcan4ftvEeoaFaXmriN3tvs0KRPKucAuT8oYkE7sfKORya8+8Q+HPFGqavJeahbWEdxIBlIryABQOAPvknp3JPvU+i/EuXQ4Zo7PQdLiE2zzRGsihyoIBOH7ZP5mtKf4xXc0kcq6Hp0c0asqS/vWZA2NwUl+AcA4Oeeap3HxTvbuQyXFjbSSN1eQOzfmXrntZ8V3urXaTMI02R+WAqY4yT6n1rAooor/2Q==
+58303,Shirt,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APOvhT4TtvFPid1v4jJZWkQlZCcK7lgqqfzJx/smvVdX+GvgzWrpJngksFQeUp0+RIUbHI3Ao2WwTkjn5e/UyXnwv8OXXhmTTbLSY0mjhbybksfOWXHys8mBuBJHBGMHgDINfNde2/BDytK8NeI9dlZHSL5niJOQkMbOxwOud/Hutek6fcO2k2JvZlNy9pCZJJHBJkMQLHaVJ4JHRSB7c4RbqSz8Uadavjyr61nIjRSuZIGikBHc/u2cAdwOc5JPzF4rsl07xbq9ohQpFdyBNhyNu4kc8dsVQ0+CS71C3tYmKyXEiwgj/aO3+tfXMrwxs0bSRIBlFdmQA4GM7dvUYTop7DIywHnPxmspf+EJs7tQqGC+UExoR8rxsDhiMkfKo9wQTnOa8DqxYXsunaja30IQy20yTIJF3KWUgjI7jI6V7X4d+JmrazbTNcafpqMkRlBiSRQWyRyN+O3pXG+OPiHqurQXegvaafBaGVdzQxN5jBT8oLMx9ByME45JHFefV//Z
+3048,T-shirt/top,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/AOM+EngCw8WyXt/qqSS2tqyxxwKxUSuwJOSOcAAcDqSPofXWbQ7O3S0h0bRYLRFEqQ3UCKzj5l3rFsJ+8ACzc43egpW0rw54tto7S+0XTriPBWGe1A2HbjKxyIFZe5x0xjucV87eOfDS+EvF97pEcrSwR7XhkbqyMoYZ9xnB9waPD3jnxH4Vtnt9G1E20LyGRkMMbgtgDPzKewFfT95DcahJK8UsZtLhQwWRdzFSAQMnJA6HjFY3jTXdY8N+DNV1hLuNr5DCISUBVcyAEbTwcqW7D1r5s8QeI9U8UaiL/V7kT3CxiJWEaphQSQMKB6n3rKr7Pt1AtoVxjEacf8BFcX8YIy/w01JgfuSQE/8AfwD+tfMlFfSEXxE1Ihs2Gn8DjiXjj/frlfHPjfUNa8EX1pPa2cUcrRhjEJM8OD3c+leM1//Z

examples/eval-image-classification/prompt.js

@@ -0,0 +1,78 @@
+const dedent = require('dedent');
+
+/**
+ * Detects MIME type from base64 magic numbers
+ *
+ * Note: This is duplicated from src/evaluatorHelpers.ts to keep the example
+ * self-contained and runnable without dependencies on core promptfoo internals.
+ *
+ * Magic numbers (base64-encoded file signatures):
+ * - JPEG: /9j/
+ * - PNG: iVBORw0KGgo
+ * - GIF: R0lGODlh or R0lGODdh
+ * - WebP: UklGR (RIFF)
+ * - BMP: Qk0 or Qk1
+ * - TIFF: SUkq or TU0A
+ * - ICO: AAABAA
+ */
+function getMimeTypeFromBase64(base64Data) {
+  if (base64Data.startsWith('/9j/')) {
+    return 'image/jpeg';
+  } else if (base64Data.startsWith('iVBORw0KGgo')) {
+    return 'image/png';
+  } else if (base64Data.startsWith('R0lGODlh') || base64Data.startsWith('R0lGODdh')) {
+    return 'image/gif';
+  } else if (base64Data.startsWith('UklGR')) {
+    return 'image/webp';
+  } else if (base64Data.startsWith('Qk0') || base64Data.startsWith('Qk1')) {
+    return 'image/bmp';
+  } else if (base64Data.startsWith('SUkq') || base64Data.startsWith('TU0A')) {
+    return 'image/tiff';
+  } else if (base64Data.startsWith('AAABAA')) {
+    return 'image/x-icon';
+  }
+  // Fallback to JPEG if no magic number is matched
+  return 'image/jpeg';
+}
+
+module.exports = (context) => {
+  return [
+    {
+      role: 'system',
+      content: dedent`
+        You are an AI assistant tasked with analyzing and classifying images. Your goal is to determine the type of clothing item depicted in the image and provide additional relevant information.
+
+        Please analyze the image and provide:
+        1. Classification (must be one of: T-shirt/top, Trouser, Pullover, Dress, Coat, Sandal, Shirt, Sneaker, Bag, Ankle boot)
+        2. Primary color or color scheme
+        3. Notable features or patterns
+        4. Approximate style or era (e.g., modern, vintage, classic)
+        5. Confidence level (1-10, where 1 is least confident and 10 is most confident)
+        6. Brief reasoning for the classification
+
+        Provide your response as a JSON object with the following structure:
+        {
+          "classification": string (one of the allowed categories),
+          "color": string,
+          "features": string,
+          "style": string,
+          "confidence": number (1-10),
+          "reasoning": string
+        }
+      `,
+    },
+    {
+      role: 'user',
+      content: [
+        {
+          type: 'image_url',
+          image_url: {
+            url: `data:${getMimeTypeFromBase64(context.vars.image_base64)};base64,${
+              context.vars.image_base64
+            }`,
+          },
+        },
+      ],
+    },
+  ];
+};

examples/eval-image-classification/promptfooconfig.yaml

@@ -0,0 +1,72 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Image Classification Example of Fashion MNIST dataset
+providers:
+  - openai:chat:gpt-4o
+  - openai:chat:gpt-4.1-mini
+prompts:
+  - label: Image Classification
+    raw: file://prompt.js
+    config:
+      response_format:
+        type: json_schema
+        json_schema:
+          name: image_classification
+          schema:
+            type: object
+            properties:
+              classification:
+                type: string
+                enum:
+                  [
+                    'T-shirt/top',
+                    'Trouser',
+                    'Pullover',
+                    'Dress',
+                    'Coat',
+                    'Sandal',
+                    'Shirt',
+                    'Sneaker',
+                    'Bag',
+                    'Ankle boot',
+                  ]
+              color:
+                type: string
+              features:
+                type: string
+              style:
+                type: string
+              confidence:
+                type: integer
+              reasoning:
+                type: string
+            required:
+              - classification
+              - color
+              - features
+              - style
+              - confidence
+              - reasoning
+            additionalProperties: false
+defaultTest:
+  assert:
+    - type: is-json
+      value:
+        type: object
+        properties:
+          classification:
+            type: string
+          color:
+            type: string
+          features:
+            type: string
+          style:
+            type: string
+          confidence:
+            type: integer
+          reasoning:
+            type: string
+    - type: javascript
+      value: 'output.classification === context.vars.label'
+      metric: accuracy
+
+tests: file://fashion_mnist_sample_base64.csv

examples/eval-image-classification/requirements.txt

@@ -0,0 +1,4 @@
+numpy==1.26.4
+pandas==2.3.3
+pillow==12.2.0
+tensorflow==2.20.0