import requests import pytest import hypothesis.extra.numpy as npst import hypothesis.strategies as st import io import numpy as np from hypothesis import given, settings from cleanlab.dataset import ( health_summary, find_overlapping_classes, rank_classes_by_label_quality, overall_label_health_score, ) from cleanlab.count import estimate_joint, num_label_issues, compute_confident_joint cifar100 = [ "apple", "aquarium_fish", "baby", "bear", "beaver", "bed", "bee", "beetle", "bicycle", "bottle", "bowl", "boy", "bridge", "bus", "butterfly", "camel", "can", "castle", "caterpillar", "cattle", "chair", "chimpanzee", "clock", "cloud", "cockroach", "couch", "crab", "crocodile", "cup", "dinosaur", "dolphin", "elephant", "flatfish", "forest", "fox", "girl", "hamster", "house", "kangaroo", "keyboard", "lamp", "lawn_mower", "leopard", "lion", "lizard", "lobster", "man", "maple_tree", "motorcycle", "mountain", "mouse", "mushroom", "oak_tree", "orange", "orchid", "otter", "palm_tree", "pear", "pickup_truck", "pine_tree", "plain", "plate", "poppy", "porcupine", "possum", "rabbit", "raccoon", "ray", "road", "rocket", "rose", "sea", "seal", "shark", "shrew", "skunk", "skyscraper", "snail", "snake", "spider", "squirrel", "streetcar", "sunflower", "sweet_pepper", "table", "tank", "telephone", "television", "tiger", "tractor", "train", "trout", "tulip", "turtle", "wardrobe", "whale", "willow_tree", "wolf", "woman", "worm", ] caltech256 = [ "ak47", "american-flag", "backpack", "baseball-bat", "baseball-glove", "basketball-hoop", "bat", "bathtub", "bear", "beer-mug", "billiards", "binoculars", "birdbath", "blimp", "bonsai", "boom-box", "bowling-ball", "bowling-pin", "boxing-glove", "brain", "breadmaker", "buddha", "bulldozer", "butterfly", "cactus", "cake", "calculator", "camel", "cannon", "canoe", "car-tire", "cartman", "cd", "centipede", "cereal-box", "chandelier", "chess-board", "chimp", "chopsticks", "cockroach", "coffee-mug", "coffin", "coin", "comet", "computer-keyboard", "computer-monitor", "computer-mouse", "conch", "cormorant", "covered-wagon", "cowboy-hat", "crab", "desk-globe", "diamond-ring", "dice", "dog", "dolphin", "doorknob", "drinking-straw", "duck", "dumb-bell", "eiffel-tower", "electric-guitar", "elephant", "elk", "ewer", "eyeglasses", "fern", "fighter-jet", "fire-extinguisher", "fire-hydrant", "fire-truck", "fireworks", "flashlight", "floppy-disk", "football-helmet", "french-horn", "fried-egg", "frisbee", "frog", "frying-pan", "galaxy", "gas-pump", "giraffe", "goat", "golden-gate-bridge", "goldfish", "golf-ball", "goose", "gorilla", "grand-piano", "grapes", "grasshopper", "guitar-pick", "hamburger", "hammock", "harmonica", "harp", "harpsichord", "hawksbill", "head-phones", "helicopter", "hibiscus", "homer-simpson", "horse", "horseshoe-crab", "hot-air-balloon", "hot-dog", "hot-tub", "hourglass", "house-fly", "human-skeleton", "hummingbird", "ibis", "ice-cream-cone", "iguana", "ipod", "iris", "jesus-christ", "joy-stick", "kangaroo", "kayak", "ketch", "killer-whale", "knife", "ladder", "laptop", "lathe", "leopards", "license-plate", "lightbulb", "light-house", "lightning", "llama", "mailbox", "mandolin", "mars", "mattress", "megaphone", "menorah", "microscope", "microwave", "minaret", "minotaur", "motorbikes", "mountain-bike", "mushroom", "mussels", "necktie", "octopus", "ostrich", "owl", "palm-pilot", "palm-tree", "paperclip", "paper-shredder", "pci-card", "penguin", "people", "pez-dispenser", "photocopier", "picnic-table", "playing-card", "porcupine", "pram", "praying-mantis", "pyramid", "raccoon", "radio-telescope", "rainbow", "refrigerator", "revolver", "rifle", "rotary-phone", "roulette-wheel", "saddle", "saturn", "school-bus", "scorpion", "screwdriver", "segway", "self-propelled-lawn-mower", "sextant", "sheet-music", "skateboard", "skunk", "skyscraper", "smokestack", "snail", "snake", "sneaker", "snowmobile", "soccer-ball", "socks", "soda-can", "spaghetti", "speed-boat", "spider", "spoon", "stained-glass", "starfish", "steering-wheel", "stirrups", "sunflower", "superman", "sushi", "swan", "swiss-army-knife", "sword", "syringe", "tambourine", "teapot", "teddy-bear", "teepee", "telephone-box", "tennis-ball", "tennis-court", "tennis-racket", "theodolite", "toaster", "tomato", "tombstone", "top-hat", "touring-bike", "tower-pisa", "traffic-light", "treadmill", "triceratops", "tricycle", "trilobite", "tripod", "t-shirt", "tuning-fork", "tweezer", "umbrella", "unicorn", "vcr", "video-projector", "washing-machine", "watch", "waterfall", "watermelon", "welding-mask", "wheelbarrow", "windmill", "wine-bottle", "xylophone", "yarmulke", "yo-yo", "zebra", "airplanes", "car-side", "faces-easy", "greyhound", "tennis-shoes", "toad", ] imdb = ["Negative", "Positive"] mnist = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] urls = { "caltech256": [ "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d/" "original_test_labels/caltech256_original_labels.npy", "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/cross_validated_predicted_probabilities/caltech256_pyx.npy", ], "mnist": [ "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/original_test_labels/mnist_test_set_original_labels.npy", "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/cross_validated_predicted_probabilities/mnist_test_set_pyx.npy", ], "imdb": [ "https://github.com/cleanlab/label-errors/raw" "/5392f6c71473055060be3044becdde1cbc18284d/original_test_labels" "/imdb_test_set_original_labels.npy", "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/cross_validated_predicted_probabilities/imdb_test_set_pyx.npy", ], "cifar100": [ "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/original_test_labels/cifar100_test_set_original_labels.npy", "https://github.com/cleanlab/label-errors/raw/5392f6c71473055060be3044becdde1cbc18284d" "/cross_validated_predicted_probabilities/cifar100_test_set_pyx.npy", ], } def _get_pred_probs_labels_from_labelerrors_datasets(dataset_name): """Helper function to load data from the labelerrors.com datasets.""" labels_url, pred_probs_url = urls[dataset_name] response = requests.get(pred_probs_url) response.raise_for_status() pred_probs = np.load(io.BytesIO(response.content), allow_pickle=True) response = requests.get(labels_url) response.raise_for_status() labels = np.load(io.BytesIO(response.content), allow_pickle=True) return pred_probs, labels @pytest.mark.parametrize("dataset_name", ["mnist", "caltech256", "cifar100"]) def test_real_datasets(dataset_name): print("\n" + dataset_name.capitalize() + "\n") class_names = eval(dataset_name) pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) # if this runs without issue no all four datasets, the test passes _ = health_summary( pred_probs=pred_probs, labels=labels, class_names=class_names, verbose=dataset_name != "mnist", # test out verbose=False on one of the datasets. ) @pytest.mark.parametrize("dataset_name", ["mnist"]) def test_multilabel_error(dataset_name): print("\n" + dataset_name.capitalize() + "\n") class_names = eval(dataset_name) pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) # if this runs without issue no all four datasets, the test passes with pytest.raises(ValueError) as e: _ = find_overlapping_classes(labels=labels, pred_probs=pred_probs, multi_label=True) @pytest.mark.parametrize("asymmetric", [True, False]) @pytest.mark.parametrize("dataset_name", ["mnist", "imdb"]) def test_symmetry_df_size(asymmetric, dataset_name): pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) joint = estimate_joint(labels=labels, pred_probs=pred_probs) num_classes = pred_probs.shape[1] df = find_overlapping_classes( joint=joint, asymmetric=asymmetric, class_names=eval(dataset_name), num_examples=len(labels), ) if asymmetric: assert len(df) == num_classes**2 - num_classes else: # symmetric assert len(df) == (num_classes**2 - num_classes) / 2 # Second test for symmetric # check that the row, col value returned is actually the sum from the joint. sum_0_1 = joint[0, 1] + joint[1, 0] df_0_1 = df[(df["Class Index A"] == 0) & (df["Class Index B"] == 1)]["Joint Probability"] assert sum_0_1 - df_0_1.values[0] < 1e-8 # Check two floats are equal @pytest.mark.parametrize("use_num_examples", [True, False]) @pytest.mark.parametrize("use_labels", [True, False]) @pytest.mark.parametrize( "func", [find_overlapping_classes, rank_classes_by_label_quality, overall_label_health_score] ) def test_value_error_missing_num_examples_with_joint(use_num_examples, use_labels, func): dataset_name = "imdb" pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) joint = estimate_joint(labels=labels, pred_probs=pred_probs) if use_num_examples is False and use_labels is False: # can't infer num_examples. Throw error! with pytest.raises(ValueError) as e: df = func( labels=labels if use_labels else None, joint=joint, num_examples=len(labels) if use_num_examples else None, ) else: # at least one of use_num_examples and use_labels must be True. Can infer num_examples. # If this runs without error, the test passes. df = func( labels=labels if use_labels else None, joint=joint, num_examples=len(labels) if use_num_examples else None, ) @pytest.mark.parametrize("dataset_name", ["mnist", "caltech256", "cifar100"]) def test_overall_label_health_score_matched_num_issues(dataset_name): # Matches num_label_issues pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) num_issues = num_label_issues(labels=labels, pred_probs=pred_probs) score = overall_label_health_score(labels=labels, pred_probs=pred_probs) assert 1 - num_issues / labels.shape[0] == score def test_overall_label_health_score_function_calls(): dataset_name = "caltech256" pred_probs, labels = _get_pred_probs_labels_from_labelerrors_datasets(dataset_name) score = overall_label_health_score(labels=labels, pred_probs=pred_probs) confident_joint = compute_confident_joint(labels=labels, pred_probs=pred_probs) num_examples = len(labels) score_cj = overall_label_health_score( labels=None, pred_probs=pred_probs, confident_joint=confident_joint ) joint = estimate_joint(labels=labels, pred_probs=pred_probs) score_joint = overall_label_health_score( labels=None, pred_probs=pred_probs, joint=joint, num_examples=num_examples ) joint_cj = estimate_joint(labels=labels, pred_probs=pred_probs, confident_joint=confident_joint) score_joint_cj = overall_label_health_score( labels=None, pred_probs=pred_probs, joint=joint_cj, num_examples=num_examples ) assert score_cj != score assert score_cj == score_joint assert score_joint_cj == score_joint confident_joint_strategy = npst.arrays( np.int32, shape=npst.array_shapes(min_dims=2, max_dims=2, min_side=2, max_side=10), elements=st.integers(min_value=0, max_value=int(1e6)), ).filter(lambda arr: arr.shape[0] == arr.shape[1]) @pytest.mark.issue_651 @given(confident_joint=confident_joint_strategy) @settings(deadline=500) def test_find_overlapping_classes_with_confident_joint(confident_joint): # Setup K = confident_joint.shape[0] overlapping_classes = find_overlapping_classes(confident_joint=confident_joint) # Test that the output dataframe has the expected columns expected_columns = [ "Class Index A", "Class Index B", "Num Overlapping Examples", "Joint Probability", ] assert set(overlapping_classes.columns) == set(expected_columns) # Class indices must be valid assert overlapping_classes["Class Index A"].between(0, K - 1).all() assert overlapping_classes["Class Index B"].between(0, K - 1).all() # Overlapping example count should be non-negative integers assert (overlapping_classes["Num Overlapping Examples"] >= 0).all() assert overlapping_classes["Num Overlapping Examples"].dtype == int # Joint probabilities should be between 0 and 1 assert (overlapping_classes["Joint Probability"] >= 0).all() assert (overlapping_classes["Joint Probability"] <= 1).all() # Joint probabilities sorted in descending order if K > 2: assert (overlapping_classes["Joint Probability"].diff()[1:] <= 0).all()