chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,167 @@
|
||||
# ------------------------------------------
|
||||
# TextDiffuser: Diffusion Models as Text Painters
|
||||
# Paper Link: https://arxiv.org/abs/2305.10855
|
||||
# Code Link: https://github.com/microsoft/unilm/tree/master/textdiffuser
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# This file provides the inference script.
|
||||
# ------------------------------------------
|
||||
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import argparse
|
||||
from clipscore import cal_clipscore
|
||||
from fid_score import calculate_fid_given_paths
|
||||
|
||||
|
||||
def eval_clipscore(root_eval, root_res, dataset, device="cuda:0", num_images_per_prompt=4):
|
||||
with open(os.path.join(root_eval, dataset, dataset + '.txt'), 'r') as fr:
|
||||
text_list = fr.readlines()
|
||||
text_list = [_.strip() for _ in text_list]
|
||||
clip_scores = []
|
||||
scores = []
|
||||
for seed in range(num_images_per_prompt):
|
||||
if 'stablediffusion' in root_res:
|
||||
format = '.png'
|
||||
else:
|
||||
format = '.jpg'
|
||||
image_list = [os.path.join(root_res, dataset, 'images_' + str(seed),
|
||||
str(idx) + '_' + str(seed) + format) for idx in range(len(text_list))]
|
||||
image_ids = [str(idx) + '_' + str(seed) + format for idx in range(len(text_list))]
|
||||
score = cal_clipscore(image_ids=image_ids, image_paths=image_list, text_list=text_list, device=device)
|
||||
clip_score = np.mean([s['CLIPScore'] for s in score.values()])
|
||||
clip_scores.append(clip_score)
|
||||
scores.append(score)
|
||||
print("clip_score:", np.mean(clip_scores), clip_scores)
|
||||
return np.mean(clip_scores), scores
|
||||
|
||||
|
||||
def MARIOEval_evaluate_results(root, datasets_with_images, datasets, methods, gpu,
|
||||
eval_clipscore_flag=True, eval_fid_flag=True, num_images_per_prompt=4):
|
||||
root_eval = os.path.join(root, "MARIOEval")
|
||||
method_res = {}
|
||||
device = "cuda:" + str(gpu)
|
||||
for method_idx, method in enumerate(methods):
|
||||
if method_idx != gpu: # running in different gpus simultaneously to save time
|
||||
continue
|
||||
print("\nmethod:", method)
|
||||
dataset_res = {}
|
||||
root_res = os.path.join(root, 'generation', method)
|
||||
for dataset in datasets:
|
||||
print("dataset:", dataset)
|
||||
dataset_res[dataset] = {}
|
||||
if eval_clipscore_flag:
|
||||
dataset_res[dataset]['clipscore'], dataset_res[dataset]['scores'] =\
|
||||
eval_clipscore(root_eval, root_res, dataset, device, num_images_per_prompt)
|
||||
if eval_fid_flag and dataset in datasets_with_images:
|
||||
gt_path = os.path.join(root_eval, dataset, 'images')
|
||||
fids = []
|
||||
for idx in range(num_images_per_prompt):
|
||||
gen_path = os.path.join(root_res, dataset, 'images_' + str(idx))
|
||||
fids.append(calculate_fid_given_paths(paths=[gt_path, gen_path]))
|
||||
print("fid:", np.mean(fids), fids)
|
||||
dataset_res[dataset]['fid'] = np.mean(fids)
|
||||
|
||||
if eval_clipscore_flag:
|
||||
method_clipscores = []
|
||||
for seed in range(num_images_per_prompt):
|
||||
clipscore_list = []
|
||||
for dataset in dataset_res.keys():
|
||||
clipscore_list += [_['CLIPScore'] for _ in dataset_res[dataset]['scores'][seed].values()]
|
||||
method_clipscores.append(np.mean(clipscore_list))
|
||||
method_clipscore = np.mean(method_clipscores)
|
||||
dataset_res['clipscore'] = method_clipscore
|
||||
if eval_fid_flag:
|
||||
method_fids = []
|
||||
for idx in range(num_images_per_prompt):
|
||||
gt_paths = []
|
||||
gen_paths = []
|
||||
for dataset in dataset_res.keys():
|
||||
if dataset in datasets_with_images:
|
||||
gt_paths.append(os.path.join(root_eval, dataset, 'images'))
|
||||
gen_paths.append(os.path.join(root_res, dataset, 'images_' + str(idx)))
|
||||
if len(gt_paths):
|
||||
method_fids.append(calculate_fid_given_paths(paths=[gt_paths, gen_paths]))
|
||||
print("fid:", np.mean(method_fids), method_fids)
|
||||
method_fid = np.mean(method_fids)
|
||||
dataset_res['fid'] = method_fid
|
||||
|
||||
method_res[method] = dataset_res
|
||||
with open(os.path.join(root_res, 'eval.json'), 'w') as fw:
|
||||
json.dump(dataset_res, fw)
|
||||
|
||||
print(method_res)
|
||||
with open(os.path.join(root, 'generation', 'eval.json'), 'w') as fw:
|
||||
json.dump(method_res, fw)
|
||||
|
||||
|
||||
def merge_eval_results(root, methods):
|
||||
method_res = {}
|
||||
for method_idx, method in enumerate(methods):
|
||||
root_res = os.path.join(root, 'generation', method)
|
||||
with open(os.path.join(root_res, 'eval.json'), 'r') as fr:
|
||||
dataset_res = json.load(fr)
|
||||
for k, v in dataset_res.items():
|
||||
if type(v) is dict:
|
||||
del v['scores'] # too long
|
||||
method_res[method] = dataset_res
|
||||
|
||||
with open(os.path.join(root, 'generation', 'eval.json'), 'w') as fw:
|
||||
json.dump(method_res, fw)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Simple example of a training script.")
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
type=str,
|
||||
default='TMDBEval500',
|
||||
required=False,
|
||||
choices=['TMDBEval500', 'OpenLibraryEval500', 'LAIONEval4000',
|
||||
'ChineseDrawText', 'DrawBenchText', 'DrawTextCreative']
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
type=str,
|
||||
default="/path/to/data/TextDiffuser/evaluation/",
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--method",
|
||||
type=str,
|
||||
default='controlnet',
|
||||
required=False,
|
||||
choices=['controlnet', 'deepfloyd', 'stablediffusion', 'textdiffuser']
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gpu",
|
||||
type=int,
|
||||
default=0,
|
||||
required=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--split",
|
||||
type=int,
|
||||
default=0,
|
||||
required=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--total_split",
|
||||
type=int,
|
||||
default=1,
|
||||
required=False,
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
|
||||
datasets_with_images = ['TMDBEval500', 'OpenLibraryEval500', 'LAIONEval4000']
|
||||
datasets = datasets_with_images + ['ChineseDrawText', 'DrawBenchText', 'DrawTextCreative']
|
||||
methods = ['textdiffuser', 'controlnet', 'deepfloyd', 'stablediffusion']
|
||||
|
||||
MARIOEval_evaluate_results(args.root, datasets_with_images, datasets, methods, args.gpu,
|
||||
eval_clipscore_flag=True, eval_fid_flag=True, num_images_per_prompt=4)
|
||||
merge_eval_results(args.root, methods)
|
||||
@@ -0,0 +1,192 @@
|
||||
# ------------------------------------------
|
||||
# TextDiffuser: Diffusion Models as Text Painters
|
||||
# Paper Link: https://arxiv.org/abs/2305.10855
|
||||
# Code Link: https://github.com/microsoft/unilm/tree/master/textdiffuser
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# This file provides the inference script.
|
||||
# ------------------------------------------
|
||||
|
||||
import os
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
import argparse
|
||||
import cv2
|
||||
import torchvision.transforms as transforms
|
||||
|
||||
to_pil_image = transforms.ToPILImage()
|
||||
|
||||
def load_stablediffusion():
|
||||
from diffusers import StableDiffusionPipeline
|
||||
pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
|
||||
pipe.enable_xformers_memory_efficient_attention()
|
||||
pipe.enable_model_cpu_offload()
|
||||
return pipe
|
||||
|
||||
def test_stablediffusion(prompt, save_path, num_images_per_prompt=4,
|
||||
pipe=None, generator=None):
|
||||
images = pipe(prompt, num_inference_steps=50, generator=generator, num_images_per_prompt=num_images_per_prompt).images
|
||||
for idx, image in enumerate(images):
|
||||
image.save(save_path.replace('.jpg', '_' + str(idx) + '.jpg').replace('/images/', '/images_'+ str(idx) +'/'))
|
||||
|
||||
def load_deepfloyd_if():
|
||||
from diffusers import DiffusionPipeline
|
||||
stage_1 = DiffusionPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16)
|
||||
# stage_1.enable_xformers_memory_efficient_attention() # remove line if torch.__version__ >= 2.0.0
|
||||
stage_1.enable_model_cpu_offload()
|
||||
stage_2 = DiffusionPipeline.from_pretrained("DeepFloyd/IF-II-L-v1.0", text_encoder=None, variant="fp16",
|
||||
torch_dtype=torch.float16)
|
||||
# stage_2.enable_xformers_memory_efficient_attention() # remove line if torch.__version__ >= 2.0.0
|
||||
stage_2.enable_model_cpu_offload()
|
||||
safety_modules = {"feature_extractor": stage_1.feature_extractor, "safety_checker": stage_1.safety_checker,
|
||||
"watermarker": stage_1.watermarker}
|
||||
stage_3 = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-x4-upscaler", **safety_modules,
|
||||
torch_dtype=torch.float16)
|
||||
# stage_3.enable_xformers_memory_efficient_attention() # remove line if torch.__version__ >= 2.0.0
|
||||
stage_3.enable_model_cpu_offload()
|
||||
return stage_1, stage_2, stage_3
|
||||
|
||||
|
||||
def test_deepfloyd_if(stage_1, stage_2, stage_3, prompt, save_path, num_images_per_prompt=4, generator=None):
|
||||
idx = num_images_per_prompt - 1 # if the last image of a case exists, then return
|
||||
new_save_path = save_path.replace('.jpg', '_' + str(idx) + '.jpg').replace('/images/', '/images_' + str(idx) + '/')
|
||||
if os.path.exists(new_save_path):
|
||||
return
|
||||
if not stage_1 or not stage_2 or not stage_3:
|
||||
stage_1, stage_2, stage_3 = load_deepfloyd_if()
|
||||
if generator is None:
|
||||
generator = torch.manual_seed(0)
|
||||
prompt_embeds, negative_embeds = stage_1.encode_prompt(prompt)
|
||||
stage_1.set_progress_bar_config(disable=True)
|
||||
stage_2.set_progress_bar_config(disable=True)
|
||||
stage_3.set_progress_bar_config(disable=True)
|
||||
images = stage_1(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator,
|
||||
output_type="pt", num_images_per_prompt=num_images_per_prompt).images
|
||||
for idx, image in enumerate(images):
|
||||
image = stage_2(image=image.unsqueeze(0), prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds,
|
||||
generator=generator, output_type="pt").images
|
||||
image = stage_3(prompt=prompt, image=image, generator=generator, noise_level=100).images
|
||||
# image = to_pil_image(image[0].cpu())
|
||||
new_save_path = save_path.replace('.jpg', '_' + str(idx) + '.jpg').replace('/images/', '/images_'+ str(idx) +'/')
|
||||
image[0].save(new_save_path)
|
||||
|
||||
|
||||
def load_controlnet_cannyedge():
|
||||
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
|
||||
|
||||
controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
|
||||
pipe = StableDiffusionControlNetPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", controlnet=controlnet,
|
||||
safety_checker=None, torch_dtype=torch.float16)
|
||||
pipe.set_progress_bar_config(disable=True)
|
||||
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
|
||||
pipe.enable_xformers_memory_efficient_attention()
|
||||
pipe.enable_model_cpu_offload()
|
||||
return pipe
|
||||
|
||||
|
||||
def test_controlnet_cannyedge(prompt, save_path, canny_path, num_images_per_prompt=4,
|
||||
pipe=None, generator=None, low_threshold=100, high_threshold=200):
|
||||
'''ref: https://github.com/huggingface/diffusers/blob/131312caba0af97da98fc498dfdca335c9692f8c/docs/source/en/api/pipelines/stable_diffusion/controlnet.mdx'''
|
||||
from diffusers.utils import load_image
|
||||
if pipe is None:
|
||||
pipe = load_controlnet_cannyedge()
|
||||
|
||||
if os.path.exists(canny_path):
|
||||
canny_path = Image.open(canny_path)
|
||||
image = load_image(canny_path)
|
||||
image = np.array(image)
|
||||
image = cv2.Canny(image, low_threshold, high_threshold)
|
||||
image = image[:, :, None]
|
||||
image = np.concatenate([image, image, image], axis=2)
|
||||
image = Image.fromarray(image)
|
||||
images = pipe(prompt, image, num_inference_steps=20, generator=generator, num_images_per_prompt=num_images_per_prompt).images
|
||||
for idx, image in enumerate(images):
|
||||
image.save(save_path.replace('.jpg', '_' + str(idx) + '.jpg').replace('/images/', '/images_'+ str(idx) +'/'))
|
||||
|
||||
|
||||
def MARIOEval_generate_results(root, dataset, method='controlnet', num_images_per_prompt=4, split=0, total_split=1):
|
||||
root_eval = os.path.join(root, "MARIOEval")
|
||||
render_path = os.path.join(root_eval, dataset, 'render')
|
||||
root_res = os.path.join(root, "generation", method)
|
||||
for idx in range(num_images_per_prompt):
|
||||
os.makedirs(os.path.join(root_res, dataset, 'images_' + str(idx)), exist_ok=True)
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
if method == 'controlnet':
|
||||
pipe = load_controlnet_cannyedge()
|
||||
elif method == 'stablediffusion':
|
||||
pipe = load_stablediffusion()
|
||||
elif method == 'deepfloyd':
|
||||
stage_1, stage_2, stage_3 = load_deepfloyd_if()
|
||||
|
||||
with open(os.path.join(root_eval, dataset, dataset + '.txt'), 'r') as fr:
|
||||
prompts = fr.readlines()
|
||||
prompts = [_.strip() for _ in prompts]
|
||||
for idx, prompt in tqdm(enumerate(prompts)):
|
||||
if idx < split * len(prompts) / total_split or idx > (split + 1) * len(prompts) / total_split:
|
||||
continue
|
||||
if method == 'controlnet':
|
||||
test_controlnet_cannyedge(prompt=prompt, num_images_per_prompt=num_images_per_prompt,
|
||||
save_path=os.path.join(root_res, dataset, 'images', str(idx) + '.jpg'),
|
||||
canny_path=os.path.join(render_path, str(idx) + '.png'),
|
||||
pipe=pipe, generator=generator)
|
||||
elif method == 'stablediffusion':
|
||||
test_stablediffusion(prompt=prompt, num_images_per_prompt=num_images_per_prompt,
|
||||
save_path=os.path.join(root_res, dataset, 'images', str(idx) + '.jpg'),
|
||||
pipe=pipe, generator=generator)
|
||||
elif method == 'deepfloyd':
|
||||
test_deepfloyd_if(stage_1, stage_2, stage_3, num_images_per_prompt=num_images_per_prompt,
|
||||
save_path=os.path.join(root_res, dataset, 'images', str(idx) + '.jpg'),
|
||||
prompt=prompt, generator=generator)
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Simple example of a training script.")
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
type=str,
|
||||
default='TMDBEval500',
|
||||
required=False,
|
||||
choices=['TMDBEval500', 'OpenLibraryEval500', 'LAIONEval4000',
|
||||
'ChineseDrawText', 'DrawBenchText', 'DrawTextCreative']
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root",
|
||||
type=str,
|
||||
default="/path/to/eval",
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--method",
|
||||
type=str,
|
||||
default='controlnet',
|
||||
required=False,
|
||||
choices=['controlnet', 'deepfloyd', 'stablediffusion', 'textdiffuser']
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gpu",
|
||||
type=int,
|
||||
default=0,
|
||||
required=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--split",
|
||||
type=int,
|
||||
default=0,
|
||||
required=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--total_split",
|
||||
type=int,
|
||||
default=1,
|
||||
required=False,
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
|
||||
|
||||
MARIOEval_generate_results(root=args.root, dataset=args.dataset, method=args.method,
|
||||
split=args.split, total_split=args.total_split)
|
||||
@@ -0,0 +1,10 @@
|
||||
# Evaluation
|
||||
|
||||
We provide the code for sampling from Stable Diffusion, ControlNet, DeepFloyd at ```MARIOEval_generate.py```. Since these methods rely on *diffusers* of the original version, it is recommended to create a **NEW** environment and install packages with command ```pip install requirements.txt```. It is recommended to install pytorch with version >= 2.0 to avoid the OOM error.
|
||||
|
||||
Once the generation is complete, evaluation of FID and CLIPScore can be performed using the ```MARIOEval_evaluate.py``` file. For OCR metrics, please install [MaskTextSpotterV3](https://github.com/MhLiao/MaskTextSpotterV3) to obtain the OCR result of each image and refer to ```ocr_eval.py``` for evaluation. It should be noted that the output image of DeepFloyd contains a watermark "IF" at the right-bottom corner, which needs to be masked before performing OCR.
|
||||
|
||||
```python
|
||||
if method is 'deepfloyd':
|
||||
image[-64:, -64:] = 0 # remove watermark, the input image is resized to 512x512
|
||||
```
|
||||
@@ -0,0 +1,366 @@
|
||||
# Adapted from https://github.com/jmhessel/clipscore/blob/1036465276513621f77f1c2208d742e4a430781f/clipscore.py
|
||||
'''
|
||||
Code for CLIPScore (https://arxiv.org/abs/2104.08718)
|
||||
@inproceedings{hessel2021clipscore,
|
||||
title={{CLIPScore:} A Reference-free Evaluation Metric for Image Captioning},
|
||||
author={Hessel, Jack and Holtzman, Ari and Forbes, Maxwell and Bras, Ronan Le and Choi, Yejin},
|
||||
booktitle={EMNLP},
|
||||
year={2021}
|
||||
}
|
||||
'''
|
||||
import argparse
|
||||
import clip
|
||||
from PIL import Image
|
||||
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
|
||||
import torch
|
||||
import tqdm
|
||||
import numpy as np
|
||||
import sklearn.preprocessing
|
||||
import collections
|
||||
import os
|
||||
import pathlib
|
||||
import json
|
||||
import warnings
|
||||
from packaging import version
|
||||
from pycocoevalcap.tokenizer.ptbtokenizer import PTBTokenizer
|
||||
from pycocoevalcap.meteor.meteor import Meteor
|
||||
from pycocoevalcap.bleu.bleu import Bleu
|
||||
from pycocoevalcap.cider.cider import Cider
|
||||
from pycocoevalcap.rouge.rouge import Rouge
|
||||
from pycocoevalcap.spice.spice import Spice
|
||||
|
||||
|
||||
def get_all_metrics(refs, cands, return_per_cap=False):
|
||||
metrics = []
|
||||
names = []
|
||||
|
||||
pycoco_eval_cap_scorers = [(Bleu(4), 'bleu'),
|
||||
(Meteor(), 'meteor'),
|
||||
(Rouge(), 'rouge'),
|
||||
(Cider(), 'cider'),
|
||||
(Spice(), 'spice')]
|
||||
|
||||
for scorer, name in pycoco_eval_cap_scorers:
|
||||
overall, per_cap = pycoco_eval(scorer, refs, cands)
|
||||
if return_per_cap:
|
||||
metrics.append(per_cap)
|
||||
else:
|
||||
metrics.append(overall)
|
||||
names.append(name)
|
||||
|
||||
metrics = dict(zip(names, metrics))
|
||||
return metrics
|
||||
|
||||
|
||||
def tokenize(refs, cands, no_op=False):
|
||||
# no_op is a debug option to see how significantly not using the PTB tokenizer
|
||||
# affects things
|
||||
tokenizer = PTBTokenizer()
|
||||
|
||||
if no_op:
|
||||
refs = {idx: [r for r in c_refs] for idx, c_refs in enumerate(refs)}
|
||||
cands = {idx: [c] for idx, c in enumerate(cands)}
|
||||
|
||||
else:
|
||||
refs = {idx: [{'caption':r} for r in c_refs] for idx, c_refs in enumerate(refs)}
|
||||
cands = {idx: [{'caption':c}] for idx, c in enumerate(cands)}
|
||||
|
||||
refs = tokenizer.tokenize(refs)
|
||||
cands = tokenizer.tokenize(cands)
|
||||
|
||||
return refs, cands
|
||||
|
||||
|
||||
def pycoco_eval(scorer, refs, cands):
|
||||
'''
|
||||
scorer is assumed to have a compute_score function.
|
||||
refs is a list of lists of strings
|
||||
cands is a list of predictions
|
||||
'''
|
||||
refs, cands = tokenize(refs, cands)
|
||||
average_score, scores = scorer.compute_score(refs, cands)
|
||||
return average_score, scores
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'candidates_json',
|
||||
type=str,
|
||||
help='Candidates json mapping from image_id --> candidate.')
|
||||
|
||||
parser.add_argument(
|
||||
'image_dir',
|
||||
type=str,
|
||||
help='Directory of images, with the filenames as image ids.')
|
||||
|
||||
parser.add_argument(
|
||||
'--references_json',
|
||||
default=None,
|
||||
help='Optional references json mapping from image_id --> [list of references]')
|
||||
|
||||
parser.add_argument(
|
||||
'--compute_other_ref_metrics',
|
||||
default=1,
|
||||
type=int,
|
||||
help='If references is specified, should we compute standard reference-based metrics?')
|
||||
|
||||
parser.add_argument(
|
||||
'--save_per_instance',
|
||||
default=None,
|
||||
help='if set, we will save per instance clipscores to this file')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if isinstance(args.save_per_instance, str) and not args.save_per_instance.endswith('.json'):
|
||||
print('if you\'re saving per-instance, please make sure the filepath ends in json.')
|
||||
quit()
|
||||
return args
|
||||
|
||||
|
||||
class CLIPCapDataset(torch.utils.data.Dataset):
|
||||
def __init__(self, data, prefix='A photo depicts'):
|
||||
self.data = data
|
||||
self.prefix = prefix
|
||||
if self.prefix[-1] != ' ':
|
||||
self.prefix += ' '
|
||||
|
||||
def __getitem__(self, idx):
|
||||
c_data = self.data[idx]
|
||||
c_data = clip.tokenize(self.prefix + c_data, truncate=True).squeeze()
|
||||
return {'caption': c_data}
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
|
||||
class CLIPImageDataset(torch.utils.data.Dataset):
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
# only 224x224 ViT-B/32 supported for now
|
||||
self.preprocess = self._transform_test(224)
|
||||
|
||||
def _transform_test(self, n_px):
|
||||
return Compose([
|
||||
Resize(n_px, interpolation=Image.BICUBIC),
|
||||
CenterCrop(n_px),
|
||||
lambda image: image.convert("RGB"),
|
||||
ToTensor(),
|
||||
Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
|
||||
])
|
||||
|
||||
def __getitem__(self, idx):
|
||||
c_data = self.data[idx]
|
||||
image = Image.open(c_data)
|
||||
image = self.preprocess(image)
|
||||
return {'image':image}
|
||||
|
||||
def __len__(self):
|
||||
return len(self.data)
|
||||
|
||||
|
||||
def extract_all_captions(captions, model, device, batch_size=256, num_workers=8):
|
||||
data = torch.utils.data.DataLoader(
|
||||
CLIPCapDataset(captions),
|
||||
batch_size=batch_size, num_workers=num_workers, shuffle=False)
|
||||
all_text_features = []
|
||||
with torch.no_grad():
|
||||
for b in tqdm.tqdm(data):
|
||||
b = b['caption'].to(device)
|
||||
all_text_features.append(model.encode_text(b).cpu().numpy())
|
||||
all_text_features = np.vstack(all_text_features)
|
||||
return all_text_features
|
||||
|
||||
|
||||
def extract_all_images(images, model, device, batch_size=64, num_workers=8):
|
||||
data = torch.utils.data.DataLoader(
|
||||
CLIPImageDataset(images),
|
||||
batch_size=batch_size, num_workers=num_workers, shuffle=False)
|
||||
all_image_features = []
|
||||
with torch.no_grad():
|
||||
for b in tqdm.tqdm(data):
|
||||
b = b['image'].to(device)
|
||||
if device == 'cuda':
|
||||
b = b.to(torch.float16)
|
||||
all_image_features.append(model.encode_image(b).cpu().numpy())
|
||||
all_image_features = np.vstack(all_image_features)
|
||||
return all_image_features
|
||||
|
||||
|
||||
def get_clip_score(model, images, candidates, device, w=2.5):
|
||||
'''
|
||||
get standard image-text clipscore.
|
||||
images can either be:
|
||||
- a list of strings specifying filepaths for images
|
||||
- a precomputed, ordered matrix of image features
|
||||
'''
|
||||
if isinstance(images, list):
|
||||
# need to extract image features
|
||||
images = extract_all_images(images, model, device)
|
||||
|
||||
candidates = extract_all_captions(candidates, model, device)
|
||||
|
||||
#as of numpy 1.21, normalize doesn't work properly for float16
|
||||
if version.parse(np.__version__) < version.parse('1.21'):
|
||||
images = sklearn.preprocessing.normalize(images, axis=1)
|
||||
candidates = sklearn.preprocessing.normalize(candidates, axis=1)
|
||||
else:
|
||||
warnings.warn(
|
||||
'due to a numerical instability, new numpy normalization is slightly different than paper results. '
|
||||
'to exactly replicate paper results, please use numpy version less than 1.21, e.g., 1.20.3.')
|
||||
images = images / np.sqrt(np.sum(images**2, axis=1, keepdims=True))
|
||||
candidates = candidates / np.sqrt(np.sum(candidates**2, axis=1, keepdims=True))
|
||||
|
||||
per = w*np.clip(np.sum(images * candidates, axis=1), 0, None)
|
||||
return np.mean(per), per, candidates
|
||||
|
||||
|
||||
def get_refonlyclipscore(model, references, candidates, device):
|
||||
'''
|
||||
The text only side for refclipscore
|
||||
'''
|
||||
if isinstance(candidates, list):
|
||||
candidates = extract_all_captions(candidates, model, device)
|
||||
|
||||
flattened_refs = []
|
||||
flattened_refs_idxs = []
|
||||
for idx, refs in enumerate(references):
|
||||
flattened_refs.extend(refs)
|
||||
flattened_refs_idxs.extend([idx for _ in refs])
|
||||
|
||||
flattened_refs = extract_all_captions(flattened_refs, model, device)
|
||||
|
||||
if version.parse(np.__version__) < version.parse('1.21'):
|
||||
candidates = sklearn.preprocessing.normalize(candidates, axis=1)
|
||||
flattened_refs = sklearn.preprocessing.normalize(flattened_refs, axis=1)
|
||||
else:
|
||||
warnings.warn(
|
||||
'due to a numerical instability, new numpy normalization is slightly different than paper results. '
|
||||
'to exactly replicate paper results, please use numpy version less than 1.21, e.g., 1.20.3.')
|
||||
|
||||
candidates = candidates / np.sqrt(np.sum(candidates**2, axis=1, keepdims=True))
|
||||
flattened_refs = flattened_refs / np.sqrt(np.sum(flattened_refs**2, axis=1, keepdims=True))
|
||||
|
||||
cand_idx2refs = collections.defaultdict(list)
|
||||
for ref_feats, cand_idx in zip(flattened_refs, flattened_refs_idxs):
|
||||
cand_idx2refs[cand_idx].append(ref_feats)
|
||||
|
||||
assert len(cand_idx2refs) == len(candidates)
|
||||
|
||||
cand_idx2refs = {k: np.vstack(v) for k, v in cand_idx2refs.items()}
|
||||
|
||||
per = []
|
||||
for c_idx, cand in tqdm.tqdm(enumerate(candidates)):
|
||||
cur_refs = cand_idx2refs[c_idx]
|
||||
all_sims = cand.dot(cur_refs.transpose())
|
||||
per.append(np.max(all_sims))
|
||||
|
||||
return np.mean(per), per
|
||||
|
||||
|
||||
def cal_clipscore(image_ids, image_paths, text_list, device=None, references=None, scale_weight=1):
|
||||
if device is None:
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model, transform = clip.load("ViT-B/32", device=device, jit=False)
|
||||
model.eval()
|
||||
|
||||
image_feats = extract_all_images(image_paths, model, device, batch_size=64, num_workers=8)
|
||||
|
||||
# get image-text clipscore
|
||||
_, per_instance_image_text, candidate_feats = get_clip_score(model, image_feats, text_list, device, w=scale_weight)
|
||||
|
||||
if references:
|
||||
# get text-text clipscore
|
||||
_, per_instance_text_text = get_refonlyclipscore(model, references, candidate_feats, device)
|
||||
# F-score
|
||||
refclipscores = 2 * per_instance_image_text * per_instance_text_text / (per_instance_image_text + per_instance_text_text)
|
||||
scores = {image_id: {'CLIPScore': float(clipscore), 'RefCLIPScore': float(refclipscore)}
|
||||
for image_id, clipscore, refclipscore in
|
||||
zip(image_ids, per_instance_image_text, refclipscores)}
|
||||
|
||||
other_metrics = get_all_metrics(references, text_list)
|
||||
for k, v in other_metrics.items():
|
||||
if k == 'bleu':
|
||||
for bidx, sc in enumerate(v):
|
||||
print('BLEU-{}: {:.4f}'.format(bidx+1, sc))
|
||||
else:
|
||||
print('{}: {:.4f}'.format(k.upper(), v))
|
||||
print('CLIPScore: {:.4f}'.format(np.mean([s['CLIPScore'] for s in scores.values()])))
|
||||
print('RefCLIPScore: {:.4f}'.format(np.mean([s['RefCLIPScore'] for s in scores.values()])))
|
||||
|
||||
else:
|
||||
scores = {image_id: {'CLIPScore': float(clipscore)}
|
||||
for image_id, clipscore in
|
||||
zip(image_ids, per_instance_image_text)}
|
||||
print('CLIPScore: {:.4f}'.format(np.mean([s['CLIPScore'] for s in scores.values()])))
|
||||
|
||||
return scores
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
image_paths = [os.path.join(args.image_dir, path) for path in os.listdir(args.image_dir)
|
||||
if path.endswith(('.png', '.jpg', '.jpeg', '.tiff'))]
|
||||
image_ids = [pathlib.Path(path).stem for path in image_paths]
|
||||
|
||||
with open(args.candidates_json) as f:
|
||||
candidates = json.load(f)
|
||||
candidates = [candidates[cid] for cid in image_ids]
|
||||
|
||||
if args.references_json:
|
||||
with open(args.references_json) as f:
|
||||
references = json.load(f)
|
||||
references = [references[cid] for cid in image_ids]
|
||||
if isinstance(references[0], str):
|
||||
references = [[r] for r in references]
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
if device == 'cpu':
|
||||
warnings.warn(
|
||||
'CLIP runs in full float32 on CPU. Results in paper were computed on GPU, which uses float16. '
|
||||
'If you\'re reporting results on CPU, please note this when you report.')
|
||||
model, transform = clip.load("ViT-B/32", device=device, jit=False)
|
||||
model.eval()
|
||||
|
||||
image_feats = extract_all_images(
|
||||
image_paths, model, device, batch_size=64, num_workers=8)
|
||||
|
||||
# get image-text clipscore
|
||||
_, per_instance_image_text, candidate_feats = get_clip_score(
|
||||
model, image_feats, candidates, device)
|
||||
|
||||
if args.references_json:
|
||||
# get text-text clipscore
|
||||
_, per_instance_text_text = get_refonlyclipscore(
|
||||
model, references, candidate_feats, device)
|
||||
# F-score
|
||||
refclipscores = 2 * per_instance_image_text * per_instance_text_text / (per_instance_image_text + per_instance_text_text)
|
||||
scores = {image_id: {'CLIPScore': float(clipscore), 'RefCLIPScore': float(refclipscore)}
|
||||
for image_id, clipscore, refclipscore in
|
||||
zip(image_ids, per_instance_image_text, refclipscores)}
|
||||
|
||||
else:
|
||||
scores = {image_id: {'CLIPScore': float(clipscore)}
|
||||
for image_id, clipscore in
|
||||
zip(image_ids, per_instance_image_text)}
|
||||
print('CLIPScore: {:.4f}'.format(np.mean([s['CLIPScore'] for s in scores.values()])))
|
||||
|
||||
if args.references_json:
|
||||
if args.compute_other_ref_metrics:
|
||||
other_metrics = generation_eval_utils.get_all_metrics(references, candidates)
|
||||
for k, v in other_metrics.items():
|
||||
if k == 'bleu':
|
||||
for bidx, sc in enumerate(v):
|
||||
print('BLEU-{}: {:.4f}'.format(bidx+1, sc))
|
||||
else:
|
||||
print('{}: {:.4f}'.format(k.upper(), v))
|
||||
print('CLIPScore: {:.4f}'.format(np.mean([s['CLIPScore'] for s in scores.values()])))
|
||||
print('RefCLIPScore: {:.4f}'.format(np.mean([s['RefCLIPScore'] for s in scores.values()])))
|
||||
|
||||
if args.save_per_instance:
|
||||
with open(args.save_per_instance, 'w') as f:
|
||||
f.write(json.dumps(scores))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
python MARIOEval_evaluate.py \
|
||||
--gpu 0 \
|
||||
--dataset TMDBEval500 \
|
||||
--root /path/to/eval \
|
||||
--method textdiffuser
|
||||
@@ -0,0 +1,333 @@
|
||||
# Adapted from https://github.com/mseitzer/pytorch-fid/blob/0a754fb8e66021700478fd365b79c2eaa316e31b/src/pytorch_fid/fid_score.py
|
||||
"""Calculates the Frechet Inception Distance (FID) to evalulate GANs
|
||||
|
||||
The FID metric calculates the distance between two distributions of images.
|
||||
Typically, we have summary statistics (mean & covariance matrix) of one
|
||||
of these distributions, while the 2nd distribution is given by a GAN.
|
||||
|
||||
When run as a stand-alone program, it compares the distribution of
|
||||
images that are stored as PNG/JPEG at a specified location with a
|
||||
distribution given by summary statistics (in pickle format).
|
||||
|
||||
The FID is calculated by assuming that X_1 and X_2 are the activations of
|
||||
the pool_3 layer of the inception net for generated samples and real world
|
||||
samples respectively.
|
||||
|
||||
See --help to see further details.
|
||||
|
||||
Code apapted from https://github.com/bioinf-jku/TTUR to use PyTorch instead
|
||||
of Tensorflow
|
||||
|
||||
Copyright 2018 Institute of Bioinformatics, JKU Linz
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
import os
|
||||
import pathlib
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchvision.transforms as TF
|
||||
from PIL import Image
|
||||
from scipy import linalg
|
||||
from torch.nn.functional import adaptive_avg_pool2d
|
||||
|
||||
try:
|
||||
from tqdm import tqdm
|
||||
except ImportError:
|
||||
# If tqdm is not available, provide a mock version of it
|
||||
def tqdm(x):
|
||||
return x
|
||||
|
||||
from pytorch_fid.inception import InceptionV3
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--batch-size', type=int, default=50,
|
||||
help='Batch size to use')
|
||||
parser.add_argument('--num-workers', type=int,
|
||||
help=('Number of processes to use for data loading. '
|
||||
'Defaults to `min(8, num_cpus)`'))
|
||||
parser.add_argument('--device', type=str, default=None,
|
||||
help='Device to use. Like cuda, cuda:0 or cpu')
|
||||
parser.add_argument('--dims', type=int, default=2048,
|
||||
choices=list(InceptionV3.BLOCK_INDEX_BY_DIM),
|
||||
help=('Dimensionality of Inception features to use. '
|
||||
'By default, uses pool3 features'))
|
||||
parser.add_argument('--save-stats', action='store_true',
|
||||
help=('Generate an npz archive from a directory of samples. '
|
||||
'The first path is used as input and the second as output.'))
|
||||
parser.add_argument('path', type=str, nargs=2,
|
||||
help=('Paths to the generated images or '
|
||||
'to .npz statistic files'))
|
||||
|
||||
IMAGE_EXTENSIONS = {'bmp', 'jpg', 'jpeg', 'pgm', 'png', 'ppm',
|
||||
'tif', 'tiff', 'webp'}
|
||||
|
||||
|
||||
class ImagePathDataset(torch.utils.data.Dataset):
|
||||
def __init__(self, files, transforms=None):
|
||||
self.files = files
|
||||
self.transforms = transforms
|
||||
|
||||
def __len__(self):
|
||||
return len(self.files)
|
||||
|
||||
def __getitem__(self, i):
|
||||
path = self.files[i]
|
||||
img = Image.open(path).convert('RGB')
|
||||
if self.transforms is not None:
|
||||
img = self.transforms(img)
|
||||
return img
|
||||
|
||||
|
||||
def get_activations(files, model, batch_size=50, dims=2048, device='cpu',
|
||||
num_workers=1):
|
||||
"""Calculates the activations of the pool_3 layer for all images.
|
||||
|
||||
Params:
|
||||
-- files : List of image files paths
|
||||
-- model : Instance of inception model
|
||||
-- batch_size : Batch size of images for the model to process at once.
|
||||
Make sure that the number of samples is a multiple of
|
||||
the batch size, otherwise some samples are ignored. This
|
||||
behavior is retained to match the original FID score
|
||||
implementation.
|
||||
-- dims : Dimensionality of features returned by Inception
|
||||
-- device : Device to run calculations
|
||||
-- num_workers : Number of parallel dataloader workers
|
||||
|
||||
Returns:
|
||||
-- A numpy array of dimension (num images, dims) that contains the
|
||||
activations of the given tensor when feeding inception with the
|
||||
query tensor.
|
||||
"""
|
||||
model.eval()
|
||||
|
||||
if batch_size > len(files):
|
||||
print(('Warning: batch size is bigger than the data size. '
|
||||
'Setting batch size to data size'))
|
||||
batch_size = len(files)
|
||||
|
||||
dataset = ImagePathDataset(files, transforms=TF.ToTensor())
|
||||
dataloader = torch.utils.data.DataLoader(dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
drop_last=False,
|
||||
num_workers=num_workers)
|
||||
|
||||
pred_arr = np.empty((len(files), dims))
|
||||
|
||||
start_idx = 0
|
||||
|
||||
for batch in tqdm(dataloader):
|
||||
batch = batch.to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
pred = model(batch)[0]
|
||||
|
||||
# If model output is not scalar, apply global spatial average pooling.
|
||||
# This happens if you choose a dimensionality not equal 2048.
|
||||
if pred.size(2) != 1 or pred.size(3) != 1:
|
||||
pred = adaptive_avg_pool2d(pred, output_size=(1, 1))
|
||||
|
||||
pred = pred.squeeze(3).squeeze(2).cpu().numpy()
|
||||
|
||||
pred_arr[start_idx:start_idx + pred.shape[0]] = pred
|
||||
|
||||
start_idx = start_idx + pred.shape[0]
|
||||
|
||||
return pred_arr
|
||||
|
||||
|
||||
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
|
||||
"""Numpy implementation of the Frechet Distance.
|
||||
The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
|
||||
and X_2 ~ N(mu_2, C_2) is
|
||||
d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
|
||||
|
||||
Stable version by Dougal J. Sutherland.
|
||||
|
||||
Params:
|
||||
-- mu1 : Numpy array containing the activations of a layer of the
|
||||
inception net (like returned by the function 'get_predictions')
|
||||
for generated samples.
|
||||
-- mu2 : The sample mean over activations, precalculated on an
|
||||
representative data set.
|
||||
-- sigma1: The covariance matrix over activations for generated samples.
|
||||
-- sigma2: The covariance matrix over activations, precalculated on an
|
||||
representative data set.
|
||||
|
||||
Returns:
|
||||
-- : The Frechet Distance.
|
||||
"""
|
||||
|
||||
mu1 = np.atleast_1d(mu1)
|
||||
mu2 = np.atleast_1d(mu2)
|
||||
|
||||
sigma1 = np.atleast_2d(sigma1)
|
||||
sigma2 = np.atleast_2d(sigma2)
|
||||
|
||||
assert mu1.shape == mu2.shape, \
|
||||
'Training and test mean vectors have different lengths'
|
||||
assert sigma1.shape == sigma2.shape, \
|
||||
'Training and test covariances have different dimensions'
|
||||
|
||||
diff = mu1 - mu2
|
||||
|
||||
# Product might be almost singular
|
||||
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
|
||||
if not np.isfinite(covmean).all():
|
||||
msg = ('fid calculation produces singular product; '
|
||||
'adding %s to diagonal of cov estimates') % eps
|
||||
print(msg)
|
||||
offset = np.eye(sigma1.shape[0]) * eps
|
||||
covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))
|
||||
|
||||
# Numerical error might give slight imaginary component
|
||||
if np.iscomplexobj(covmean):
|
||||
if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
|
||||
m = np.max(np.abs(covmean.imag))
|
||||
raise ValueError('Imaginary component {}'.format(m))
|
||||
covmean = covmean.real
|
||||
|
||||
tr_covmean = np.trace(covmean)
|
||||
|
||||
return (diff.dot(diff) + np.trace(sigma1)
|
||||
+ np.trace(sigma2) - 2 * tr_covmean)
|
||||
|
||||
|
||||
def calculate_activation_statistics(files, model, batch_size=50, dims=2048,
|
||||
device='cpu', num_workers=1):
|
||||
"""Calculation of the statistics used by the FID.
|
||||
Params:
|
||||
-- files : List of image files paths
|
||||
-- model : Instance of inception model
|
||||
-- batch_size : The images numpy array is split into batches with
|
||||
batch size batch_size. A reasonable batch size
|
||||
depends on the hardware.
|
||||
-- dims : Dimensionality of features returned by Inception
|
||||
-- device : Device to run calculations
|
||||
-- num_workers : Number of parallel dataloader workers
|
||||
|
||||
Returns:
|
||||
-- mu : The mean over samples of the activations of the pool_3 layer of
|
||||
the inception model.
|
||||
-- sigma : The covariance matrix of the activations of the pool_3 layer of
|
||||
the inception model.
|
||||
"""
|
||||
act = get_activations(files, model, batch_size, dims, device, num_workers)
|
||||
mu = np.mean(act, axis=0)
|
||||
sigma = np.cov(act, rowvar=False)
|
||||
return mu, sigma
|
||||
|
||||
|
||||
def compute_statistics_of_path(path, model, batch_size, dims, device,
|
||||
num_workers=1):
|
||||
if type(path) is not list and path.endswith('.npz'):
|
||||
with np.load(path) as f:
|
||||
m, s = f['mu'][:], f['sigma'][:]
|
||||
else:
|
||||
if type(path) is list:
|
||||
files = []
|
||||
for p in path:
|
||||
p = pathlib.Path(p)
|
||||
files += sorted([file for ext in IMAGE_EXTENSIONS for file in p.glob('*.{}'.format(ext))])
|
||||
files = sorted(files)
|
||||
else:
|
||||
path = pathlib.Path(path)
|
||||
files = sorted([file for ext in IMAGE_EXTENSIONS for file in path.glob('*.{}'.format(ext))])
|
||||
m, s = calculate_activation_statistics(files, model, batch_size, dims, device, num_workers)
|
||||
|
||||
return m, s
|
||||
|
||||
|
||||
def calculate_fid_given_paths(paths, batch_size=50, device="cuda:0", dims=2048, num_workers=1):
|
||||
"""Calculates the FID of two paths"""
|
||||
for p in paths:
|
||||
if type(p) is list:
|
||||
for subp in p:
|
||||
if not os.path.exists(subp):
|
||||
raise RuntimeError('Invalid path: %s' % subp)
|
||||
else:
|
||||
if not os.path.exists(p):
|
||||
raise RuntimeError('Invalid path: %s' % p)
|
||||
|
||||
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]
|
||||
|
||||
model = InceptionV3([block_idx]).to(device)
|
||||
|
||||
m1, s1 = compute_statistics_of_path(paths[0], model, batch_size,
|
||||
dims, device, num_workers)
|
||||
m2, s2 = compute_statistics_of_path(paths[1], model, batch_size,
|
||||
dims, device, num_workers)
|
||||
fid_value = calculate_frechet_distance(m1, s1, m2, s2)
|
||||
|
||||
return fid_value
|
||||
|
||||
|
||||
def save_fid_stats(paths, batch_size, device, dims, num_workers=1):
|
||||
"""Calculates the FID of two paths"""
|
||||
if not os.path.exists(paths[0]):
|
||||
raise RuntimeError('Invalid path: %s' % paths[0])
|
||||
|
||||
if os.path.exists(paths[1]):
|
||||
raise RuntimeError('Existing output file: %s' % paths[1])
|
||||
|
||||
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]
|
||||
|
||||
model = InceptionV3([block_idx]).to(device)
|
||||
|
||||
print(f"Saving statistics for {paths[0]}")
|
||||
|
||||
m1, s1 = compute_statistics_of_path(paths[0], model, batch_size,
|
||||
dims, device, num_workers)
|
||||
|
||||
np.savez_compressed(paths[1], mu=m1, sigma=s1)
|
||||
|
||||
|
||||
def main():
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.device is None:
|
||||
device = torch.device('cuda' if (torch.cuda.is_available()) else 'cpu')
|
||||
else:
|
||||
device = torch.device(args.device)
|
||||
|
||||
if args.num_workers is None:
|
||||
try:
|
||||
num_cpus = len(os.sched_getaffinity(0))
|
||||
except AttributeError:
|
||||
# os.sched_getaffinity is not available under Windows, use
|
||||
# os.cpu_count instead (which may not return the *available* number
|
||||
# of CPUs).
|
||||
num_cpus = os.cpu_count()
|
||||
|
||||
num_workers = min(num_cpus, 8) if num_cpus is not None else 0
|
||||
else:
|
||||
num_workers = args.num_workers
|
||||
|
||||
if args.save_stats:
|
||||
save_fid_stats(args.path, args.batch_size, device, args.dims, num_workers)
|
||||
return
|
||||
|
||||
fid_value = calculate_fid_given_paths(args.path,
|
||||
args.batch_size,
|
||||
device,
|
||||
args.dims,
|
||||
num_workers)
|
||||
print('FID: ', fid_value)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
python MARIOEval_generate.py \
|
||||
--dataset TMDBEval500 \
|
||||
--root /path/to/eval \
|
||||
--method stablediffusion \
|
||||
--gpu 0
|
||||
@@ -0,0 +1,342 @@
|
||||
# Copied from https://github.com/mseitzer/pytorch-fid/blob/0a754fb8e66021700478fd365b79c2eaa316e31b/src/pytorch_fid/inception.py
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torchvision
|
||||
|
||||
try:
|
||||
from torchvision.models.utils import load_state_dict_from_url
|
||||
except ImportError:
|
||||
from torch.utils.model_zoo import load_url as load_state_dict_from_url
|
||||
|
||||
# Inception weights ported to Pytorch from
|
||||
# http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz
|
||||
FID_WEIGHTS_URL = 'https://github.com/mseitzer/pytorch-fid/releases/download/fid_weights/pt_inception-2015-12-05-6726825d.pth' # noqa: E501
|
||||
|
||||
|
||||
class InceptionV3(nn.Module):
|
||||
"""Pretrained InceptionV3 network returning feature maps"""
|
||||
|
||||
# Index of default block of inception to return,
|
||||
# corresponds to output of final average pooling
|
||||
DEFAULT_BLOCK_INDEX = 3
|
||||
|
||||
# Maps feature dimensionality to their output blocks indices
|
||||
BLOCK_INDEX_BY_DIM = {
|
||||
64: 0, # First max pooling features
|
||||
192: 1, # Second max pooling featurs
|
||||
768: 2, # Pre-aux classifier features
|
||||
2048: 3 # Final average pooling features
|
||||
}
|
||||
|
||||
def __init__(self,
|
||||
output_blocks=(DEFAULT_BLOCK_INDEX,),
|
||||
resize_input=True,
|
||||
normalize_input=True,
|
||||
requires_grad=False,
|
||||
use_fid_inception=True):
|
||||
"""Build pretrained InceptionV3
|
||||
|
||||
Parameters
|
||||
----------
|
||||
output_blocks : list of int
|
||||
Indices of blocks to return features of. Possible values are:
|
||||
- 0: corresponds to output of first max pooling
|
||||
- 1: corresponds to output of second max pooling
|
||||
- 2: corresponds to output which is fed to aux classifier
|
||||
- 3: corresponds to output of final average pooling
|
||||
resize_input : bool
|
||||
If true, bilinearly resizes input to width and height 299 before
|
||||
feeding input to model. As the network without fully connected
|
||||
layers is fully convolutional, it should be able to handle inputs
|
||||
of arbitrary size, so resizing might not be strictly needed
|
||||
normalize_input : bool
|
||||
If true, scales the input from range (0, 1) to the range the
|
||||
pretrained Inception network expects, namely (-1, 1)
|
||||
requires_grad : bool
|
||||
If true, parameters of the model require gradients. Possibly useful
|
||||
for finetuning the network
|
||||
use_fid_inception : bool
|
||||
If true, uses the pretrained Inception model used in Tensorflow's
|
||||
FID implementation. If false, uses the pretrained Inception model
|
||||
available in torchvision. The FID Inception model has different
|
||||
weights and a slightly different structure from torchvision's
|
||||
Inception model. If you want to compute FID scores, you are
|
||||
strongly advised to set this parameter to true to get comparable
|
||||
results.
|
||||
"""
|
||||
super(InceptionV3, self).__init__()
|
||||
|
||||
self.resize_input = resize_input
|
||||
self.normalize_input = normalize_input
|
||||
self.output_blocks = sorted(output_blocks)
|
||||
self.last_needed_block = max(output_blocks)
|
||||
|
||||
assert self.last_needed_block <= 3, \
|
||||
'Last possible output block index is 3'
|
||||
|
||||
self.blocks = nn.ModuleList()
|
||||
|
||||
if use_fid_inception:
|
||||
inception = fid_inception_v3()
|
||||
else:
|
||||
inception = _inception_v3(weights='DEFAULT')
|
||||
|
||||
# Block 0: input to maxpool1
|
||||
block0 = [
|
||||
inception.Conv2d_1a_3x3,
|
||||
inception.Conv2d_2a_3x3,
|
||||
inception.Conv2d_2b_3x3,
|
||||
nn.MaxPool2d(kernel_size=3, stride=2)
|
||||
]
|
||||
self.blocks.append(nn.Sequential(*block0))
|
||||
|
||||
# Block 1: maxpool1 to maxpool2
|
||||
if self.last_needed_block >= 1:
|
||||
block1 = [
|
||||
inception.Conv2d_3b_1x1,
|
||||
inception.Conv2d_4a_3x3,
|
||||
nn.MaxPool2d(kernel_size=3, stride=2)
|
||||
]
|
||||
self.blocks.append(nn.Sequential(*block1))
|
||||
|
||||
# Block 2: maxpool2 to aux classifier
|
||||
if self.last_needed_block >= 2:
|
||||
block2 = [
|
||||
inception.Mixed_5b,
|
||||
inception.Mixed_5c,
|
||||
inception.Mixed_5d,
|
||||
inception.Mixed_6a,
|
||||
inception.Mixed_6b,
|
||||
inception.Mixed_6c,
|
||||
inception.Mixed_6d,
|
||||
inception.Mixed_6e,
|
||||
]
|
||||
self.blocks.append(nn.Sequential(*block2))
|
||||
|
||||
# Block 3: aux classifier to final avgpool
|
||||
if self.last_needed_block >= 3:
|
||||
block3 = [
|
||||
inception.Mixed_7a,
|
||||
inception.Mixed_7b,
|
||||
inception.Mixed_7c,
|
||||
nn.AdaptiveAvgPool2d(output_size=(1, 1))
|
||||
]
|
||||
self.blocks.append(nn.Sequential(*block3))
|
||||
|
||||
for param in self.parameters():
|
||||
param.requires_grad = requires_grad
|
||||
|
||||
def forward(self, inp):
|
||||
"""Get Inception feature maps
|
||||
|
||||
Parameters
|
||||
----------
|
||||
inp : torch.autograd.Variable
|
||||
Input tensor of shape Bx3xHxW. Values are expected to be in
|
||||
range (0, 1)
|
||||
|
||||
Returns
|
||||
-------
|
||||
List of torch.autograd.Variable, corresponding to the selected output
|
||||
block, sorted ascending by index
|
||||
"""
|
||||
outp = []
|
||||
x = inp
|
||||
|
||||
if self.resize_input:
|
||||
x = F.interpolate(x,
|
||||
size=(299, 299),
|
||||
mode='bilinear',
|
||||
align_corners=False)
|
||||
|
||||
if self.normalize_input:
|
||||
x = 2 * x - 1 # Scale from range (0, 1) to range (-1, 1)
|
||||
|
||||
for idx, block in enumerate(self.blocks):
|
||||
x = block(x)
|
||||
if idx in self.output_blocks:
|
||||
outp.append(x)
|
||||
|
||||
if idx == self.last_needed_block:
|
||||
break
|
||||
|
||||
return outp
|
||||
|
||||
|
||||
def _inception_v3(*args, **kwargs):
|
||||
"""Wraps `torchvision.models.inception_v3`"""
|
||||
try:
|
||||
version = tuple(map(int, torchvision.__version__.split('.')[:2]))
|
||||
except ValueError:
|
||||
# Just a caution against weird version strings
|
||||
version = (0,)
|
||||
|
||||
# Skips default weight inititialization if supported by torchvision
|
||||
# version. See https://github.com/mseitzer/pytorch-fid/issues/28.
|
||||
if version >= (0, 6):
|
||||
kwargs['init_weights'] = False
|
||||
|
||||
# Backwards compatibility: `weights` argument was handled by `pretrained`
|
||||
# argument prior to version 0.13.
|
||||
if version < (0, 13) and 'weights' in kwargs:
|
||||
if kwargs['weights'] == 'DEFAULT':
|
||||
kwargs['pretrained'] = True
|
||||
elif kwargs['weights'] is None:
|
||||
kwargs['pretrained'] = False
|
||||
else:
|
||||
raise ValueError(
|
||||
'weights=={} not supported in torchvision {}'.format(
|
||||
kwargs['weights'], torchvision.__version__
|
||||
)
|
||||
)
|
||||
del kwargs['weights']
|
||||
|
||||
return torchvision.models.inception_v3(*args, **kwargs)
|
||||
|
||||
|
||||
def fid_inception_v3():
|
||||
"""Build pretrained Inception model for FID computation
|
||||
|
||||
The Inception model for FID computation uses a different set of weights
|
||||
and has a slightly different structure than torchvision's Inception.
|
||||
|
||||
This method first constructs torchvision's Inception and then patches the
|
||||
necessary parts that are different in the FID Inception model.
|
||||
"""
|
||||
inception = _inception_v3(num_classes=1008,
|
||||
aux_logits=False,
|
||||
weights=None)
|
||||
inception.Mixed_5b = FIDInceptionA(192, pool_features=32)
|
||||
inception.Mixed_5c = FIDInceptionA(256, pool_features=64)
|
||||
inception.Mixed_5d = FIDInceptionA(288, pool_features=64)
|
||||
inception.Mixed_6b = FIDInceptionC(768, channels_7x7=128)
|
||||
inception.Mixed_6c = FIDInceptionC(768, channels_7x7=160)
|
||||
inception.Mixed_6d = FIDInceptionC(768, channels_7x7=160)
|
||||
inception.Mixed_6e = FIDInceptionC(768, channels_7x7=192)
|
||||
inception.Mixed_7b = FIDInceptionE_1(1280)
|
||||
inception.Mixed_7c = FIDInceptionE_2(2048)
|
||||
|
||||
state_dict = load_state_dict_from_url(FID_WEIGHTS_URL, progress=True)
|
||||
inception.load_state_dict(state_dict)
|
||||
return inception
|
||||
|
||||
|
||||
class FIDInceptionA(torchvision.models.inception.InceptionA):
|
||||
"""InceptionA block patched for FID computation"""
|
||||
def __init__(self, in_channels, pool_features):
|
||||
super(FIDInceptionA, self).__init__(in_channels, pool_features)
|
||||
|
||||
def forward(self, x):
|
||||
branch1x1 = self.branch1x1(x)
|
||||
|
||||
branch5x5 = self.branch5x5_1(x)
|
||||
branch5x5 = self.branch5x5_2(branch5x5)
|
||||
|
||||
branch3x3dbl = self.branch3x3dbl_1(x)
|
||||
branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)
|
||||
branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)
|
||||
|
||||
# Patch: Tensorflow's average pool does not use the padded zero's in
|
||||
# its average calculation
|
||||
branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1,
|
||||
count_include_pad=False)
|
||||
branch_pool = self.branch_pool(branch_pool)
|
||||
|
||||
outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]
|
||||
return torch.cat(outputs, 1)
|
||||
|
||||
|
||||
class FIDInceptionC(torchvision.models.inception.InceptionC):
|
||||
"""InceptionC block patched for FID computation"""
|
||||
def __init__(self, in_channels, channels_7x7):
|
||||
super(FIDInceptionC, self).__init__(in_channels, channels_7x7)
|
||||
|
||||
def forward(self, x):
|
||||
branch1x1 = self.branch1x1(x)
|
||||
|
||||
branch7x7 = self.branch7x7_1(x)
|
||||
branch7x7 = self.branch7x7_2(branch7x7)
|
||||
branch7x7 = self.branch7x7_3(branch7x7)
|
||||
|
||||
branch7x7dbl = self.branch7x7dbl_1(x)
|
||||
branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl)
|
||||
branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl)
|
||||
branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl)
|
||||
branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl)
|
||||
|
||||
# Patch: Tensorflow's average pool does not use the padded zero's in
|
||||
# its average calculation
|
||||
branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1,
|
||||
count_include_pad=False)
|
||||
branch_pool = self.branch_pool(branch_pool)
|
||||
|
||||
outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool]
|
||||
return torch.cat(outputs, 1)
|
||||
|
||||
|
||||
class FIDInceptionE_1(torchvision.models.inception.InceptionE):
|
||||
"""First InceptionE block patched for FID computation"""
|
||||
def __init__(self, in_channels):
|
||||
super(FIDInceptionE_1, self).__init__(in_channels)
|
||||
|
||||
def forward(self, x):
|
||||
branch1x1 = self.branch1x1(x)
|
||||
|
||||
branch3x3 = self.branch3x3_1(x)
|
||||
branch3x3 = [
|
||||
self.branch3x3_2a(branch3x3),
|
||||
self.branch3x3_2b(branch3x3),
|
||||
]
|
||||
branch3x3 = torch.cat(branch3x3, 1)
|
||||
|
||||
branch3x3dbl = self.branch3x3dbl_1(x)
|
||||
branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)
|
||||
branch3x3dbl = [
|
||||
self.branch3x3dbl_3a(branch3x3dbl),
|
||||
self.branch3x3dbl_3b(branch3x3dbl),
|
||||
]
|
||||
branch3x3dbl = torch.cat(branch3x3dbl, 1)
|
||||
|
||||
# Patch: Tensorflow's average pool does not use the padded zero's in
|
||||
# its average calculation
|
||||
branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1,
|
||||
count_include_pad=False)
|
||||
branch_pool = self.branch_pool(branch_pool)
|
||||
|
||||
outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool]
|
||||
return torch.cat(outputs, 1)
|
||||
|
||||
|
||||
class FIDInceptionE_2(torchvision.models.inception.InceptionE):
|
||||
"""Second InceptionE block patched for FID computation"""
|
||||
def __init__(self, in_channels):
|
||||
super(FIDInceptionE_2, self).__init__(in_channels)
|
||||
|
||||
def forward(self, x):
|
||||
branch1x1 = self.branch1x1(x)
|
||||
|
||||
branch3x3 = self.branch3x3_1(x)
|
||||
branch3x3 = [
|
||||
self.branch3x3_2a(branch3x3),
|
||||
self.branch3x3_2b(branch3x3),
|
||||
]
|
||||
branch3x3 = torch.cat(branch3x3, 1)
|
||||
|
||||
branch3x3dbl = self.branch3x3dbl_1(x)
|
||||
branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)
|
||||
branch3x3dbl = [
|
||||
self.branch3x3dbl_3a(branch3x3dbl),
|
||||
self.branch3x3dbl_3b(branch3x3dbl),
|
||||
]
|
||||
branch3x3dbl = torch.cat(branch3x3dbl, 1)
|
||||
|
||||
# Patch: The FID Inception model uses max pooling instead of average
|
||||
# pooling. This is likely an error in this specific Inception
|
||||
# implementation, as other Inception models use average pooling here
|
||||
# (which matches the description in the paper).
|
||||
branch_pool = F.max_pool2d(x, kernel_size=3, stride=1, padding=1)
|
||||
branch_pool = self.branch_pool(branch_pool)
|
||||
|
||||
outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool]
|
||||
return torch.cat(outputs, 1)
|
||||
@@ -0,0 +1,98 @@
|
||||
# ------------------------------------------
|
||||
# TextDiffuser: Diffusion Models as Text Painters
|
||||
# Paper Link: https://arxiv.org/abs/2305.10855
|
||||
# Code Link: https://github.com/microsoft/unilm/tree/master/textdiffuser
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# This file provides the inference script.
|
||||
# ------------------------------------------
|
||||
|
||||
import os
|
||||
import re
|
||||
import copy
|
||||
|
||||
gts = {
|
||||
'ChineseDrawText': [],
|
||||
'DrawBenchText': [],
|
||||
'DrawTextCreative': [],
|
||||
'LAIONEval4000': [],
|
||||
'OpenLibraryEval500': [],
|
||||
'TMDBEval500': [],
|
||||
}
|
||||
|
||||
results = {
|
||||
'stablediffusion': {'cnt':0, 'p':0, 'r':0, 'f':0, 'acc':0},
|
||||
'textdiffuser': {'cnt':0, 'p':0, 'r':0, 'f':0, 'acc':0},
|
||||
'controlnet': {'cnt':0, 'p':0, 'r':0, 'f':0, 'acc':0},
|
||||
'deepfloyd': {'cnt':0, 'p':0, 'r':0, 'f':0, 'acc':0},
|
||||
}
|
||||
|
||||
def get_key_words(text: str):
|
||||
words = []
|
||||
text = text
|
||||
matches = re.findall(r"'(.*?)'", text) # find the keywords enclosed by ''
|
||||
if matches:
|
||||
for match in matches:
|
||||
words.extend(match.split())
|
||||
|
||||
return words
|
||||
|
||||
|
||||
# load gt
|
||||
files = os.listdir('/path/to/MARIOEval')
|
||||
for file in files:
|
||||
lines = open(os.path.join('/path/to/MARIOEval', file, f'{file}.txt')).readlines()
|
||||
for line in lines:
|
||||
line = line.strip().lower()
|
||||
gts[file].append(get_key_words(line))
|
||||
print(gts['ChineseDrawText'][:10])
|
||||
|
||||
|
||||
def get_p_r_acc(method, pred, gt):
|
||||
|
||||
pred = [p.strip().lower() for p in pred]
|
||||
gt = [g.strip().lower() for g in gt]
|
||||
|
||||
pred_orig = copy.deepcopy(pred)
|
||||
gt_orig = copy.deepcopy(gt)
|
||||
|
||||
pred_length = len(pred)
|
||||
gt_length = len(gt)
|
||||
|
||||
for p in pred:
|
||||
if p in gt_orig:
|
||||
pred_orig.remove(p)
|
||||
gt_orig.remove(p)
|
||||
|
||||
p = (pred_length - len(pred_orig)) / (pred_length + 1e-8)
|
||||
r = (gt_length - len(gt_orig)) / (gt_length + 1e-8)
|
||||
|
||||
pred_sorted = sorted(pred)
|
||||
gt_sorted = sorted(gt)
|
||||
if ''.join(pred_sorted) == ''.join(gt_sorted):
|
||||
acc = 1
|
||||
else:
|
||||
acc = 0
|
||||
|
||||
return p, r, acc
|
||||
|
||||
|
||||
files = os.listdir('/path/to/MaskTextSpotterV3/tools/ocr_result')
|
||||
print(len(files))
|
||||
|
||||
for file in files:
|
||||
method, dataset, prompt_index, image_index = file.strip().split('_')
|
||||
ocrs = open(os.path.join('/path/to/MaskTextSpotterV3/tools/ocr_result', file)).readlines()
|
||||
p, r, acc = get_p_r_acc(method, ocrs, gts[dataset][int(prompt_index)])
|
||||
results[method]['cnt'] += 1
|
||||
results[method]['p'] += p
|
||||
results[method]['r'] += r
|
||||
results[method]['acc'] += acc
|
||||
|
||||
for method in results.keys():
|
||||
results[method]['p'] /= results[method]['cnt']
|
||||
results[method]['r'] /= results[method]['cnt']
|
||||
results[method]['f'] = 2 * results[method]['p'] * results[method]['r'] / (results[method]['p'] + results[method]['r'] + 1e-8)
|
||||
results[method]['acc'] /= results[method]['cnt']
|
||||
|
||||
print(results)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
diffusers==0.16.0
|
||||
torch==2.0.1
|
||||
pycocoevalcap
|
||||
pytorch_fid
|
||||
sentencepiece
|
||||
-e git+https://github.com/openai/CLIP.git@main#egg=clip
|
||||
Reference in New Issue
Block a user