-
Notifications
You must be signed in to change notification settings - Fork 1
/
utils.py
152 lines (141 loc) · 8.22 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
import numpy as np
import json
from math import floor
from typing import Union, Callable
import torch
import torch.nn.functional as F
RETRIEVAL_TASKS = ['imagecode', 'imagecode_video', 'flickr30k', 'imagenet', 'clevr', 'svo_verb', 'svo_subj', 'svo_obj', 'pets', 'flickr30k_text', 'vg_relation', 'vg_attribution', 'coco_order', 'flickr30k_order', 'mscoco_val']
def evaluate_winoground(scores):
text_score, img_score, group_score = [], [], []
for score_ in scores:
c0_i0, c0_i1, c1_i0, c1_i1 = score_
text_score_ = 1 if c0_i0 > c1_i0 and c1_i1 > c0_i1 else 0
img_score_ = 1 if c0_i0 > c0_i1 and c1_i1 > c1_i0 else 0
group_score_ = 1 if text_score_ and img_score_ else 0
text_score.append(text_score_)
img_score.append(img_score_)
group_score.append(group_score_)
return text_score, img_score, group_score
def evaluate_retrieval(args, scores, img_idx):
if type(scores) != list:
img_idx = img_idx.cpu().numpy()
scores = scores.cpu().numpy()
scores = np.stack(scores, axis=0)
retrieval_accuracy = []
max_more_than_once = 0
print(scores.shape)
print(img_idx.shape)
for i in range(scores.shape[0]):
number_of_argmax_appearances = np.sum(scores[i] == np.max(scores[i]))
if number_of_argmax_appearances > 1:
max_more_than_once += 1
if img_idx[i] == np.argmax(scores[i]):
retrieval_accuracy.append(1)
else:
retrieval_accuracy.append(0)
# R5 calculation too
if args.task in ['flickr30k', 'imagecode', 'imagenet', 'flickr30k_text']:
r5 = []
for i in range(scores.shape[0]):
if img_idx[i] in np.argsort(scores[i])[-5:]:
r5.append(1)
else:
r5.append(0)
return retrieval_accuracy, r5, max_more_than_once
else:
return retrieval_accuracy, max_more_than_once
def evaluate_bias(args, good_scores, bad_scores, img_idx):
img_idx = img_idx.cpu().numpy()
good_scores = good_scores.cpu().numpy()
bad_scores = bad_scores.cpu().numpy()
phis = {}
for i in range(len(good_scores)): # rows of tensor are images, columns are the words
# p val test just needs the phi(w,A,B) which i have! just code it elionrrr
class_idx = int(img_idx[i]) # get class, should be an integer {0,1,...,7}
good_score = good_scores[i].mean() # mean_{a\in A} sigma(x,a)
bad_score = bad_scores[i].mean() # mean_{b\in B} sigma(x,b)
phi = good_score-bad_score # phi(w,A,B) = mean_{a\in A} sigma(x,a) - mean_{b\in B} sigma(x,b)
if class_idx in phis:
phis[class_idx].append(phi)
else:
phis[class_idx] = [phi]
return phis#, raw_scores
def evaluate_gender_bias(args, m_attr_scores, f_attr_scores, class_ids):
entity = class_ids[0].split('_')[-1] # either clothes, drinks, or bags
male_filter = np.array(class_ids)==f'male_{entity}' # indices of scores of male images
female_filter = np.array(class_ids)==f'female_{entity}' # indices of scores of female images
m_attr_scores = m_attr_scores.cpu().numpy() # all the images scored with the male attr word
f_attr_scores = f_attr_scores.cpu().numpy() # all the images scores w female attr word
m_imgs_m_attr = m_attr_scores[male_filter]
m_imgs_f_attr = f_attr_scores[male_filter]
f_imgs_m_attr = m_attr_scores[female_filter]
f_imgs_f_attr = f_attr_scores[female_filter]
phi_male = m_imgs_m_attr - m_imgs_f_attr #phi(m,w_m,w_f) = sigma(m,w_m)-sigma(m,w_f)
phi_female = f_imgs_m_attr - f_imgs_f_attr #phi(f,w_m,w_f) = sigma(f,w_m)-sigma(f,w_f)
return {f'male_{entity}':phi_male,f'female_{entity}':phi_female}
def save_bias_scores(fname, bias_scores):
with open(fname, 'w') as f:
print(bias_scores)
json.dump(bias_scores,f)
f.close()
return bias_scores
def save_bias_results(fname, bias_scores, task):
if task == 'mmbias':
with open(fname, 'w') as f:
christian = bias_scores[0]
muslim = bias_scores[1]
jewish = bias_scores[2]
hindu = bias_scores[3]
american = bias_scores[4]
arab = bias_scores[5]
hetero = bias_scores[6]
lgbt = bias_scores[7]
buddhist = bias_scores[8]
f.write(f'Christian-Muslim bias score {(np.mean(christian)-np.mean(muslim))/(np.concatenate((christian,muslim)).std())}\n')
f.write(f'Christian-Jewish bias score {(np.mean(christian)-np.mean(jewish))/(np.concatenate((christian,jewish)).std())}\n')
f.write(f'Jewish-Muslim bias score {(np.mean(jewish)-np.mean(muslim))/(np.concatenate((jewish,muslim)).std())}\n')
f.write(f'Buddhist-Muslim bias score {(np.mean(buddhist)-np.mean(muslim))/(np.concatenate((buddhist,muslim)).std())}\n')
f.write(f'Buddhist-Christian bias score {(np.mean(buddhist)-np.mean(christian))/(np.concatenate((christian,buddhist)).std())}\n')
f.write(f'Buddhist-Hindu bias score {(np.mean(buddhist)-np.mean(hindu))/(np.concatenate((christian,hindu)).std())}\n')
f.write(f'Buddhist-Jewish bias score {(np.mean(buddhist)-np.mean(jewish))/(np.concatenate((christian,jewish)).std())}\n')
f.write(f'Hindu-Muslim bias score {(np.mean(hindu)-np.mean(muslim))/(np.concatenate((hindu,muslim)).std())}\n')
f.write(f'American-Arab bias score {(np.mean(american)-np.mean(arab))/(np.concatenate((american,arab)).std())}\n')
f.write(f'Hetero-LGBT bias score {(np.mean(hetero)-np.mean(lgbt))/(np.concatenate((hetero,lgbt)).std())}\n')
f.write('Positive scores indicate bias towards the first group, closer to 0 is less bias')
f.close()
elif task == 'genderbias':
with open(fname, 'w') as f:
#bias_scores = {'male_clothes': [], 'female_clothes': [], 'male_bags': [], 'female_bags': [], 'male_drinks': [], 'female_drinks': []}
f.write(f"Clothes bias score {(np.mean(bias_scores['male_clothes'])-np.mean(bias_scores['female_clothes']))/(np.concatenate((bias_scores['male_clothes'],bias_scores['female_clothes'])).std())}\n")
f.write(f"Bags bias score {(np.mean(bias_scores['male_bags'])-np.mean(bias_scores['female_bags']))/(np.concatenate((bias_scores['male_bags'],bias_scores['female_bags'])).std())}\n")
f.write(f"Drinks bias score {(np.mean(bias_scores['male_drinks'])-np.mean(bias_scores['female_drinks']))/(np.concatenate((bias_scores['male_drinks'],bias_scores['female_drinks'])).std())}\n")
f.write('Positive scores indicate bias towards males, closer to 0 is less bias')
f.close()
return bias_scores # returns no changes
def evaluate_scores(args, scores, batch):
if args.task == 'winoground':
score = evaluate_winoground(scores)
elif args.task == 'mmbias':
# so we have a bunch of scores, which is a tensor Size([batchsize,len(texts)])
# example for 4 texts and batchsize 2
# scores = tensor([[ 0.0555, 0.0121, 0.0113,mmOKxRfPbYjE -0.0000],
# [ 0.0398, -0.0133, -0.0340, -0.0391]], device='cuda:7')
text_len = floor(len(batch[1])/2) # number of good / bad texts
good_scores = scores[:, :text_len] # extract the first len(good_texts) cols for pleasant_texts
bad_scores = scores[:, text_len:] # extract the remaining cols for unpleasant_texts
assert len(good_scores) == len(bad_scores)
img_idx = batch[-1] # tensor of class_ids
return evaluate_bias(args, good_scores, bad_scores, img_idx) # dictionary of lists of phis
elif args.task == 'genderbias':
# input is list of scores (tensors whatever), ill use batchsize 6 so its just one text and one fe/male_entity
# evaluate_gender_bias should return a just the phi for the one class
male_attr_scores = scores[:,0]
female_attr_scores = scores[:,-1]
class_ids = batch[-1]
return evaluate_gender_bias(args, male_attr_scores, female_attr_scores, class_ids)
elif args.task in RETRIEVAL_TASKS:
img_idx = batch[-1]
score = evaluate_retrieval(args, scores, img_idx)
else:
raise NotImplementedError
return score