metrics.py 10.3 KB
Newer Older
1
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
2
3
4
5
6
7
8
9
10
11
12
13
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14

Gunho Park's avatar
Gunho Park committed
15
"""Evaluation metrics for BASNet.
16
17
18
19
20
21

The MAE and maxFscore implementations are a modified version of
https://github.com/xuebinqin/Binary-Segmentation-Evaluation-Tool

"""
import numpy as np
22
import scipy.signal
23
24


25
class MAE:
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
  """Mean Absolute Error(MAE) metric for basnet."""

  def __init__(self):
    """Constructs MAE metric class."""
    self.reset_states()

  @property
  def name(self):
    return 'MAE'

  def reset_states(self):
    """Resets internal states for a fresh run."""
    self._predictions = []
    self._groundtruths = []

  def result(self):
    """Evaluates segmentation results, and reset_states."""
    metric_result = self.evaluate()
    # Cleans up the internal variables in order for a fresh eval next time.
    self.reset_states()
    return metric_result

  def evaluate(self):
    """Evaluates with masks from all images.

    Returns:
      average_mae: average MAE with float numpy.
    """
    mae_total = 0.0

56
57
    for (true, pred) in zip(self._groundtruths, self._predictions):
      # Computes MAE
58
59
60
      mae = self._compute_mae(true, pred)
      mae_total += mae

61
    average_mae = mae_total / len(self._groundtruths)
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

    return average_mae

  def _mask_normalize(self, mask):
    return mask/(np.amax(mask)+1e-8)

  def _compute_mae(self, true, pred):
    h, w = true.shape[0], true.shape[1]
    mask1 = self._mask_normalize(true)
    mask2 = self._mask_normalize(pred)
    sum_error = np.sum(np.absolute((mask1.astype(float) - mask2.astype(float))))
    mae_error = sum_error/(float(h)*float(w)+1e-8)

    return mae_error

  def _convert_to_numpy(self, groundtruths, predictions):
    """Converts tesnors to numpy arrays."""
    numpy_groundtruths = groundtruths.numpy()
    numpy_predictions = predictions.numpy()
81

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    return numpy_groundtruths, numpy_predictions

  def update_state(self, groundtruths, predictions):
    """Update segmentation results and groundtruth data.

    Args:
      groundtruths : Tuple of single Tensor [batch, width, height, 1],
                     groundtruth masks. range [0, 1]
      predictions  : Tuple of single Tensor [batch, width, height, 1],
                     predicted masks. range [0, 1]
    """
    groundtruths, predictions = self._convert_to_numpy(groundtruths[0],
                                                       predictions[0])
    for (true, pred) in zip(groundtruths, predictions):
      self._groundtruths.append(true)
      self._predictions.append(pred)


100
class MaxFscore:
101
  """Maximum F-score metric for basnet."""
102

103
104
105
106
107
108
  def __init__(self):
    """Constructs BASNet evaluation class."""
    self.reset_states()

  @property
  def name(self):
109
    return 'MaxFScore'
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

  def reset_states(self):
    """Resets internal states for a fresh run."""
    self._predictions = []
    self._groundtruths = []

  def result(self):
    """Evaluates segmentation results, and reset_states."""
    metric_result = self.evaluate()
    # Cleans up the internal variables in order for a fresh eval next time.
    self.reset_states()
    return metric_result

  def evaluate(self):
    """Evaluates with masks from all images.

    Returns:
      f_max: maximum F-score value.
    """

    mybins = np.arange(0, 256)
    beta = 0.3
    precisions = np.zeros((len(self._groundtruths), len(mybins)-1))
    recalls = np.zeros((len(self._groundtruths), len(mybins)-1))

    for i, (true, pred) in enumerate(zip(self._groundtruths,
                                         self._predictions)):
      # Compute F-score
138
139
140
      true = self._mask_normalize(true) * 255.0
      pred = self._mask_normalize(pred) * 255.0
      pre, rec = self._compute_pre_rec(true, pred, mybins=np.arange(0, 256))
141

142
143
      precisions[i, :] = pre
      recalls[i, :] = rec
144

145
146
147
148
    precisions = np.sum(precisions, 0) / (len(self._groundtruths) + 1e-8)
    recalls = np.sum(recalls, 0) / (len(self._groundtruths) + 1e-8)
    f = (1 + beta) * precisions * recalls / (beta * precisions + recalls + 1e-8)
    f_max = np.max(f)
149
150
151
152
153
    f_max = f_max.astype(np.float32)

    return f_max

  def _mask_normalize(self, mask):
154
    return mask / (np.amax(mask) + 1e-8)
155

156
157
  def _compute_pre_rec(self, true, pred, mybins=np.arange(0, 256)):
    """Computes relaxed precision and recall."""
158
    # pixel number of ground truth foreground regions
159
160
    gt_num = true[true > 128].size

161
    # mask predicted pixel values in the ground truth foreground region
162
    pp = pred[true > 128]
163
    # mask predicted pixel values in the ground truth bacground region
164
    nn = pred[true <= 128]
165

166
167
    pp_hist, _ = np.histogram(pp, bins=mybins)
    nn_hist, _ = np.histogram(nn, bins=mybins)
168
169
170
171
172
173
174

    pp_hist_flip = np.flipud(pp_hist)
    nn_hist_flip = np.flipud(nn_hist)

    pp_hist_flip_cum = np.cumsum(pp_hist_flip)
    nn_hist_flip_cum = np.cumsum(nn_hist_flip)

175
176
177
    precision = pp_hist_flip_cum / (pp_hist_flip_cum + nn_hist_flip_cum + 1e-8
                                   )  # TP/(TP+FP)
    recall = pp_hist_flip_cum / (gt_num + 1e-8)  # TP/(TP+FN)
178

179
    precision[np.isnan(precision)] = 0.0
180
181
182
183
184
    recall[np.isnan(recall)] = 0.0

    pre_len = len(precision)
    rec_len = len(recall)

185
    return np.reshape(precision, (pre_len)), np.reshape(recall, (rec_len))
186
187
188
189
190

  def _convert_to_numpy(self, groundtruths, predictions):
    """Converts tesnors to numpy arrays."""
    numpy_groundtruths = groundtruths.numpy()
    numpy_predictions = predictions.numpy()
191

192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    return numpy_groundtruths, numpy_predictions

  def update_state(self, groundtruths, predictions):
    """Update segmentation results and groundtruth data.

    Args:
      groundtruths : Tuple of single Tensor [batch, width, height, 1],
                     groundtruth masks. range [0, 1]
      predictions  : Tuple of signle Tensor [batch, width, height, 1],
                     predicted masks. range [0, 1]
    """
    groundtruths, predictions = self._convert_to_numpy(groundtruths[0],
                                                       predictions[0])
    for (true, pred) in zip(groundtruths, predictions):
      self._groundtruths.append(true)
      self._predictions.append(pred)

209
210

class RelaxedFscore:
211
212
213
214
215
216
217
218
  """Relaxed F-score metric for basnet."""

  def __init__(self):
    """Constructs BASNet evaluation class."""
    self.reset_states()

  @property
  def name(self):
219
    return 'RelaxFScore'
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243

  def reset_states(self):
    """Resets internal states for a fresh run."""
    self._predictions = []
    self._groundtruths = []

  def result(self):
    """Evaluates segmentation results, and reset_states."""
    metric_result = self.evaluate()
    # Cleans up the internal variables in order for a fresh eval next time.
    self.reset_states()
    return metric_result

  def evaluate(self):
    """Evaluates with masks from all images.

    Returns:
      relax_f: relaxed F-score value.
    """

    beta = 0.3
    rho = 3
    relax_fs = np.zeros(len(self._groundtruths))

244
    erode_kernel = np.ones((3, 3))
245

246
247
    for i, (true,
            pred) in enumerate(zip(self._groundtruths, self._predictions)):
248
249
250
251
252
253
      true = self._mask_normalize(true)
      pred = self._mask_normalize(pred)

      true = np.squeeze(true, axis=-1)
      pred = np.squeeze(pred, axis=-1)
      # binary saliency mask (S_bw), threshold 0.5
254
255
      pred[pred >= 0.5] = 1
      pred[pred < 0.5] = 0
256
257
      # compute eroded binary mask (S_erd) of S_bw
      pred_erd = self._compute_erosion(pred, erode_kernel)
258

259
260
261
      pred_xor = np.logical_xor(pred_erd, pred)
      # convert True/False to 1/0
      pred_xor = pred_xor * 1
262

263
      # same method for ground truth
264
265
      true[true >= 0.5] = 1
      true[true < 0.5] = 0
266
267
268
      true_erd = self._compute_erosion(true, erode_kernel)
      true_xor = np.logical_xor(true_erd, true)
      true_xor = true_xor * 1
269

270
      pre, rec = self._compute_relax_pre_rec(true_xor, pred_xor, rho)
271
      relax_fs[i] = (1 + beta) * pre * rec / (beta * pre + rec + 1e-8)
272

273
    relax_f = np.sum(relax_fs, 0) / (len(self._groundtruths) + 1e-8)
274
275
276
277
278
279
280
281
282
    relax_f = relax_f.astype(np.float32)

    return relax_f

  def _mask_normalize(self, mask):
    return mask/(np.amax(mask)+1e-8)

  def _compute_erosion(self, mask, kernel):
    kernel_full = np.sum(kernel)
283
284
285
    mask_erd = scipy.signal.convolve2d(mask, kernel, mode='same')
    mask_erd[mask_erd < kernel_full] = 0
    mask_erd[mask_erd == kernel_full] = 1
286
287
288
    return mask_erd

  def _compute_relax_pre_rec(self, true, pred, rho):
289
290
    """Computes relaxed precision and recall."""
    kernel = np.ones((2 * rho - 1, 2 * rho - 1))
291
292
293
    map_zeros = np.zeros_like(pred)
    map_ones = np.ones_like(pred)

294
    pred_filtered = scipy.signal.convolve2d(pred, kernel, mode='same')
295
    # True positive for relaxed precision
296
297
298
299
    relax_pre_tp = np.where((true == 1) & (pred_filtered > 0), map_ones,
                            map_zeros)

    true_filtered = scipy.signal.convolve2d(true, kernel, mode='same')
300
    # True positive for relaxed recall
301
302
303
304
305
    relax_rec_tp = np.where((pred == 1) & (true_filtered > 0), map_ones,
                            map_zeros)

    return np.sum(relax_pre_tp) / np.sum(pred), np.sum(relax_rec_tp) / np.sum(
        true)
306
307
308
309
310

  def _convert_to_numpy(self, groundtruths, predictions):
    """Converts tesnors to numpy arrays."""
    numpy_groundtruths = groundtruths.numpy()
    numpy_predictions = predictions.numpy()
311

312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
    return numpy_groundtruths, numpy_predictions

  def update_state(self, groundtruths, predictions):
    """Update segmentation results and groundtruth data.

    Args:
      groundtruths : Tuple of single Tensor [batch, width, height, 1],
                     groundtruth masks. range [0, 1]
      predictions  : Tuple of single Tensor [batch, width, height, 1],
                     predicted masks. range [0, 1]
    """

    groundtruths, predictions = self._convert_to_numpy(groundtruths[0],
                                                       predictions[0])
    for (true, pred) in zip(groundtruths, predictions):
      self._groundtruths.append(true)
      self._predictions.append(pred)