first commit

ppocr/postprocess/cls_postprocess.py

+import paddle
+
+
+class ClsPostProcess(object):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, label_list=None, key=None, **kwargs):
+        super(ClsPostProcess, self).__init__()
+        self.label_list = label_list
+        self.key = key
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        if self.key is not None:
+            preds = preds[self.key]
+
+        label_list = self.label_list
+        if label_list is None:
+            label_list = {idx: idx for idx in range(preds.shape[-1])}
+
+        if isinstance(preds, paddle.Tensor):
+            preds = preds.numpy()
+
+        pred_idxs = preds.argmax(axis=1)
+        decode_out = [(label_list[idx], preds[i, idx])
+                      for i, idx in enumerate(pred_idxs)]
+        if label is None:
+            return decode_out
+        label = [(label_list[idx], 1.0) for idx in label]
+        return decode_out, label

ppocr/postprocess/db_postprocess.py

+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import cv2
+import paddle
+from shapely.geometry import Polygon
+import pyclipper
+
+
+class DBPostProcess(object):
+    """
+    The post process for Differentiable Binarization (DB).
+    """
+
+    def __init__(self,
+                 thresh=0.3,
+                 box_thresh=0.7,
+                 max_candidates=1000,
+                 unclip_ratio=2.0,
+                 use_dilation=False,
+                 score_mode="fast",
+                 visual_output=False,
+                 **kwargs):
+        self.thresh = thresh
+        self.box_thresh = box_thresh
+        self.max_candidates = max_candidates
+        self.unclip_ratio = unclip_ratio
+        self.min_size = 3
+        self.score_mode = score_mode
+        assert score_mode in [
+            "slow", "fast"
+        ], "Score mode must be in [slow, fast] but got: {}".format(score_mode)
+
+        self.dilation_kernel = None if not use_dilation else np.array(
+            [[1, 1], [1, 1]])
+        self.visual = visual_output
+    
+    def boxes_from_bitmap_section(self, pred, _bitmap, dest_width, dest_height, resize_h, resize_w):
+        '''
+        _bitmap: single map with shape (1, H, W),
+                whose values are binarized as {0, 1}
+        '''
+
+        bitmap = _bitmap
+        height, width = bitmap.shape
+        offset_h = int((height - resize_h) / 2)
+        offset_w = int((width - resize_w) / 2)
+
+        outs = cv2.findContours((bitmap * 255).astype(np.uint8), cv2.RETR_LIST,
+                                cv2.CHAIN_APPROX_SIMPLE)
+        if len(outs) == 3:
+            img, contours, _ = outs[0], outs[1], outs[2]
+        elif len(outs) == 2:
+            contours, _ = outs[0], outs[1]
+
+        num_contours = min(len(contours), self.max_candidates)
+
+        boxes = []
+        scores = []
+        for index in range(num_contours):
+            contour = contours[index]
+            points, sside = self.get_mini_boxes(contour)
+            if sside < self.min_size:
+                continue
+            points = np.array(points)
+            if self.score_mode == "fast":
+                score = self.box_score_fast(pred, points.reshape(-1, 2))
+            else:
+                score = self.box_score_slow(pred, contour)
+            if self.box_thresh > score:
+                continue
+
+            box = self.unclip(points).reshape(-1, 1, 2)
+            box, sside = self.get_mini_boxes(box)
+            if sside < self.min_size + 2:
+                continue
+            box = np.array(box)
+
+            box[:, 0] = np.clip(
+                np.round((box[:, 0] - offset_w) / resize_w * dest_width), 0, dest_width)
+            box[:, 1] = np.clip(
+                np.round((box[:, 1] - offset_h) / resize_h * dest_height), 0, dest_height)
+            boxes.append(box.astype(np.int16))
+            scores.append(score)
+        return np.array(boxes, dtype=np.int16), scores    
+
+    def unclip(self, box):
+        unclip_ratio = self.unclip_ratio
+        poly = Polygon(box)
+        distance = poly.area * unclip_ratio / poly.length
+        offset = pyclipper.PyclipperOffset()
+        offset.AddPath(box, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
+        expanded = np.array(offset.Execute(distance))
+        return expanded
+
+    def get_mini_boxes(self, contour):
+        bounding_box = cv2.minAreaRect(contour)
+        points = sorted(list(cv2.boxPoints(bounding_box)), key=lambda x: x[0])
+
+        index_1, index_2, index_3, index_4 = 0, 1, 2, 3
+        if points[1][1] > points[0][1]:
+            index_1 = 0
+            index_4 = 1
+        else:
+            index_1 = 1
+            index_4 = 0
+        if points[3][1] > points[2][1]:
+            index_2 = 2
+            index_3 = 3
+        else:
+            index_2 = 3
+            index_3 = 2
+
+        box = [
+            points[index_1], points[index_2], points[index_3], points[index_4]
+        ]
+        return box, min(bounding_box[1])
+
+    def box_score_fast(self, bitmap, _box):
+        '''
+        box_score_fast: use bbox mean score as the mean score
+        '''
+        h, w = bitmap.shape[:2]
+        box = _box.copy()
+        xmin = np.clip(np.floor(box[:, 0].min()).astype(int), 0, w - 1)
+        xmax = np.clip(np.ceil(box[:, 0].max()).astype(int), 0, w - 1)
+        ymin = np.clip(np.floor(box[:, 1].min()).astype(int), 0, h - 1)
+        ymax = np.clip(np.ceil(box[:, 1].max()).astype(int), 0, h - 1)
+
+        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
+        box[:, 0] = box[:, 0] - xmin
+        box[:, 1] = box[:, 1] - ymin
+        cv2.fillPoly(mask, box.reshape(1, -1, 2).astype(np.int32), 1)
+        return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0]
+
+    def box_score_slow(self, bitmap, contour):
+        '''
+        box_score_slow: use polyon mean score as the mean score
+        '''
+        h, w = bitmap.shape[:2]
+        contour = contour.copy()
+        contour = np.reshape(contour, (-1, 2))
+
+        xmin = np.clip(np.min(contour[:, 0]), 0, w - 1)
+        xmax = np.clip(np.max(contour[:, 0]), 0, w - 1)
+        ymin = np.clip(np.min(contour[:, 1]), 0, h - 1)
+        ymax = np.clip(np.max(contour[:, 1]), 0, h - 1)
+
+        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
+
+        contour[:, 0] = contour[:, 0] - xmin
+        contour[:, 1] = contour[:, 1] - ymin
+
+        cv2.fillPoly(mask, contour.reshape(1, -1, 2).astype(np.int32), 1)
+        return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0]
+
+    def visual_output(self, pred):
+        im = np.array(pred[0] * 255).astype(np.uint8)
+        cv2.imwrite("db_probability_map.png", im)
+        print("The probalibity map is visualized in db_probability_map.png")
+
+    def __call__(self, outs_dict, shape_list):
+        pred = outs_dict['maps']
+        if isinstance(pred, paddle.Tensor):
+            pred = pred.numpy()
+        pred = pred[:, 0, :, :]
+        segmentation = pred > self.thresh
+        if self.visual:
+            self.visual_output(pred)
+
+        boxes_batch = []
+        for batch_index in range(pred.shape[0]):
+            # src_h, src_w, ratio_h, ratio_w = shape_list[batch_index]
+            src_h, src_w, ratio_h, ratio_w, resize_h, resize_w = shape_list[batch_index]
+            if self.dilation_kernel is not None:
+                mask = cv2.dilate(
+                    np.array(segmentation[batch_index]).astype(np.uint8),
+                    self.dilation_kernel)
+            else:
+                mask = segmentation[batch_index]
+            # boxes, scores = self.boxes_from_bitmap(pred[batch_index], mask,
+            #                                        src_w, src_h)
+            boxes, scores = self.boxes_from_bitmap_section(pred[batch_index], mask,
+                                                   src_w, src_h, resize_h, resize_w)
+
+            boxes_batch.append({'points': boxes})
+        return boxes_batch
+

ppocr/postprocess/rec_postprocess.py

+import numpy as np
+import paddle
+from paddle.nn import functional as F
+import re
+
+
+class BaseRecLabelDecode(object):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False):
+        self.beg_str = "sos"
+        self.end_str = "eos"
+
+        self.character_str = []
+        if character_dict_path is None:
+            self.character_str = "0123456789abcdefghijklmnopqrstuvwxyz"
+            dict_character = list(self.character_str)
+        else:
+            with open(character_dict_path, "rb") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    line = line.decode('utf-8').strip("\n").strip("\r\n")
+                    self.character_str.append(line)
+            if use_space_char:
+                self.character_str.append(" ")
+            dict_character = list(self.character_str)
+
+        dict_character = self.add_special_char(dict_character)
+        self.dict = {}
+        for i, char in enumerate(dict_character):
+            self.dict[char] = i
+        self.character = dict_character
+
+    def add_special_char(self, dict_character):
+        return dict_character
+
+    def decode(self, text_index, text_prob=None, is_remove_duplicate=False):
+        """ convert text-index into text-label. """
+        result_list = []
+        ignored_tokens = self.get_ignored_tokens()
+        batch_size = len(text_index)
+        for batch_idx in range(batch_size):
+            selection = np.ones(len(text_index[batch_idx]), dtype=bool)
+            if is_remove_duplicate:
+                selection[1:] = text_index[batch_idx][1:] != text_index[
+                    batch_idx][:-1]
+            for ignored_token in ignored_tokens:
+                selection &= text_index[batch_idx] != ignored_token
+
+            char_list = [
+                self.character[text_id - 1]
+                for text_id in text_index[batch_idx][selection]
+            ]
+            if text_prob is not None:
+                conf_list = text_prob[batch_idx][selection]
+            else:
+                conf_list = [1] * len(selection)
+            if len(conf_list) == 0:
+                conf_list = [0]
+
+            text = ''.join(char_list)
+            result_list.append((text, np.mean(conf_list).tolist()))
+        return result_list
+    
+
+    def get_ignored_tokens(self):
+        return [0]  # for ctc blank
+
+
+class CTCLabelDecode(BaseRecLabelDecode):
+    """ Convert between text-label and text-index """
+
+    def __init__(self, character_dict_path=None, use_space_char=False,
+                 **kwargs):
+        super(CTCLabelDecode, self).__init__(character_dict_path,
+                                             use_space_char)
+
+    def __call__(self, preds, label=None, *args, **kwargs):
+        if isinstance(preds, tuple) or isinstance(preds, list):
+            preds = preds[-1]
+        if isinstance(preds, paddle.Tensor):
+            preds = preds.numpy()
+        preds_idx = preds.argmax(axis=2)
+        preds_prob = preds.max(axis=2)
+        preds_idx = preds_idx.transpose(1,0)
+        preds_prob = preds_prob.transpose(1,0)
+        text = self.decode(preds_idx, preds_prob, is_remove_duplicate=True)
+        if label is None:
+            return text
+        label = self.decode(label)
+        return text, label
+
+    def add_special_char(self, dict_character):
+        dict_character = ['blank'] + dict_character
+        return dict_character
+

ppocr/utils/EN_symbol_dict.txt

+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+^
+_
+`
+{
+|
+}
+~
\ No newline at end of file

ppocr/utils/dict90.txt

+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+_
+`
+~
\ No newline at end of file

ppocr/utils/e2e_metric/Deteval.py

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import scipy.io as io
+from ppocr.utils.e2e_metric.polygon_fast import iod, area_of_intersection, area
+
+
+def get_socre_A(gt_dir, pred_dict):
+    allInputs = 1
+
+    def input_reading_mod(pred_dict):
+        """This helper reads input from txt files"""
+        det = []
+        n = len(pred_dict)
+        for i in range(n):
+            points = pred_dict[i]['points']
+            text = pred_dict[i]['texts']
+            point = ",".join(map(str, points.reshape(-1, )))
+            det.append([point, text])
+        return det
+
+    def gt_reading_mod(gt_dict):
+        """This helper reads groundtruths from mat files"""
+        gt = []
+        n = len(gt_dict)
+        for i in range(n):
+            points = gt_dict[i]['points'].tolist()
+            h = len(points)
+            text = gt_dict[i]['text']
+            xx = [
+                np.array(
+                    ['x:'], dtype='<U2'), 0, np.array(
+                        ['y:'], dtype='<U2'), 0, np.array(
+                            ['#'], dtype='<U1'), np.array(
+                                ['#'], dtype='<U1')
+            ]
+            t_x, t_y = [], []
+            for j in range(h):
+                t_x.append(points[j][0])
+                t_y.append(points[j][1])
+            xx[1] = np.array([t_x], dtype='int16')
+            xx[3] = np.array([t_y], dtype='int16')
+            if text != "":
+                xx[4] = np.array([text], dtype='U{}'.format(len(text)))
+                xx[5] = np.array(['c'], dtype='<U1')
+            gt.append(xx)
+        return gt
+
+    def detection_filtering(detections, groundtruths, threshold=0.5):
+        for gt_id, gt in enumerate(groundtruths):
+            if (gt[5] == '#') and (gt[1].shape[1] > 1):
+                gt_x = list(map(int, np.squeeze(gt[1])))
+                gt_y = list(map(int, np.squeeze(gt[3])))
+                for det_id, detection in enumerate(detections):
+                    detection_orig = detection
+                    detection = [float(x) for x in detection[0].split(',')]
+                    detection = list(map(int, detection))
+                    det_x = detection[0::2]
+                    det_y = detection[1::2]
+                    det_gt_iou = iod(det_x, det_y, gt_x, gt_y)
+                    if det_gt_iou > threshold:
+                        detections[det_id] = []
+
+                detections[:] = [item for item in detections if item != []]
+        return detections
+
+    def sigma_calculation(det_x, det_y, gt_x, gt_y):
+        """
+        sigma = inter_area / gt_area
+        """
+        return np.round((area_of_intersection(det_x, det_y, gt_x, gt_y) /
+                         area(gt_x, gt_y)), 2)
+
+    def tau_calculation(det_x, det_y, gt_x, gt_y):
+        if area(det_x, det_y) == 0.0:
+            return 0
+        return np.round((area_of_intersection(det_x, det_y, gt_x, gt_y) /
+                         area(det_x, det_y)), 2)
+
+    ##############################Initialization###################################
+    # global_sigma = []
+    # global_tau = []
+    # global_pred_str = []
+    # global_gt_str = []
+    ###############################################################################
+
+    for input_id in range(allInputs):
+        if (input_id != '.DS_Store') and (input_id != 'Pascal_result.txt') and (
+                input_id != 'Pascal_result_curved.txt') and (input_id != 'Pascal_result_non_curved.txt') and (
+                input_id != 'Deteval_result.txt') and (input_id != 'Deteval_result_curved.txt') \
+                and (input_id != 'Deteval_result_non_curved.txt'):
+            detections = input_reading_mod(pred_dict)
+            groundtruths = gt_reading_mod(gt_dir)
+            detections = detection_filtering(
+                detections,
+                groundtruths)  # filters detections overlapping with DC area
+            dc_id = []
+            for i in range(len(groundtruths)):
+                if groundtruths[i][5] == '#':
+                    dc_id.append(i)
+            cnt = 0
+            for a in dc_id:
+                num = a - cnt
+                del groundtruths[num]
+                cnt += 1
+
+            local_sigma_table = np.zeros((len(groundtruths), len(detections)))
+            local_tau_table = np.zeros((len(groundtruths), len(detections)))
+            local_pred_str = {}
+            local_gt_str = {}
+
+            for gt_id, gt in enumerate(groundtruths):
+                if len(detections) > 0:
+                    for det_id, detection in enumerate(detections):
+                        detection_orig = detection
+                        detection = [float(x) for x in detection[0].split(',')]
+                        detection = list(map(int, detection))
+                        pred_seq_str = detection_orig[1].strip()
+                        det_x = detection[0::2]
+                        det_y = detection[1::2]
+                        gt_x = list(map(int, np.squeeze(gt[1])))
+                        gt_y = list(map(int, np.squeeze(gt[3])))
+                        gt_seq_str = str(gt[4].tolist()[0])
+
+                        local_sigma_table[gt_id, det_id] = sigma_calculation(
+                            det_x, det_y, gt_x, gt_y)
+                        local_tau_table[gt_id, det_id] = tau_calculation(
+                            det_x, det_y, gt_x, gt_y)
+                        local_pred_str[det_id] = pred_seq_str
+                        local_gt_str[gt_id] = gt_seq_str
+
+            global_sigma = local_sigma_table
+            global_tau = local_tau_table
+            global_pred_str = local_pred_str
+            global_gt_str = local_gt_str
+
+    single_data = {}
+    single_data['sigma'] = global_sigma
+    single_data['global_tau'] = global_tau
+    single_data['global_pred_str'] = global_pred_str
+    single_data['global_gt_str'] = global_gt_str
+    return single_data
+
+
+def get_socre_B(gt_dir, img_id, pred_dict):
+    allInputs = 1
+
+    def input_reading_mod(pred_dict):
+        """This helper reads input from txt files"""
+        det = []
+        n = len(pred_dict)
+        for i in range(n):
+            points = pred_dict[i]['points']
+            text = pred_dict[i]['texts']
+            point = ",".join(map(str, points.reshape(-1, )))
+            det.append([point, text])
+        return det
+
+    def gt_reading_mod(gt_dir, gt_id):
+        gt = io.loadmat('%s/poly_gt_img%s.mat' % (gt_dir, gt_id))
+        gt = gt['polygt']
+        return gt
+
+    def detection_filtering(detections, groundtruths, threshold=0.5):
+        for gt_id, gt in enumerate(groundtruths):
+            if (gt[5] == '#') and (gt[1].shape[1] > 1):
+                gt_x = list(map(int, np.squeeze(gt[1])))
+                gt_y = list(map(int, np.squeeze(gt[3])))
+                for det_id, detection in enumerate(detections):
+                    detection_orig = detection
+                    detection = [float(x) for x in detection[0].split(',')]
+                    detection = list(map(int, detection))
+                    det_x = detection[0::2]
+                    det_y = detection[1::2]
+                    det_gt_iou = iod(det_x, det_y, gt_x, gt_y)
+                    if det_gt_iou > threshold:
+                        detections[det_id] = []
+
+                detections[:] = [item for item in detections if item != []]
+        return detections
+
+    def sigma_calculation(det_x, det_y, gt_x, gt_y):
+        """
+        sigma = inter_area / gt_area
+        """
+        return np.round((area_of_intersection(det_x, det_y, gt_x, gt_y) /
+                         area(gt_x, gt_y)), 2)
+
+    def tau_calculation(det_x, det_y, gt_x, gt_y):
+        if area(det_x, det_y) == 0.0:
+            return 0
+        return np.round((area_of_intersection(det_x, det_y, gt_x, gt_y) /
+                         area(det_x, det_y)), 2)
+
+    ##############################Initialization###################################
+    # global_sigma = []
+    # global_tau = []
+    # global_pred_str = []
+    # global_gt_str = []
+    ###############################################################################
+
+    for input_id in range(allInputs):
+        if (input_id != '.DS_Store') and (input_id != 'Pascal_result.txt') and (
+                input_id != 'Pascal_result_curved.txt') and (input_id != 'Pascal_result_non_curved.txt') and (
+                input_id != 'Deteval_result.txt') and (input_id != 'Deteval_result_curved.txt') \
+                and (input_id != 'Deteval_result_non_curved.txt'):
+            detections = input_reading_mod(pred_dict)
+            groundtruths = gt_reading_mod(gt_dir, img_id).tolist()
+            detections = detection_filtering(
+                detections,
+                groundtruths)  # filters detections overlapping with DC area
+            dc_id = []
+            for i in range(len(groundtruths)):
+                if groundtruths[i][5] == '#':
+                    dc_id.append(i)
+            cnt = 0
+            for a in dc_id:
+                num = a - cnt
+                del groundtruths[num]
+                cnt += 1
+
+            local_sigma_table = np.zeros((len(groundtruths), len(detections)))
+            local_tau_table = np.zeros((len(groundtruths), len(detections)))
+            local_pred_str = {}
+            local_gt_str = {}
+
+            for gt_id, gt in enumerate(groundtruths):
+                if len(detections) > 0:
+                    for det_id, detection in enumerate(detections):
+                        detection_orig = detection
+                        detection = [float(x) for x in detection[0].split(',')]
+                        detection = list(map(int, detection))
+                        pred_seq_str = detection_orig[1].strip()
+                        det_x = detection[0::2]
+                        det_y = detection[1::2]
+                        gt_x = list(map(int, np.squeeze(gt[1])))
+                        gt_y = list(map(int, np.squeeze(gt[3])))
+                        gt_seq_str = str(gt[4].tolist()[0])
+
+                        local_sigma_table[gt_id, det_id] = sigma_calculation(
+                            det_x, det_y, gt_x, gt_y)
+                        local_tau_table[gt_id, det_id] = tau_calculation(
+                            det_x, det_y, gt_x, gt_y)
+                        local_pred_str[det_id] = pred_seq_str
+                        local_gt_str[gt_id] = gt_seq_str
+
+            global_sigma = local_sigma_table
+            global_tau = local_tau_table
+            global_pred_str = local_pred_str
+            global_gt_str = local_gt_str
+
+    single_data = {}
+    single_data['sigma'] = global_sigma
+    single_data['global_tau'] = global_tau
+    single_data['global_pred_str'] = global_pred_str
+    single_data['global_gt_str'] = global_gt_str
+    return single_data
+
+
+def combine_results(all_data):
+    tr = 0.7
+    tp = 0.6
+    fsc_k = 0.8
+    k = 2
+    global_sigma = []
+    global_tau = []
+    global_pred_str = []
+    global_gt_str = []
+    for data in all_data:
+        global_sigma.append(data['sigma'])
+        global_tau.append(data['global_tau'])
+        global_pred_str.append(data['global_pred_str'])
+        global_gt_str.append(data['global_gt_str'])
+
+    global_accumulative_recall = 0
+    global_accumulative_precision = 0
+    total_num_gt = 0
+    total_num_det = 0
+    hit_str_count = 0
+    hit_count = 0
+
+    def one_to_one(local_sigma_table, local_tau_table,
+                   local_accumulative_recall, local_accumulative_precision,
+                   global_accumulative_recall, global_accumulative_precision,
+                   gt_flag, det_flag, idy):
+        hit_str_num = 0
+        for gt_id in range(num_gt):
+            gt_matching_qualified_sigma_candidates = np.where(
+                local_sigma_table[gt_id, :] > tr)
+            gt_matching_num_qualified_sigma_candidates = gt_matching_qualified_sigma_candidates[
+                0].shape[0]
+            gt_matching_qualified_tau_candidates = np.where(
+                local_tau_table[gt_id, :] > tp)
+            gt_matching_num_qualified_tau_candidates = gt_matching_qualified_tau_candidates[
+                0].shape[0]
+
+            det_matching_qualified_sigma_candidates = np.where(
+                local_sigma_table[:, gt_matching_qualified_sigma_candidates[0]]
+                > tr)
+            det_matching_num_qualified_sigma_candidates = det_matching_qualified_sigma_candidates[
+                0].shape[0]
+            det_matching_qualified_tau_candidates = np.where(
+                local_tau_table[:, gt_matching_qualified_tau_candidates[0]] >
+                tp)
+            det_matching_num_qualified_tau_candidates = det_matching_qualified_tau_candidates[
+                0].shape[0]
+
+            if (gt_matching_num_qualified_sigma_candidates == 1) and (gt_matching_num_qualified_tau_candidates == 1) and \
+                    (det_matching_num_qualified_sigma_candidates == 1) and (
+                    det_matching_num_qualified_tau_candidates == 1):
+                global_accumulative_recall = global_accumulative_recall + 1.0
+                global_accumulative_precision = global_accumulative_precision + 1.0
+                local_accumulative_recall = local_accumulative_recall + 1.0
+                local_accumulative_precision = local_accumulative_precision + 1.0
+
+                gt_flag[0, gt_id] = 1
+                matched_det_id = np.where(local_sigma_table[gt_id, :] > tr)
+                # recg start
+                gt_str_cur = global_gt_str[idy][gt_id]
+                pred_str_cur = global_pred_str[idy][matched_det_id[0].tolist()[
+                    0]]
+                if pred_str_cur == gt_str_cur:
+                    hit_str_num += 1
+                else:
+                    if pred_str_cur.lower() == gt_str_cur.lower():
+                        hit_str_num += 1
+                # recg end
+                det_flag[0, matched_det_id] = 1
+        return local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, gt_flag, det_flag, hit_str_num
+
+    def one_to_many(local_sigma_table, local_tau_table,
+                    local_accumulative_recall, local_accumulative_precision,
+                    global_accumulative_recall, global_accumulative_precision,
+                    gt_flag, det_flag, idy):
+        hit_str_num = 0
+        for gt_id in range(num_gt):
+            # skip the following if the groundtruth was matched
+            if gt_flag[0, gt_id] > 0:
+                continue
+
+            non_zero_in_sigma = np.where(local_sigma_table[gt_id, :] > 0)
+            num_non_zero_in_sigma = non_zero_in_sigma[0].shape[0]
+
+            if num_non_zero_in_sigma >= k:
+                ####search for all detections that overlaps with this groundtruth
+                qualified_tau_candidates = np.where((local_tau_table[
+                    gt_id, :] >= tp) & (det_flag[0, :] == 0))
+                num_qualified_tau_candidates = qualified_tau_candidates[
+                    0].shape[0]
+
+                if num_qualified_tau_candidates == 1:
+                    if ((local_tau_table[gt_id, qualified_tau_candidates] >= tp)
+                            and
+                        (local_sigma_table[gt_id, qualified_tau_candidates] >=
+                         tr)):
+                        # became an one-to-one case
+                        global_accumulative_recall = global_accumulative_recall + 1.0
+                        global_accumulative_precision = global_accumulative_precision + 1.0
+                        local_accumulative_recall = local_accumulative_recall + 1.0
+                        local_accumulative_precision = local_accumulative_precision + 1.0
+
+                        gt_flag[0, gt_id] = 1
+                        det_flag[0, qualified_tau_candidates] = 1
+                        # recg start
+                        gt_str_cur = global_gt_str[idy][gt_id]
+                        pred_str_cur = global_pred_str[idy][
+                            qualified_tau_candidates[0].tolist()[0]]
+                        if pred_str_cur == gt_str_cur:
+                            hit_str_num += 1
+                        else:
+                            if pred_str_cur.lower() == gt_str_cur.lower():
+                                hit_str_num += 1
+                        # recg end
+                elif (np.sum(local_sigma_table[gt_id, qualified_tau_candidates])
+                      >= tr):
+                    gt_flag[0, gt_id] = 1
+                    det_flag[0, qualified_tau_candidates] = 1
+                    # recg start
+                    gt_str_cur = global_gt_str[idy][gt_id]
+                    pred_str_cur = global_pred_str[idy][
+                        qualified_tau_candidates[0].tolist()[0]]
+                    if pred_str_cur == gt_str_cur:
+                        hit_str_num += 1
+                    else:
+                        if pred_str_cur.lower() == gt_str_cur.lower():
+                            hit_str_num += 1
+                    # recg end
+
+                    global_accumulative_recall = global_accumulative_recall + fsc_k
+                    global_accumulative_precision = global_accumulative_precision + num_qualified_tau_candidates * fsc_k
+
+                    local_accumulative_recall = local_accumulative_recall + fsc_k
+                    local_accumulative_precision = local_accumulative_precision + num_qualified_tau_candidates * fsc_k
+
+        return local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, gt_flag, det_flag, hit_str_num
+
+    def many_to_one(local_sigma_table, local_tau_table,
+                    local_accumulative_recall, local_accumulative_precision,
+                    global_accumulative_recall, global_accumulative_precision,
+                    gt_flag, det_flag, idy):
+        hit_str_num = 0
+        for det_id in range(num_det):
+            # skip the following if the detection was matched
+            if det_flag[0, det_id] > 0:
+                continue
+
+            non_zero_in_tau = np.where(local_tau_table[:, det_id] > 0)
+            num_non_zero_in_tau = non_zero_in_tau[0].shape[0]
+
+            if num_non_zero_in_tau >= k:
+                ####search for all detections that overlaps with this groundtruth
+                qualified_sigma_candidates = np.where((
+                    local_sigma_table[:, det_id] >= tp) & (gt_flag[0, :] == 0))
+                num_qualified_sigma_candidates = qualified_sigma_candidates[
+                    0].shape[0]
+
+                if num_qualified_sigma_candidates == 1:
+                    if ((local_tau_table[qualified_sigma_candidates, det_id] >=
+                         tp) and
+                        (local_sigma_table[qualified_sigma_candidates, det_id]
+                         >= tr)):
+                        # became an one-to-one case
+                        global_accumulative_recall = global_accumulative_recall + 1.0
+                        global_accumulative_precision = global_accumulative_precision + 1.0
+                        local_accumulative_recall = local_accumulative_recall + 1.0
+                        local_accumulative_precision = local_accumulative_precision + 1.0
+
+                        gt_flag[0, qualified_sigma_candidates] = 1
+                        det_flag[0, det_id] = 1
+                        # recg start
+                        pred_str_cur = global_pred_str[idy][det_id]
+                        gt_len = len(qualified_sigma_candidates[0])
+                        for idx in range(gt_len):
+                            ele_gt_id = qualified_sigma_candidates[0].tolist()[
+                                idx]
+                            if ele_gt_id not in global_gt_str[idy]:
+                                continue
+                            gt_str_cur = global_gt_str[idy][ele_gt_id]
+                            if pred_str_cur == gt_str_cur:
+                                hit_str_num += 1
+                                break
+                            else:
+                                if pred_str_cur.lower() == gt_str_cur.lower():
+                                    hit_str_num += 1
+                                break
+                        # recg end
+                elif (np.sum(local_tau_table[qualified_sigma_candidates,
+                                             det_id]) >= tp):
+                    det_flag[0, det_id] = 1
+                    gt_flag[0, qualified_sigma_candidates] = 1
+                    # recg start
+                    pred_str_cur = global_pred_str[idy][det_id]
+                    gt_len = len(qualified_sigma_candidates[0])
+                    for idx in range(gt_len):
+                        ele_gt_id = qualified_sigma_candidates[0].tolist()[idx]
+                        if ele_gt_id not in global_gt_str[idy]:
+                            continue
+                        gt_str_cur = global_gt_str[idy][ele_gt_id]
+                        if pred_str_cur == gt_str_cur:
+                            hit_str_num += 1
+                            break
+                        else:
+                            if pred_str_cur.lower() == gt_str_cur.lower():
+                                hit_str_num += 1
+                                break
+                    # recg end
+
+                    global_accumulative_recall = global_accumulative_recall + num_qualified_sigma_candidates * fsc_k
+                    global_accumulative_precision = global_accumulative_precision + fsc_k
+
+                    local_accumulative_recall = local_accumulative_recall + num_qualified_sigma_candidates * fsc_k
+                    local_accumulative_precision = local_accumulative_precision + fsc_k
+        return local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, gt_flag, det_flag, hit_str_num
+
+    for idx in range(len(global_sigma)):
+        local_sigma_table = np.array(global_sigma[idx])
+        local_tau_table = global_tau[idx]
+
+        num_gt = local_sigma_table.shape[0]
+        num_det = local_sigma_table.shape[1]
+
+        total_num_gt = total_num_gt + num_gt
+        total_num_det = total_num_det + num_det
+
+        local_accumulative_recall = 0
+        local_accumulative_precision = 0
+        gt_flag = np.zeros((1, num_gt))
+        det_flag = np.zeros((1, num_det))
+
+        #######first check for one-to-one case##########
+        local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, \
+        gt_flag, det_flag, hit_str_num = one_to_one(local_sigma_table, local_tau_table,
+                                                    local_accumulative_recall, local_accumulative_precision,
+                                                    global_accumulative_recall, global_accumulative_precision,
+                                                    gt_flag, det_flag, idx)
+
+        hit_str_count += hit_str_num
+        #######then check for one-to-many case##########
+        local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, \
+        gt_flag, det_flag, hit_str_num = one_to_many(local_sigma_table, local_tau_table,
+                                                     local_accumulative_recall, local_accumulative_precision,
+                                                     global_accumulative_recall, global_accumulative_precision,
+                                                     gt_flag, det_flag, idx)
+        hit_str_count += hit_str_num
+        #######then check for many-to-one case##########
+        local_accumulative_recall, local_accumulative_precision, global_accumulative_recall, global_accumulative_precision, \
+        gt_flag, det_flag, hit_str_num = many_to_one(local_sigma_table, local_tau_table,
+                                                     local_accumulative_recall, local_accumulative_precision,
+                                                     global_accumulative_recall, global_accumulative_precision,
+                                                     gt_flag, det_flag, idx)
+        hit_str_count += hit_str_num
+
+    try:
+        recall = global_accumulative_recall / total_num_gt
+    except ZeroDivisionError:
+        recall = 0
+
+    try:
+        precision = global_accumulative_precision / total_num_det
+    except ZeroDivisionError:
+        precision = 0
+
+    try:
+        f_score = 2 * precision * recall / (precision + recall)
+    except ZeroDivisionError:
+        f_score = 0
+
+    try:
+        seqerr = 1 - float(hit_str_count) / global_accumulative_recall
+    except ZeroDivisionError:
+        seqerr = 1
+
+    try:
+        recall_e2e = float(hit_str_count) / total_num_gt
+    except ZeroDivisionError:
+        recall_e2e = 0
+
+    try:
+        precision_e2e = float(hit_str_count) / total_num_det
+    except ZeroDivisionError:
+        precision_e2e = 0
+
+    try:
+        f_score_e2e = 2 * precision_e2e * recall_e2e / (
+            precision_e2e + recall_e2e)
+    except ZeroDivisionError:
+        f_score_e2e = 0
+
+    final = {
+        'total_num_gt': total_num_gt,
+        'total_num_det': total_num_det,
+        'global_accumulative_recall': global_accumulative_recall,
+        'hit_str_count': hit_str_count,
+        'recall': recall,
+        'precision': precision,
+        'f_score': f_score,
+        'seqerr': seqerr,
+        'recall_e2e': recall_e2e,
+        'precision_e2e': precision_e2e,
+        'f_score_e2e': f_score_e2e
+    }
+    return final