Merge branch 'main' into 'main'

yolov5增加了mpi单机多卡和多机多卡启动方式，其readme文件进行了更新，对maskrcnn的debug输出日志进行了删除，并更新了该模型的readme文件

See merge request dcutoolkit/deeplearing/dlexamples_new!46

PyTorch/Compute-Vision/Objection/yolov5/utils/__init__.py

-# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
-"""
-utils/initialization
-"""
-
-
-def notebook_init(verbose=True):
-    # Check system software and hardware
-    print('Checking setup...')
-
-    import os
-    import shutil
-
-    from utils.general import check_requirements, emojis, is_colab
-    from utils.torch_utils import select_device  # imports
-
-    check_requirements(('psutil', 'IPython'))
-    import psutil
-    from IPython import display  # to display images and clear console output
-
-    if is_colab():
-        shutil.rmtree('/content/sample_data', ignore_errors=True)  # remove colab /sample_data directory
-
-    if verbose:
-        # System info
-        # gb = 1 / 1000 ** 3  # bytes to GB
-        gib = 1 / 1024 ** 3  # bytes to GiB
-        ram = psutil.virtual_memory().total
-        total, used, free = shutil.disk_usage("/")
-        display.clear_output()
-        s = f'({os.cpu_count()} CPUs, {ram * gib:.1f} GB RAM, {(total - free) * gib:.1f}/{total * gib:.1f} GB disk)'
-    else:
-        s = ''
-
-    select_device(newline=False)
-    print(emojis(f'Setup complete ✅ {s}'))
-    return display

PyTorch/Compute-Vision/Objection/yolov5/utils/activations.py

@@ -18,8 +18,8 @@ class SiLU(nn.Module):  # export-friendly version of nn.SiLU()
 class Hardswish(nn.Module):  # export-friendly version of nn.Hardswish()
     @staticmethod
     def forward(x):
-        # return x * F.hardsigmoid(x)  # for TorchScript and CoreML
-        return x * F.hardtanh(x + 3, 0.0, 6.0) / 6.0  # for TorchScript, CoreML and ONNX
+        # return x * F.hardsigmoid(x)  # for torchscript and CoreML
+        return x * F.hardtanh(x + 3, 0., 6.) / 6.  # for torchscript, CoreML and ONNX
 
 
 # Mish https://github.com/digantamisra98/Mish --------------------------------------------------------------------------

PyTorch/Compute-Vision/Objection/yolov5/utils/augmentations.py

@@ -3,13 +3,14 @@
 Image augmentation functions
 """
 
+import logging
 import math
 import random
 
 import cv2
 import numpy as np
 
-from utils.general import LOGGER, check_version, colorstr, resample_segments, segment2box
+from utils.general import colorstr, segment2box, resample_segments, check_version
 from utils.metrics import bbox_ioa
 
 
@@ -19,7 +20,7 @@ class Albumentations:
         self.transform = None
         try:
             import albumentations as A
-            check_version(A.__version__, '1.0.3', hard=True)  # version requirement
+            check_version(A.__version__, '1.0.3')  # version requirement
 
             self.transform = A.Compose([
                 A.Blur(p=0.01),
@@ -31,11 +32,11 @@ class Albumentations:
                 A.ImageCompression(quality_lower=75, p=0.0)],
                 bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels']))
 
-            LOGGER.info(colorstr('albumentations: ') + ', '.join(f'{x}' for x in self.transform.transforms if x.p))
+            logging.info(colorstr('albumentations: ') + ', '.join(f'{x}' for x in self.transform.transforms if x.p))
         except ImportError:  # package not installed, skip
             pass
         except Exception as e:
-            LOGGER.info(colorstr('albumentations: ') + f'{e}')
+            logging.info(colorstr('albumentations: ') + f'{e}')
 
     def __call__(self, im, labels, p=1.0):
         if self.transform and random.random() < p:
@@ -123,7 +124,7 @@ def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleF
 
 def random_perspective(im, targets=(), segments=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0,
                        border=(0, 0)):
-    # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10))
+    # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10))
     # targets = [cls, xyxy]
 
     height = im.shape[0] + border[0] * 2  # shape(h,w,c)
@@ -269,7 +270,7 @@ def mixup(im, labels, im2, labels2):
     return im, labels
 
 
-def box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16):  # box1(4,n), box2(4,n)
+def box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1, eps=1e-16):  # box1(4,n), box2(4,n)
     # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio
     w1, h1 = box1[2] - box1[0], box1[3] - box1[1]
     w2, h2 = box2[2] - box2[0], box2[3] - box2[1]

PyTorch/Compute-Vision/Objection/yolov5/utils/autoanchor.py

@@ -10,9 +10,7 @@ import torch
 import yaml
 from tqdm import tqdm
 
-from utils.general import LOGGER, colorstr, emojis
-
-PREFIX = colorstr('AutoAnchor: ')
+from utils.general import colorstr
 
 
 def check_anchor_order(m):
@@ -21,12 +19,14 @@ def check_anchor_order(m):
     da = a[-1] - a[0]  # delta a
     ds = m.stride[-1] - m.stride[0]  # delta s
     if da.sign() != ds.sign():  # same order
-        LOGGER.info(f'{PREFIX}Reversing anchor order')
+        print('Reversing anchor order')
         m.anchors[:] = m.anchors.flip(0)
 
 
 def check_anchors(dataset, model, thr=4.0, imgsz=640):
     # Check anchor fit to data, recompute if necessary
+    prefix = colorstr('autoanchor: ')
+    print(f'\n{prefix}Analyzing anchors... ', end='')
     m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1]  # Detect()
     shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True)
     scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1))  # augment scale
@@ -34,32 +34,31 @@ def check_anchors(dataset, model, thr=4.0, imgsz=640):
 
     def metric(k):  # compute metric
         r = wh[:, None] / k[None]
-        x = torch.min(r, 1 / r).min(2)[0]  # ratio metric
+        x = torch.min(r, 1. / r).min(2)[0]  # ratio metric
         best = x.max(1)[0]  # best_x
-        aat = (x > 1 / thr).float().sum(1).mean()  # anchors above threshold
-        bpr = (best > 1 / thr).float().mean()  # best possible recall
+        aat = (x > 1. / thr).float().sum(1).mean()  # anchors above threshold
+        bpr = (best > 1. / thr).float().mean()  # best possible recall
         return bpr, aat
 
     anchors = m.anchors.clone() * m.stride.to(m.anchors.device).view(-1, 1, 1)  # current anchors
     bpr, aat = metric(anchors.cpu().view(-1, 2))
-    s = f'\n{PREFIX}{aat:.2f} anchors/target, {bpr:.3f} Best Possible Recall (BPR). '
-    if bpr > 0.98:  # threshold to recompute
-        LOGGER.info(emojis(f'{s}Current anchors are a good fit to dataset ✅'))
-    else:
-        LOGGER.info(emojis(f'{s}Anchors are a poor fit to dataset ⚠️, attempting to improve...'))
+    print(f'anchors/target = {aat:.2f}, Best Possible Recall (BPR) = {bpr:.4f}', end='')
+    if bpr < 0.98:  # threshold to recompute
+        print('. Attempting to improve anchors, please wait...')
         na = m.anchors.numel() // 2  # number of anchors
         try:
             anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False)
         except Exception as e:
-            LOGGER.info(f'{PREFIX}ERROR: {e}')
+            print(f'{prefix}ERROR: {e}')
         new_bpr = metric(anchors)[0]
         if new_bpr > bpr:  # replace anchors
             anchors = torch.tensor(anchors, device=m.anchors.device).type_as(m.anchors)
             m.anchors[:] = anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1)  # loss
             check_anchor_order(m)
-            LOGGER.info(f'{PREFIX}New anchors saved to model. Update model *.yaml to use these anchors in the future.')
+            print(f'{prefix}New anchors saved to model. Update model *.yaml to use these anchors in the future.')
         else:
-            LOGGER.info(f'{PREFIX}Original anchors better than new anchors. Proceeding with original anchors.')
+            print(f'{prefix}Original anchors better than new anchors. Proceeding with original anchors.')
+    print('')  # newline
 
 
 def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True):
@@ -81,11 +80,12 @@ def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen
     """
     from scipy.cluster.vq import kmeans
 
-    thr = 1 / thr
+    thr = 1. / thr
+    prefix = colorstr('autoanchor: ')
 
     def metric(k, wh):  # compute metrics
         r = wh[:, None] / k[None]
-        x = torch.min(r, 1 / r).min(2)[0]  # ratio metric
+        x = torch.min(r, 1. / r).min(2)[0]  # ratio metric
         # x = wh_iou(wh, torch.tensor(k))  # iou metric
         return x, x.max(1)[0]  # x, best_x
 
@@ -93,17 +93,15 @@ def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen
         _, best = metric(torch.tensor(k, dtype=torch.float32), wh)
         return (best * (best > thr).float()).mean()  # fitness
 
-    def print_results(k, verbose=True):
+    def print_results(k):
         k = k[np.argsort(k.prod(1))]  # sort small to large
         x, best = metric(k, wh0)
         bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n  # best possible recall, anch > thr
-        s = f'{PREFIX}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr\n' \
-            f'{PREFIX}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, ' \
-            f'past_thr={x[x > thr].mean():.3f}-mean: '
+        print(f'{prefix}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr')
+        print(f'{prefix}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, '
+              f'past_thr={x[x > thr].mean():.3f}-mean: ', end='')
         for i, x in enumerate(k):
-            s += '%i,%i, ' % (round(x[0]), round(x[1]))
-        if verbose:
-            LOGGER.info(s[:-2])
+            print('%i,%i' % (round(x[0]), round(x[1])), end=',  ' if i < len(k) - 1 else '\n')  # use in *.cfg
         return k
 
     if isinstance(dataset, str):  # *.yaml file
@@ -119,19 +117,19 @@ def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen
     # Filter
     i = (wh0 < 3.0).any(1).sum()
     if i:
-        LOGGER.info(f'{PREFIX}WARNING: Extremely small objects found. {i} of {len(wh0)} labels are < 3 pixels in size.')
+        print(f'{prefix}WARNING: Extremely small objects found. {i} of {len(wh0)} labels are < 3 pixels in size.')
     wh = wh0[(wh0 >= 2.0).any(1)]  # filter > 2 pixels
     # wh = wh * (np.random.rand(wh.shape[0], 1) * 0.9 + 0.1)  # multiply by random scale 0-1
 
     # Kmeans calculation
-    LOGGER.info(f'{PREFIX}Running kmeans for {n} anchors on {len(wh)} points...')
+    print(f'{prefix}Running kmeans for {n} anchors on {len(wh)} points...')
     s = wh.std(0)  # sigmas for whitening
     k, dist = kmeans(wh / s, n, iter=30)  # points, mean distance
-    assert len(k) == n, f'{PREFIX}ERROR: scipy.cluster.vq.kmeans requested {n} points but returned only {len(k)}'
+    assert len(k) == n, f'{prefix}ERROR: scipy.cluster.vq.kmeans requested {n} points but returned only {len(k)}'
     k *= s
     wh = torch.tensor(wh, dtype=torch.float32)  # filtered
     wh0 = torch.tensor(wh0, dtype=torch.float32)  # unfiltered
-    k = print_results(k, verbose=False)
+    k = print_results(k)
 
     # Plot
     # k, d = [None] * 20, [None] * 20
@@ -148,7 +146,7 @@ def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen
     # Evolve
     npr = np.random
     f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1  # fitness, generations, mutation prob, sigma
-    pbar = tqdm(range(gen), desc=f'{PREFIX}Evolving anchors with Genetic Algorithm:')  # progress bar
+    pbar = tqdm(range(gen), desc=f'{prefix}Evolving anchors with Genetic Algorithm:')  # progress bar
     for _ in pbar:
         v = np.ones(sh)
         while (v == 1).all():  # mutate until a change occurs (prevent duplicates)
@@ -157,8 +155,8 @@ def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen
         fg = anchor_fitness(kg)
         if fg > f:
             f, k = fg, kg.copy()
-            pbar.desc = f'{PREFIX}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}'
+            pbar.desc = f'{prefix}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}'
             if verbose:
-                print_results(k, verbose)
+                print_results(k)
 
     return print_results(k)

PyTorch/Compute-Vision/Objection/yolov5/utils/autobatch.py

-# YOLOv5 🚀 by Ultralytics, GPL-3.0 license
-"""
-Auto-batch utils
-"""
-
-from copy import deepcopy
-
-import numpy as np
-import torch
-from torch.cuda import amp
-
-from utils.general import LOGGER, colorstr
-from utils.torch_utils import profile
-
-
-def check_train_batch_size(model, imgsz=640):
-    # Check YOLOv5 training batch size
-    with amp.autocast():
-        return autobatch(deepcopy(model).train(), imgsz)  # compute optimal batch size
-
-
-def autobatch(model, imgsz=640, fraction=0.9, batch_size=16):
-    # Automatically estimate best batch size to use `fraction` of available CUDA memory
-    # Usage:
-    #     import torch
-    #     from utils.autobatch import autobatch
-    #     model = torch.hub.load('ultralytics/yolov5', 'yolov5s', autoshape=False)
-    #     print(autobatch(model))
-
-    prefix = colorstr('AutoBatch: ')
-    LOGGER.info(f'{prefix}Computing optimal batch size for --imgsz {imgsz}')
-    device = next(model.parameters()).device  # get model device
-    if device.type == 'cpu':
-        LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}')
-        return batch_size
-
-    d = str(device).upper()  # 'CUDA:0'
-    properties = torch.cuda.get_device_properties(device)  # device properties
-    t = properties.total_memory / 1024 ** 3  # (GiB)
-    r = torch.cuda.memory_reserved(device) / 1024 ** 3  # (GiB)
-    a = torch.cuda.memory_allocated(device) / 1024 ** 3  # (GiB)
-    f = t - (r + a)  # free inside reserved
-    LOGGER.info(f'{prefix}{d} ({properties.name}) {t:.2f}G total, {r:.2f}G reserved, {a:.2f}G allocated, {f:.2f}G free')
-
-    batch_sizes = [1, 2, 4, 8, 16]
-    try:
-        img = [torch.zeros(b, 3, imgsz, imgsz) for b in batch_sizes]
-        y = profile(img, model, n=3, device=device)
-    except Exception as e:
-        LOGGER.warning(f'{prefix}{e}')
-
-    y = [x[2] for x in y if x]  # memory [2]
-    batch_sizes = batch_sizes[:len(y)]
-    p = np.polyfit(batch_sizes, y, deg=1)  # first degree polynomial fit
-    b = int((f * fraction - p[1]) / p[0])  # y intercept (optimal batch size)
-    LOGGER.info(f'{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%)')
-    return b

PyTorch/Compute-Vision/Objection/yolov5/utils/google_app_engine/additional_requirements.txt

 # add these requirements in your app on top of the existing ones
-pip==21.1
+pip==19.2
 Flask==1.0.2
 gunicorn==19.9.0