README

CODE_OF_CONDUCT.md

 # Code of Conduct
 
-Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please read the [full text](https://code.facebook.com/pages/876921332402685/open-source-code-of-conduct) so that you can understand what actions will and will not be tolerated.
-
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
+Please read the [full text](https://code.fb.com/codeofconduct/)
+so that you can understand what actions will and will not be tolerated.

README.md

@@ -143,9 +143,10 @@ To run the examples you may also need to install unrar:
 apt-get install unrar
 ```
 
+## License
+SparseConvNet is Attribution-NonCommercial 4.0 International licensed, as found in the LICENSE file.
 
-
-### Links
+## Links
 1. [ICDAR 2013 Chinese Handwriting Recognition Competition 2013](http://www.nlpr.ia.ac.cn/events/CHRcompetition2013/competition/Home.html) First place in task 3, with test error of 2.61%. Human performance on the test set was 4.81%. [Report](http://www.nlpr.ia.ac.cn/events/CHRcompetition2013/competition/ICDAR%202013%20CHR%20competition.pdf)
 2. [Spatially-sparse convolutional neural networks, 2014](http://arxiv.org/abs/1409.6070) SparseConvNets for Chinese handwriting recognition
 3. [Fractional max-pooling, 2014](http://arxiv.org/abs/1412.6071) A SparseConvNet with fractional max-pooling achieves an error rate of 3.47% for CIFAR-10.
@@ -156,7 +157,7 @@ apt-get install unrar
 8. [Workshop on Learning to See from 3D Data, 2017](https://shapenet.cs.stanford.edu/iccv17workshop/) First place in the [semantic segmentation](https://shapenet.cs.stanford.edu/iccv17/) competition. [Report](https://arxiv.org/pdf/1710.06104)
 9. [3D Semantic Segmentation with Submanifold Sparse Convolutional Networks, 2017](https://arxiv.org/abs/1711.10275) Semantic segmentation for the ShapeNet Core55 and NYU-DepthV2 datasets, CVPR 2018
 
-### Citations
+## Citations
 
 If you find this code useful in your research then please cite:
 

build.sh

 #!/bin/bash
+# Copyright 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
 rm -rf build/ dist/ sparseconvnet.egg-info
 python setup.py install
+python examples/hello-world.py

examples/hello-world.py

@@ -69,7 +69,7 @@ input.precomputeMetadata(3)
 model.train()
 if use_cuda:
     input.cuda()
-output = model.forward(input)
+output = model(input)
 
 # Output is 2x32x10x10: our minibatch has 2 samples, the network has 32 output
 # feature planes, and 10x10 is the spatial size of the output.

sparseconvnet/SCN/cuda.cu

+// Copyright 2016-present, Facebook, Inc.
+// All rights reserved.
+//
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
 #include <ATen/ATen.h>
 #include <Metadata/Metadata.h>
 

sparseconvnet/SCN/sparseconvnet.h

+// Copyright 2016-present, Facebook, Inc.
+// All rights reserved.
+//
+// This source code is licensed under the license found in the
+// LICENSE file in the root directory of this source tree.
+
 #include "Metadata/Metadata.h"
 
 double AffineReluTrivialConvolution_updateOutput(at::Tensor input_features,

sparseconvnet/__init__.py

@@ -8,13 +8,13 @@ forward_pass_multiplyAdd_count = 0
 forward_pass_hidden_states = 0
 from .activations import Tanh, Sigmoid, ReLU, LeakyReLU, ELU, SELU, BatchNormELU
 from .averagePooling import AveragePooling
-from .batchNormalization import BatchNormalization, BatchNormReLU, BatchNormLeakyReLU
+from .batchNormalization import BatchNormalization, BatchNormReLU, BatchNormLeakyReLU, MeanOnlyBNLeakyReLU
 from .classificationTrainValidate import ClassificationTrainValidate
 from .convolution import Convolution
 from .deconvolution import Deconvolution
 from .denseToSparse import DenseToSparse
 from .dropout import Dropout, BatchwiseDropout
-from .fullConvolution import FullConvolution
+from .fullConvolution import FullConvolution, TransposeConvolution
 from .identity import Identity
 from .inputBatch import InputBatch
 from .ioLayers import InputLayer, OutputLayer, BLInputLayer, BLOutputLayer, InputLayerInput

sparseconvnet/batchNormalization.py

@@ -34,8 +34,8 @@ class BatchNormalization(Module):
         self.momentum = momentum
         self.affine = affine
         self.leakiness = leakiness
-        self.register_buffer("runningMean", torch.Tensor(nPlanes).fill_(0))
-        self.register_buffer("runningVar", torch.Tensor(nPlanes).fill_(1))
+        self.register_buffer("running_mean", torch.Tensor(nPlanes).fill_(0))
+        self.register_buffer("running_var", torch.Tensor(nPlanes).fill_(1))
         if affine:
             self.weight = Parameter(torch.Tensor(nPlanes).fill_(1))
             self.bias = Parameter(torch.Tensor(nPlanes).fill_(0))
@@ -49,8 +49,8 @@ class BatchNormalization(Module):
             input.features,
             optionalTensor(self, 'weight'),
             optionalTensor(self, 'bias'),
-            self.runningMean,
-            self.runningVar,
+            self.running_mean,
+            self.running_var,
             self.eps,
             self.momentum,
             self.training,
@@ -84,8 +84,11 @@ class BatchNormLeakyReLU(BatchNormalization):
         BatchNormalization.__init__(self, nPlanes, eps, momentum, True, 0.333)
 
     def __repr__(self):
-        s = 'BatchNormReLU(' + str(self.nPlanes) + ',eps=' + str(self.eps) + \
-            ',momentum=' + str(self.momentum) + ',affine=' + str(self.affine) + ')'
+        s = 'BatchLeakyNorm(' + str(self.nPlanes) + ',eps=' + str(self.eps) + \
+            ',momentum=' + str(self.momentum) + ',affine=' + str(self.affine)
+        if self.leakiness > 0:
+            s = s + ',leakiness=' + str(self.leakiness)
+        s = s + ')'
         return s
 
 class BatchNormalizationFunction(Function):
@@ -95,25 +98,25 @@ class BatchNormalizationFunction(Function):
             input_features,
             weight,
             bias,
-            runningMean,
-            runningVar,
+            running_mean,
+            running_var,
             eps,
             momentum,
             train,
             leakiness):
-        ctx.nPlanes = runningMean.shape[0]
+        ctx.nPlanes = running_mean.shape[0]
         ctx.train = train
         ctx.leakiness = leakiness
         output_features = input_features.new()
         saveMean = input_features.new().resize_(ctx.nPlanes)
-        saveInvStd = runningMean.clone().resize_(ctx.nPlanes)
+        saveInvStd = running_mean.clone().resize_(ctx.nPlanes)
         sparseconvnet.SCN.BatchNormalization_updateOutput(
             input_features,
             output_features,
             saveMean,
             saveInvStd,
-            runningMean,
-            runningVar,
+            running_mean,
+            running_var,
             weight,
             bias,
             eps,
@@ -124,8 +127,8 @@ class BatchNormalizationFunction(Function):
                               output_features,
                               weight,
                               bias,
-                              runningMean,
-                              runningVar,
+                              running_mean,
+                              running_var,
                               saveMean,
                               saveInvStd)
         return output_features
@@ -136,8 +139,8 @@ class BatchNormalizationFunction(Function):
             output_features,\
             weight,\
             bias,\
-            runningMean,\
-            runningVar,\
+            running_mean,\
+            running_var,\
             saveMean,\
             saveInvStd = ctx.saved_tensors
         assert ctx.train
@@ -151,11 +154,58 @@ class BatchNormalizationFunction(Function):
             grad_output.contiguous(),
             saveMean,
             saveInvStd,
-            runningMean,
-            runningVar,
+            running_mean,
+            running_var,
             weight,
             bias,
             grad_weight,
             grad_bias,
             ctx.leakiness)
         return grad_input, optionalTensorReturn(grad_weight), optionalTensorReturn(grad_bias), None, None, None, None, None, None
+
+class MeanOnlyBNLeakyReLU(Module):
+    """
+    Parameters:
+    nPlanes : number of input planes
+    momentum : for calculating running average for testing (default 0.9)
+    leakiness : Apply activation def inplace: 0<=leakiness<=1.
+    0 for ReLU, values in (0,1) for LeakyReLU, 1 for no activation def.
+    """
+    def __init__(
+            self,
+            nPlanes,
+            affine=True,
+            leakiness=1,
+            momentum=0.9):
+        Module.__init__(self)
+        self.nPlanes = nPlanes
+        self.momentum = momentum
+        self.register_buffer("running_mean", torch.Tensor(nPlanes).fill_(0))
+        self.affine = affine
+        if affine:
+            self.bias = Parameter(torch.Tensor(nPlanes).fill_(0))
+        self.leakiness = leakiness
+    def forward(self, input):
+        assert input.features.nelement() == 0 or input.features.size(1) == self.nPlanes
+        output = SparseConvNetTensor()
+        output.metadata = input.metadata
+        output.spatial_size = input.spatial_size
+        if self.training:
+            with torch.no_grad():
+                m=input.features.mean(0)
+                self.running_mean=self.running_mean*self.momentum+m*(1-self.momentum)
+            output.features = input.features - m
+        else:
+            output.features = input.features - self.running_mean
+        if self.affine:
+            output_features = output_features + self.bias
+        if self.leakiness != 1:
+            output.features = torch.nn.functional.leaky_relu(output.features, self.leakiness)
+        return output
+
+    def input_spatial_size(self, out_size):
+        return out_size
+
+    def __repr__(self):
+        s = 'MeanOnlyBatchNorm(' + str(self.nPlanes) + ',momentum=' + str(self.momentum) + ',leakiness=' + str(self.leakiness) + ')'
+        return s

sparseconvnet/fullConvolution.py

@@ -148,3 +148,5 @@ class FullConvolutionFunction(Function):
             grad_weight,
             grad_bias)
         return grad_input, grad_weight, optionalTensorReturn(grad_bias), None, None, None, None, None, None, None
+
+TransposeConvolution=FullConvolution

sparseconvnet/sequential.py

@@ -31,3 +31,16 @@ class Sequential(torch.nn.Sequential):
             else:
                 input = module(input)
         return input
+
+    def rebias(self, input):
+        for module in self._modules.values():
+            if isinstance(module, Sequential):
+                input = module.reweight(input)
+            elif hasattr(input, 'features') and hasattr(module, 'bias'):
+                f = module(input).features
+                f = f - module.bias
+                module.bias = torch.nn.Parameter(-f.mean(0))
+                input = module(input)
+            else:
+                input = module(input)
+        return input

sparseconvnet/spectral_norm.py

-import torch
-
+# Copyright 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
 import torch
 from torch.nn.functional import normalize
 from torch.nn.parameter import Parameter
 
-
 class SpectralNorm(object):
 
-    def __init__(self, name='weight', n_power_iterations=1, dim=0, eps=1e-12):
+    def __init__(self, name='weight', n_power_iterations=1, dim=0, eps=1e-12, norm=1):
         self.name = name
         self.dim = dim
+        self.norm = norm
         if n_power_iterations <= 0:
             raise ValueError('Expected n_power_iterations to be positive, but '
                              'got n_power_iterations={}'.format(n_power_iterations))
@@ -19,7 +22,7 @@ class SpectralNorm(object):
     def compute_weight(self, module):
         weight = getattr(module, self.name + '_orig')
         u = getattr(module, self.name + '_u')
-        weight_mat = weight
+        weight_mat = weight #/ self.norm
         if self.dim != 0:
             # permute dim to front
             weight_mat = weight_mat.permute(self.dim,
@@ -35,7 +38,7 @@ class SpectralNorm(object):
                 u = normalize(torch.matmul(weight_mat, v), dim=0, eps=self.eps)
 
         sigma = torch.dot(u, torch.matmul(weight_mat, v))
-        weight = weight / sigma
+        weight = weight * (self.norm / sigma)
         return weight, u
 
     def remove(self, module):
@@ -55,8 +58,8 @@ class SpectralNorm(object):
             getattr(module, self.name).detach_().requires_grad_(r_g)
 
     @staticmethod
-    def apply(module, name, n_power_iterations, dim, eps):
-        fn = SpectralNorm(name, n_power_iterations, dim, eps)
+    def apply(module, name, n_power_iterations, dim, eps, norm):
+        fn = SpectralNorm(name, n_power_iterations, dim, eps, norm)
         weight = module._parameters[name]
         height = weight.size(dim)
 
@@ -75,11 +78,10 @@ class SpectralNorm(object):
         module.register_forward_pre_hook(fn)
         return fn
 
-def spectral_norm(module, n_power_iterations=1, eps=1e-12):
+def spectral_norm(module, n_power_iterations=1, eps=1e-12, norm=1):
     """
     https://github.com/pytorch/pytorch/blob/master/torch/nn/utils/spectral_norm.py
     """
-    dim=1
-    #torch.nn.utils.
-    SpectralNorm.apply(module, 'weight', n_power_iterations, dim, eps)
+    dim=module.weight.ndimension()-1
+    SpectralNorm.apply(module, 'weight', n_power_iterations, dim, eps, norm)
     return module

sparseconvnet/utils.py

@@ -69,7 +69,7 @@ def append_tensors(tensors):
     spatial_size=tensors[0].spatial_size
     dimension=len(spatial_size)
     x=SparseConvNetTensor(
-        features=torch.cat([t.features for t in features],0),
+        features=torch.cat([t.features for t in tensors],0),
         metadata=Metadata(dimension),
         spatial_size=spatial_size)
     for t in tensors:
@@ -106,3 +106,10 @@ def compare_sparse(x, y):
     if R.numel():
         e += y.features[R].pow(2).sum()
     return e / (cL.numel() + L.numel() + R.numel())
+
+def spectral_norm_svd(module):
+    w=module.weight
+    if w.ndimension()==3:
+        w=w.view(-1,w.size(2))
+    _,s,_=torch.svd(w)
+    return s[0]