Skip to content

GitLab

Unverified Commit e9b624fe authored by Minjie Wang's avatar Minjie Wang Committed by GitHub
Browse files

Merge branch 'master' into dist_part

parents 8086d1ed a88e7f7e
Hide whitespace changes
Inline Side-by-side
Showing with 2025 additions and 1019 deletions
examples/pytorch/gat/utils.py deleted 100644 → 0
View file @ 8086d1ed
import numpy as np
import torch
class EarlyStopping:
def __init__(self, patience=10):
self.patience = patience
self.counter = 0
self.best_score = None
self.early_stop = False
def step(self, acc, model):
score = acc
if self.best_score is None:
self.best_score = score
self.save_checkpoint(model)
elif score < self.best_score:
self.counter += 1
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(model)
self.counter = 0
return self.early_stop
def save_checkpoint(self, model):
'''Saves model when validation loss decrease.'''
torch.save(model.state_dict(), 'es_checkpoint.pt')
examples/pytorch/gcn/README.md
View file @ e9b624fe
......@@ -2,33 +2,21 @@ Graph Convolutional Networks (GCN)
============
- Paper link: [https://arxiv.org/abs/1609.02907](https://arxiv.org/abs/1609.02907)
- Author's code repo: [https://github.com/tkipf/gcn](https://github.com/tkipf/gcn). Note that the original code is
implemented with Tensorflow for the paper.
- Author's code repo: [https://github.com/tkipf/gcn](https://github.com/tkipf/gcn).
Dependencies
------------
- PyTorch 0.4.1+
- requests
``bash
pip install torch requests
``
Codes
-----
The folder contains three implementations of GCN:
- `gcn.py` uses DGL's predefined graph convolution module.
- `gcn_mp.py` uses user-defined message and reduce functions.
Modify `train.py` to switch between different implementations.
Results
How to run
-------
Run with following (available dataset: "cora", "citeseer", "pubmed")
### DGL built-in GraphConv module
Run with the following (available dataset: "cora", "citeseer", "pubmed")
```bash
python3 train.py --dataset cora --gpu 0 --self-loop
python3 train.py --dataset cora
```
* cora: ~0.810 (0.79-0.83) (paper: 0.815)
* citeseer: 0.707 (paper: 0.703)
* pubmed: 0.792 (paper: 0.790)
Summary
-------
* cora: ~0.810 (paper: 0.815)
* citeseer: ~0.707 (paper: 0.703)
* pubmed: ~0.792 (paper: 0.790)
examples/pytorch/gcn/gcn.py deleted 100644 → 0
View file @ 8086d1ed
"""GCN using DGL nn package
References:
- Semi-Supervised Classification with Graph Convolutional Networks
- Paper: https://arxiv.org/abs/1609.02907
- Code: https://github.com/tkipf/gcn
"""
import torch
import torch.nn as nn
from dgl.nn.pytorch import GraphConv
class GCN(nn.Module):
def __init__(self,
g,
in_feats,
n_hidden,
n_classes,
n_layers,
activation,
dropout):
super(GCN, self).__init__()
self.g = g
self.layers = nn.ModuleList()
# input layer
self.layers.append(GraphConv(in_feats, n_hidden, activation=activation))
# hidden layers
for i in range(n_layers - 1):
self.layers.append(GraphConv(n_hidden, n_hidden, activation=activation))
# output layer
self.layers.append(GraphConv(n_hidden, n_classes))
self.dropout = nn.Dropout(p=dropout)
def forward(self, features):
h = features
for i, layer in enumerate(self.layers):
if i != 0:
h = self.dropout(h)
h = layer(self.g, h)
return h
examples/pytorch/gcn/gcn_mp.py deleted 100644 → 0
View file @ 8086d1ed
"""GCN using basic message passing
References:
- Semi-Supervised Classification with Graph Convolutional Networks
- Paper: https://arxiv.org/abs/1609.02907
- Code: https://github.com/tkipf/gcn
"""
import argparse, time, math
import numpy as np
import networkx as nx
import torch
import torch.nn as nn
import torch.nn.functional as F
import dgl
from dgl.data import register_data_args
from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset
def gcn_msg(edge):
msg = edge.src['h'] * edge.src['norm']
return {'m': msg}
def gcn_reduce(node):
accum = torch.sum(node.mailbox['m'], 1) * node.data['norm']
return {'h': accum}
class NodeApplyModule(nn.Module):
def __init__(self, out_feats, activation=None, bias=True):
super(NodeApplyModule, self).__init__()
if bias:
self.bias = nn.Parameter(torch.Tensor(out_feats))
else:
self.bias = None
self.activation = activation
self.reset_parameters()
def reset_parameters(self):
if self.bias is not None:
stdv = 1. / math.sqrt(self.bias.size(0))
self.bias.data.uniform_(-stdv, stdv)
def forward(self, nodes):
h = nodes.data['h']
if self.bias is not None:
h = h + self.bias
if self.activation:
h = self.activation(h)
return {'h': h}
class GCNLayer(nn.Module):
def __init__(self,
g,
in_feats,
out_feats,
activation,
dropout,
bias=True):
super(GCNLayer, self).__init__()
self.g = g
self.weight = nn.Parameter(torch.Tensor(in_feats, out_feats))
if dropout:
self.dropout = nn.Dropout(p=dropout)
else:
self.dropout = 0.
self.node_update = NodeApplyModule(out_feats, activation, bias)
self.reset_parameters()
def reset_parameters(self):
stdv = 1. / math.sqrt(self.weight.size(1))
self.weight.data.uniform_(-stdv, stdv)
def forward(self, h):
if self.dropout:
h = self.dropout(h)
self.g.ndata['h'] = torch.mm(h, self.weight)
self.g.update_all(gcn_msg, gcn_reduce, self.node_update)
h = self.g.ndata.pop('h')
return h
class GCN(nn.Module):
def __init__(self,
g,
in_feats,
n_hidden,
n_classes,
n_layers,
activation,
dropout):
super(GCN, self).__init__()
self.layers = nn.ModuleList()
# input layer
self.layers.append(GCNLayer(g, in_feats, n_hidden, activation, dropout))
# hidden layers
for i in range(n_layers - 1):
self.layers.append(GCNLayer(g, n_hidden, n_hidden, activation, dropout))
# output layer
self.layers.append(GCNLayer(g, n_hidden, n_classes, None, dropout))
def forward(self, features):
h = features
for layer in self.layers:
h = layer(h)
return h
def evaluate(model, features, labels, mask):
model.eval()
with torch.no_grad():
logits = model(features)
logits = logits[mask]
labels = labels[mask]
_, indices = torch.max(logits, dim=1)
correct = torch.sum(indices == labels)
return correct.item() * 1.0 / len(labels)
def main(args):
# load and preprocess dataset
if args.dataset == 'cora':
data = CoraGraphDataset()
elif args.dataset == 'citeseer':
data = CiteseerGraphDataset()
elif args.dataset == 'pubmed':
data = PubmedGraphDataset()
else:
raise ValueError('Unknown dataset: {}'.format(args.dataset))
g = data[0]
if args.gpu < 0:
cuda = False
else:
cuda = True
g = g.to(args.gpu)
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
in_feats = features.shape[1]
n_classes = data.num_labels
n_edges = g.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes,
train_mask.int().sum().item(),
val_mask.int().sum().item(),
test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g,
in_feats,
args.n_hidden,
n_classes,
args.n_layers,
F.relu,
args.dropout)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
acc = evaluate(model, features, labels, val_mask)
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | "
"ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(),
acc, n_edges / np.mean(dur) / 1000))
print()
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='GCN')
register_data_args(parser)
parser.add_argument("--dropout", type=float, default=0.5,
help="dropout probability")
parser.add_argument("--gpu", type=int, default=-1,
help="gpu")
parser.add_argument("--lr", type=float, default=1e-2,
help="learning rate")
parser.add_argument("--n-epochs", type=int, default=200,
help="number of training epochs")
parser.add_argument("--n-hidden", type=int, default=16,
help="number of hidden gcn units")
parser.add_argument("--n-layers", type=int, default=1,
help="number of hidden gcn layers")
parser.add_argument("--weight-decay", type=float, default=5e-4,
help="Weight for L2 loss")
args = parser.parse_args()
print(args)
main(args)
examples/pytorch/gcn/train.py
View file @ e9b624fe
import argparse
import time
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import dgl
import dgl.nn as dglnn
from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset
from dgl import AddSelfLoop
import argparse
from gcn import GCN
#from gcn_mp import GCN
#from gcn_spmv import GCN
def evaluate(model, features, labels, mask):
class GCN(nn.Module):
def __init__(self, in_size, hid_size, out_size):
super().__init__()
self.layers = nn.ModuleList()
# two-layer GCN
self.layers.append(dglnn.GraphConv(in_size, hid_size, activation=F.relu))
self.layers.append(dglnn.GraphConv(hid_size, out_size))
self.dropout = nn.Dropout(0.5)
def forward(self, g, features):
h = features
for i, layer in enumerate(self.layers):
if i != 0:
h = self.dropout(h)
h = layer(g, h)
return h
def evaluate(g, features, labels, mask, model):
model.eval()
with torch.no_grad():
logits = model(features)
logits = model(g, features)
logits = logits[mask]
labels = labels[mask]
_, indices = torch.max(logits, dim=1)
......@@ -22,124 +35,65 @@ def evaluate(model, features, labels, mask):
return correct.item() * 1.0 / len(labels)
def main(args):
def train(g, features, labels, masks, model):
# define train/val samples, loss function and optimizer
train_mask = masks[0]
val_mask = masks[1]
loss_fcn = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2, weight_decay=5e-4)
# training loop
for epoch in range(200):
model.train()
logits = model(g, features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc = evaluate(g, features, labels, val_mask, model)
print("Epoch {:05d} | Loss {:.4f} | Accuracy {:.4f} "
. format(epoch, loss.item(), acc))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--dataset", type=str, default="cora",
help="Dataset name ('cora', 'citeseer', 'pubmed').")
args = parser.parse_args()
print(f'Training with DGL built-in GraphConv module.')
# load and preprocess dataset
transform = AddSelfLoop() # by default, it will first remove self-loops to prevent duplication
if args.dataset == 'cora':
data = CoraGraphDataset()
data = CoraGraphDataset(transform=transform)
elif args.dataset == 'citeseer':
data = CiteseerGraphDataset()
data = CiteseerGraphDataset(transform=transform)
elif args.dataset == 'pubmed':
data = PubmedGraphDataset()
data = PubmedGraphDataset(transform=transform)
else:
raise ValueError('Unknown dataset: {}'.format(args.dataset))
g = data[0]
if args.gpu < 0:
cuda = False
else:
cuda = True
g = g.int().to(args.gpu)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
g = g.int().to(device)
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
in_feats = features.shape[1]
n_classes = data.num_labels
n_edges = g.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes,
train_mask.int().sum().item(),
val_mask.int().sum().item(),
test_mask.int().sum().item()))
# add self loop
if args.self_loop:
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
masks = g.ndata['train_mask'], g.ndata['val_mask'], g.ndata['test_mask']
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm = torch.pow(degs, -0.5).to(device)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(g,
in_feats,
args.n_hidden,
n_classes,
args.n_layers,
F.relu,
args.dropout)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
acc = evaluate(model, features, labels, val_mask)
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | "
"ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(),
acc, n_edges / np.mean(dur) / 1000))
print()
acc = evaluate(model, features, labels, test_mask)
print("Test accuracy {:.2%}".format(acc))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='GCN')
parser.add_argument("--dataset", type=str, default="cora",
help="Dataset name ('cora', 'citeseer', 'pubmed').")
parser.add_argument("--dropout", type=float, default=0.5,
help="dropout probability")
parser.add_argument("--gpu", type=int, default=-1,
help="gpu")
parser.add_argument("--lr", type=float, default=1e-2,
help="learning rate")
parser.add_argument("--n-epochs", type=int, default=200,
help="number of training epochs")
parser.add_argument("--n-hidden", type=int, default=16,
help="number of hidden gcn units")
parser.add_argument("--n-layers", type=int, default=1,
help="number of hidden gcn layers")
parser.add_argument("--weight-decay", type=float, default=5e-4,
help="Weight for L2 loss")
parser.add_argument("--self-loop", action='store_true',
help="graph self-loop (default=False)")
parser.set_defaults(self_loop=False)
args = parser.parse_args()
print(args)
main(args)
# create GCN model
in_size = features.shape[1]
out_size = data.num_classes
model = GCN(in_size, 16, out_size).to(device)
# model training
print('Training...')
train(g, features, labels, masks, model)
# test the model
print('Testing...')
acc = evaluate(g, features, labels, masks[2], model)
print("Test accuracy {:.4f}".format(acc))
examples/pytorch/gin/README.md
View file @ e9b624fe
......@@ -6,78 +6,28 @@ Graph Isomorphism Network (GIN)
Dependencies
------------
- PyTorch 1.1.0+
- sklearn
- tqdm
``bash
pip install torch sklearn tqdm
``
How to run
----------
An experiment on the GIN in default settings can be run with
```bash
python main.py
```
An experiment on the GIN in customized settings can be run with
Install as follows:
```bash
python main.py [--device 0 | --disable-cuda] --dataset COLLAB \
--graph_pooling_type max --neighbor_pooling_type sum
pip install sklearn
```
add `--degree_as_nlabel` to use one-hot encodings of node degrees as node feature vectors
Results
How to run
-------
results may **fluctuate**, due to random factors and the relatively small data set. if you want to follow the paper's setting, consider the script below.
Run with the following for bioinformatics graph classification (available datasets: MUTAG (default), PTC, NCI1, and PROTEINS)
```bash
# 4 bioinformatics datasets setting graph_pooling_type=sum, the nodes have categorical input features
python main.py --dataset MUTAG --device 0 \
--graph_pooling_type sum --neighbor_pooling_type sum --filename MUTAG.txt
python main.py --dataset PTC --device 0 \
--graph_pooling_type sum --neighbor_pooling_type sum --filename PTC.txt
python main.py --dataset NCI1 --device 0 \
--graph_pooling_type sum --neighbor_pooling_type sum --filename NCI1.txt
python main.py --dataset PROTEINS --device 0 \
--graph_pooling_type sum --neighbor_pooling_type sum --filename PROTEINS.txt
# 5 social network datasets setting graph_pooling_type=mean, for the REDDIT datasets, we set all node feature vectors to be the same
# (thus, features here are uninformative); for the other social networks, we use one-hot encodings of node degrees.
python main.py --dataset COLLAB --device 0 \
--graph_pooling_type mean --neighbor_pooling_type sum --degree_as_nlabel --filename COLLAB.txt
python main.py --dataset IMDBBINARY --device 0 \
--graph_pooling_type mean --neighbor_pooling_type sum --degree_as_nlabel --filename IMDBBINARY.txt
python main.py --dataset IMDBMULTI --device 0 \
--graph_pooling_type mean --neighbor_pooling_type sum --degree_as_nlabel --filename IMDBMULTI.txt
python main.py --dataset REDDITBINARY --device 0 \
--graph_pooling_type mean --neighbor_pooling_type sum --filename REDDITBINARY.txt --fold_idx 6 --epoch 120
python main.py --dataset REDDITMULTI5K --device 0 \
--graph_pooling_type mean --neighbor_pooling_type sum --filename REDDITMULTI5K.txt
python3 train.py --dataset MUTAG
```
one fold of 10 result are below.
| dataset | our result | paper report |
| ------------- | ---------- | ------------ |
| MUTAG | 89.4 | 89.4 ± 5.6 |
| PTC | 68.5 | 64.6 ± 7.0 |
| NCI1 | 78.5 | 82.7 ± 1.7 |
| PROTEINS | 72.3 | 76.2 ± 2.8 |
| COLLAB | 81.6 | 80.2 ± 1.9 |
| IMDBBINARY | 73.0 | 75.1 ± 5.1 |
| IMDBMULTI | 54.0 | 52.3 ± 2.8 |
| REDDITBINARY | 88.0 | 92.4 ± 2.5 |
| REDDITMULTI5K | 54.8 | 57.5 ± 1.5 |
> **_NOTE:_** Users may observe results fluctuate due to the randomness with relatively small dataset. In consistence with the original [paper](https://arxiv.org/abs/1810.00826), five social network datasets, 'COLLAB', 'IMDBBINARY' 'IMDBMULTI' 'REDDITBINARY' and 'REDDITMULTI5K', are also available as the input. Users are encouraged to update the script slightly for social network applications, for example, replacing sum readout on bioinformatics datasets with mean readout on social network datasets and using one-hot encodings of node degrees by setting "degree_as_nlabel=True" in GINDataset.
Summary (10-fold cross-validation)
-------
| Dataset | Result
| ------------- | -------
| MUTAG | ~89.4
| PTC | ~68.5
| NCI1 | ~82.9
| PROTEINS | ~74.1
examples/pytorch/gin/dataloader.py deleted 100644 → 0
View file @ 8086d1ed
"""
PyTorch compatible dataloader
"""
import math
import numpy as np
import torch
from torch.utils.data.sampler import SubsetRandomSampler
from sklearn.model_selection import StratifiedKFold
import dgl
from dgl.dataloading import GraphDataLoader
class GINDataLoader():
def __init__(self,
dataset,
batch_size,
device,
collate_fn=None,
seed=0,
shuffle=True,
split_name='fold10',
fold_idx=0,
split_ratio=0.7):
self.shuffle = shuffle
self.seed = seed
self.kwargs = {'pin_memory': True} if 'cuda' in device.type else {}
labels = [l for _, l in dataset]
if split_name == 'fold10':
train_idx, valid_idx = self._split_fold10(
labels, fold_idx, seed, shuffle)
elif split_name == 'rand':
train_idx, valid_idx = self._split_rand(
labels, split_ratio, seed, shuffle)
else:
raise NotImplementedError()
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
self.train_loader = GraphDataLoader(
dataset, sampler=train_sampler,
batch_size=batch_size, collate_fn=collate_fn, **self.kwargs)
self.valid_loader = GraphDataLoader(
dataset, sampler=valid_sampler,
batch_size=batch_size, collate_fn=collate_fn, **self.kwargs)
def train_valid_loader(self):
return self.train_loader, self.valid_loader
def _split_fold10(self, labels, fold_idx=0, seed=0, shuffle=True):
''' 10 flod '''
assert 0 <= fold_idx and fold_idx < 10, print(
"fold_idx must be from 0 to 9.")
skf = StratifiedKFold(n_splits=10, shuffle=shuffle, random_state=seed)
idx_list = []
for idx in skf.split(np.zeros(len(labels)), labels): # split(x, y)
idx_list.append(idx)
train_idx, valid_idx = idx_list[fold_idx]
print(
"train_set : test_set = %d : %d",
len(train_idx), len(valid_idx))
return train_idx, valid_idx
def _split_rand(self, labels, split_ratio=0.7, seed=0, shuffle=True):
num_entries = len(labels)
indices = list(range(num_entries))
np.random.seed(seed)
np.random.shuffle(indices)
split = int(math.floor(split_ratio * num_entries))
train_idx, valid_idx = indices[:split], indices[split:]
print(
"train_set : test_set = %d : %d",
len(train_idx), len(valid_idx))
return train_idx, valid_idx
examples/pytorch/gin/gin.py deleted 100644 → 0
View file @ 8086d1ed
"""
How Powerful are Graph Neural Networks
https://arxiv.org/abs/1810.00826
https://openreview.net/forum?id=ryGs6iA5Km
Author's implementation: https://github.com/weihua916/powerful-gnns
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from dgl.nn.pytorch.conv import GINConv
from dgl.nn.pytorch.glob import SumPooling, AvgPooling, MaxPooling
class ApplyNodeFunc(nn.Module):
"""Update the node feature hv with MLP, BN and ReLU."""
def __init__(self, mlp):
super(ApplyNodeFunc, self).__init__()
self.mlp = mlp
self.bn = nn.BatchNorm1d(self.mlp.output_dim)
def forward(self, h):
h = self.mlp(h)
h = self.bn(h)
h = F.relu(h)
return h
class MLP(nn.Module):
"""MLP with linear output"""
def __init__(self, num_layers, input_dim, hidden_dim, output_dim):
"""MLP layers construction
Paramters
---------
num_layers: int
The number of linear layers
input_dim: int
The dimensionality of input features
hidden_dim: int
The dimensionality of hidden units at ALL layers
output_dim: int
The number of classes for prediction
"""
super(MLP, self).__init__()
self.linear_or_not = True # default is linear model
self.num_layers = num_layers
self.output_dim = output_dim
if num_layers < 1:
raise ValueError("number of layers should be positive!")
elif num_layers == 1:
# Linear model
self.linear = nn.Linear(input_dim, output_dim)
else:
# Multi-layer model
self.linear_or_not = False
self.linears = torch.nn.ModuleList()
self.batch_norms = torch.nn.ModuleList()
self.linears.append(nn.Linear(input_dim, hidden_dim, bias=False))
for layer in range(num_layers - 2):
self.linears.append(nn.Linear(hidden_dim, hidden_dim, bias=False))
self.linears.append(nn.Linear(hidden_dim, output_dim, bias=False))
for layer in range(num_layers - 1):
self.batch_norms.append(nn.BatchNorm1d((hidden_dim)))
def forward(self, x):
if self.linear_or_not:
# If linear model
return self.linear(x)
else:
# If MLP
h = x
for i in range(self.num_layers - 1):
h = F.relu(self.batch_norms[i](self.linears[i](h)))
return self.linears[-1](h)
class GIN(nn.Module):
"""GIN model"""
def __init__(self, num_layers, num_mlp_layers, input_dim, hidden_dim,
output_dim, final_dropout, learn_eps, graph_pooling_type,
neighbor_pooling_type):
"""model parameters setting
Paramters
---------
num_layers: int
The number of linear layers in the neural network
num_mlp_layers: int
The number of linear layers in mlps
input_dim: int
The dimensionality of input features
hidden_dim: int
The dimensionality of hidden units at ALL layers
output_dim: int
The number of classes for prediction
final_dropout: float
dropout ratio on the final linear layer
learn_eps: boolean
If True, learn epsilon to distinguish center nodes from neighbors
If False, aggregate neighbors and center nodes altogether.
neighbor_pooling_type: str
how to aggregate neighbors (sum, mean, or max)
graph_pooling_type: str
how to aggregate entire nodes in a graph (sum, mean or max)
"""
super(GIN, self).__init__()
self.num_layers = num_layers
self.learn_eps = learn_eps
# List of MLPs
self.ginlayers = torch.nn.ModuleList()
self.batch_norms = torch.nn.ModuleList()
for layer in range(self.num_layers - 1):
if layer == 0:
mlp = MLP(num_mlp_layers, input_dim, hidden_dim, hidden_dim)
else:
mlp = MLP(num_mlp_layers, hidden_dim, hidden_dim, hidden_dim)
self.ginlayers.append(
GINConv(ApplyNodeFunc(mlp), neighbor_pooling_type, 0, self.learn_eps))
self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
# Linear function for graph poolings of output of each layer
# which maps the output of different layers into a prediction score
self.linears_prediction = torch.nn.ModuleList()
for layer in range(num_layers):
if layer == 0:
self.linears_prediction.append(
nn.Linear(input_dim, output_dim))
else:
self.linears_prediction.append(
nn.Linear(hidden_dim, output_dim))
self.drop = nn.Dropout(final_dropout)
if graph_pooling_type == 'sum':
self.pool = SumPooling()
elif graph_pooling_type == 'mean':
self.pool = AvgPooling()
elif graph_pooling_type == 'max':
self.pool = MaxPooling()
else:
raise NotImplementedError
def forward(self, g, h):
# list of hidden representation at each layer (including input)
hidden_rep = [h]
for i in range(self.num_layers - 1):
h = self.ginlayers[i](g, h)
h = self.batch_norms[i](h)
h = F.relu(h)
hidden_rep.append(h)
score_over_layer = 0
# perform pooling over all nodes in each graph in every layer
for i, h in enumerate(hidden_rep):
pooled_h = self.pool(g, h)
score_over_layer += self.drop(self.linears_prediction[i](pooled_h))
return score_over_layer
examples/pytorch/gin/ginparser.py deleted 100644 → 0
View file @ 8086d1ed
"""Parser for arguments
Put all arguments in one file and group similar arguments
"""
import argparse
class Parser():
def __init__(self, description):
'''
arguments parser
'''
self.parser = argparse.ArgumentParser(description=description)
self.args = None
self._parse()
def _parse(self):
# dataset
self.parser.add_argument(
'--dataset', type=str, default="MUTAG",
choices=['MUTAG', 'COLLAB', 'IMDBBINARY', 'IMDBMULTI', 'NCI1', 'PROTEINS', 'PTC', 'REDDITBINARY', 'REDDITMULTI5K'],
help='name of dataset (default: MUTAG)')
self.parser.add_argument(
'--batch_size', type=int, default=32,
help='batch size for training and validation (default: 32)')
self.parser.add_argument(
'--fold_idx', type=int, default=0,
help='the index(<10) of fold in 10-fold validation.')
self.parser.add_argument(
'--filename', type=str, default="",
help='output file')
self.parser.add_argument(
'--degree_as_nlabel', action="store_true",
help='use one-hot encodings of node degrees as node feature vectors')
# device
self.parser.add_argument(
'--disable-cuda', action='store_true',
help='Disable CUDA')
self.parser.add_argument(
'--device', type=int, default=0,
help='which gpu device to use (default: 0)')
# net
self.parser.add_argument(
'--num_layers', type=int, default=5,
help='number of layers (default: 5)')
self.parser.add_argument(
'--num_mlp_layers', type=int, default=2,
help='number of MLP layers(default: 2). 1 means linear model.')
self.parser.add_argument(
'--hidden_dim', type=int, default=64,
help='number of hidden units (default: 64)')
# graph
self.parser.add_argument(
'--graph_pooling_type', type=str,
default="sum", choices=["sum", "mean", "max"],
help='type of graph pooling: sum, mean or max')
self.parser.add_argument(
'--neighbor_pooling_type', type=str,
default="sum", choices=["sum", "mean", "max"],
help='type of neighboring pooling: sum, mean or max')
self.parser.add_argument(
'--learn_eps', action="store_true",
help='learn the epsilon weighting')
# learning
self.parser.add_argument(
'--seed', type=int, default=0,
help='random seed (default: 0)')
self.parser.add_argument(
'--epochs', type=int, default=350,
help='number of epochs to train (default: 350)')
self.parser.add_argument(
'--lr', type=float, default=0.01,
help='learning rate (default: 0.01)')
self.parser.add_argument(
'--final_dropout', type=float, default=0.5,
help='final layer dropout (default: 0.5)')
# done
self.args = self.parser.parse_args()
examples/pytorch/gin/main.py deleted 100644 → 0
View file @ 8086d1ed
import sys
import numpy as np
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.optim as optim
from dgl.data import GINDataset
from dataloader import GINDataLoader
from ginparser import Parser
from gin import GIN
def train(args, net, trainloader, optimizer, criterion, epoch):
net.train()
running_loss = 0
total_iters = len(trainloader)
# setup the offset to avoid the overlap with mouse cursor
bar = tqdm(range(total_iters), unit='batch', position=2, file=sys.stdout)
for pos, (graphs, labels) in zip(bar, trainloader):
# batch graphs will be shipped to device in forward part of model
labels = labels.to(args.device)
graphs = graphs.to(args.device)
feat = graphs.ndata.pop('attr')
outputs = net(graphs, feat)
loss = criterion(outputs, labels)
running_loss += loss.item()
# backprop
optimizer.zero_grad()
loss.backward()
optimizer.step()
# report
bar.set_description('epoch-{}'.format(epoch))
bar.close()
# the final batch will be aligned
running_loss = running_loss / total_iters
return running_loss
def eval_net(args, net, dataloader, criterion):
net.eval()
total = 0
total_loss = 0
total_correct = 0
for data in dataloader:
graphs, labels = data
graphs = graphs.to(args.device)
labels = labels.to(args.device)
feat = graphs.ndata.pop('attr')
total += len(labels)
outputs = net(graphs, feat)
_, predicted = torch.max(outputs.data, 1)
total_correct += (predicted == labels.data).sum().item()
loss = criterion(outputs, labels)
# crossentropy(reduce=True) for default
total_loss += loss.item() * len(labels)
loss, acc = 1.0*total_loss / total, 1.0*total_correct / total
net.train()
return loss, acc
def main(args):
# set up seeds, args.seed supported
torch.manual_seed(seed=args.seed)
np.random.seed(seed=args.seed)
is_cuda = not args.disable_cuda and torch.cuda.is_available()
if is_cuda:
args.device = torch.device("cuda:" + str(args.device))
torch.cuda.manual_seed_all(seed=args.seed)
else:
args.device = torch.device("cpu")
dataset = GINDataset(args.dataset, not args.learn_eps, args.degree_as_nlabel)
trainloader, validloader = GINDataLoader(
dataset, batch_size=args.batch_size, device=args.device,
seed=args.seed, shuffle=True,
split_name='fold10', fold_idx=args.fold_idx).train_valid_loader()
# or split_name='rand', split_ratio=0.7
model = GIN(
args.num_layers, args.num_mlp_layers,
dataset.dim_nfeats, args.hidden_dim, dataset.gclasses,
args.final_dropout, args.learn_eps,
args.graph_pooling_type, args.neighbor_pooling_type).to(args.device)
criterion = nn.CrossEntropyLoss() # defaul reduce is true
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)
# it's not cost-effective to hanle the cursor and init 0
# https://stackoverflow.com/a/23121189
tbar = tqdm(range(args.epochs), unit="epoch", position=3, ncols=0, file=sys.stdout)
vbar = tqdm(range(args.epochs), unit="epoch", position=4, ncols=0, file=sys.stdout)
lrbar = tqdm(range(args.epochs), unit="epoch", position=5, ncols=0, file=sys.stdout)
for epoch, _, _ in zip(tbar, vbar, lrbar):
train(args, model, trainloader, optimizer, criterion, epoch)
scheduler.step()
train_loss, train_acc = eval_net(
args, model, trainloader, criterion)
tbar.set_description(
'train set - average loss: {:.4f}, accuracy: {:.0f}%'
.format(train_loss, 100. * train_acc))
valid_loss, valid_acc = eval_net(
args, model, validloader, criterion)
vbar.set_description(
'valid set - average loss: {:.4f}, accuracy: {:.0f}%'
.format(valid_loss, 100. * valid_acc))
if not args.filename == "":
with open(args.filename, 'a') as f:
f.write('%s %s %s %s %s' % (
args.dataset,
args.learn_eps,
args.neighbor_pooling_type,
args.graph_pooling_type,
epoch
))
f.write("\n")
f.write("%f %f %f %f" % (
train_loss,
train_acc,
valid_loss,
valid_acc
))
f.write("\n")
lrbar.set_description(
"Learning eps with learn_eps={}: {}".format(
args.learn_eps, [layer.eps.data.item() for layer in model.ginlayers]))
tbar.close()
vbar.close()
lrbar.close()
if __name__ == '__main__':
args = Parser(description='GIN').args
print('show all arguments configuration...')
print(args)
main(args)
examples/pytorch/gin/train.py 0 → 100644
View file @ e9b624fe
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data.sampler import SubsetRandomSampler
from sklearn.model_selection import StratifiedKFold
from dgl.data import GINDataset
from dgl.dataloading import GraphDataLoader
from dgl.nn.pytorch.conv import GINConv
from dgl.nn.pytorch.glob import SumPooling
import argparse
class MLP(nn.Module):
"""Construct two-layer MLP-type aggreator for GIN model"""
def __init__(self, input_dim, hidden_dim, output_dim):
super().__init__()
self.linears = nn.ModuleList()
# two-layer MLP
self.linears.append(nn.Linear(input_dim, hidden_dim, bias=False))
self.linears.append(nn.Linear(hidden_dim, output_dim, bias=False))
self.batch_norm = nn.BatchNorm1d((hidden_dim))
def forward(self, x):
h = x
h = F.relu(self.batch_norm(self.linears[0](h)))
return self.linears[1](h)
class GIN(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim):
super().__init__()
self.ginlayers = nn.ModuleList()
self.batch_norms = nn.ModuleList()
num_layers = 5
# five-layer GCN with two-layer MLP aggregator and sum-neighbor-pooling scheme
for layer in range(num_layers - 1): # excluding the input layer
if layer == 0:
mlp = MLP(input_dim, hidden_dim, hidden_dim)
else:
mlp = MLP(hidden_dim, hidden_dim, hidden_dim)
self.ginlayers.append(GINConv(mlp, learn_eps=False)) # set to True if learning epsilon
self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
# linear functions for graph sum poolings of output of each layer
self.linear_prediction = nn.ModuleList()
for layer in range(num_layers):
if layer == 0:
self.linear_prediction.append(nn.Linear(input_dim, output_dim))
else:
self.linear_prediction.append(nn.Linear(hidden_dim, output_dim))
self.drop = nn.Dropout(0.5)
self.pool = SumPooling() # change to mean readout (AvgPooling) on social network datasets
def forward(self, g, h):
# list of hidden representation at each layer (including the input layer)
hidden_rep = [h]
for i, layer in enumerate(self.ginlayers):
h = layer(g, h)
h = self.batch_norms[i](h)
h = F.relu(h)
hidden_rep.append(h)
score_over_layer = 0
# perform graph sum pooling over all nodes in each layer
for i, h in enumerate(hidden_rep):
pooled_h = self.pool(g, h)
score_over_layer += self.drop(self.linear_prediction[i](pooled_h))
return score_over_layer
def split_fold10(labels, fold_idx=0):
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)
idx_list = []
for idx in skf.split(np.zeros(len(labels)), labels):
idx_list.append(idx)
train_idx, valid_idx = idx_list[fold_idx]
return train_idx, valid_idx
def evaluate(dataloader, device, model):
model.eval()
total = 0
total_correct = 0
for batched_graph, labels in dataloader:
batched_graph = batched_graph.to(device)
labels = labels.to(device)
feat = batched_graph.ndata.pop('attr')
total += len(labels)
logits = model(batched_graph, feat)
_, predicted = torch.max(logits, 1)
total_correct += (predicted == labels).sum().item()
acc = 1.0 * total_correct / total
return acc
def train(train_loader, val_loader, device, model):
# loss function, optimizer and scheduler
loss_fcn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)
# training loop
for epoch in range(350):
model.train()
total_loss = 0
for batch, (batched_graph, labels) in enumerate(train_loader):
batched_graph = batched_graph.to(device)
labels = labels.to(device)
feat = batched_graph.ndata.pop('attr')
logits = model(batched_graph, feat)
loss = loss_fcn(logits, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_loss += loss.item()
scheduler.step()
train_acc = evaluate(train_loader, device, model)
valid_acc = evaluate(val_loader, device, model)
print("Epoch {:05d} | Loss {:.4f} | Train Acc. {:.4f} | Validation Acc. {:.4f} "
. format(epoch, total_loss / (batch + 1), train_acc, valid_acc))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, default="MUTAG",
choices=['MUTAG', 'PTC', 'NCI1', 'PROTEINS'],
help='name of dataset (default: MUTAG)')
args = parser.parse_args()
print(f'Training with DGL built-in GINConv module with a fixed epsilon = 0')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# load and split dataset
dataset = GINDataset(args.dataset, self_loop=True, degree_as_nlabel=False) # add self_loop and disable one-hot encoding for input features
labels = [l for _, l in dataset]
train_idx, val_idx = split_fold10(labels)
# create dataloader
train_loader = GraphDataLoader(dataset, sampler=SubsetRandomSampler(train_idx),
batch_size=128, pin_memory=torch.cuda.is_available())
val_loader = GraphDataLoader(dataset, sampler=SubsetRandomSampler(val_idx),
batch_size=128, pin_memory=torch.cuda.is_available())
# create GIN model
in_size = dataset.dim_nfeats
out_size = dataset.gclasses
model = GIN(in_size, 16, out_size).to(device)
# model training/validating
print('Training...')
train(train_loader, val_loader, device, model)
examples/pytorch/graphsage/advanced/train_sampling_unsupervised.py
View file @ e9b624fe
......@@ -133,10 +133,9 @@ def run(proc_id, n_gpus, args, devices, data):
# blocks.
tic_step = time.time()
for step, (input_nodes, pos_graph, neg_graph, blocks) in enumerate(dataloader):
input_nodes = input_nodes.to(nfeat.device)
batch_inputs = nfeat[input_nodes].to(device)
pos_graph = pos_graph.to(device)
neg_graph = neg_graph.to(device)
blocks = [block.int().to(device) for block in blocks]
blocks = [block.int() for block in blocks]
d_step = time.time()
# Compute loss and prediction
......
examples/pytorch/graphsage/link_pred.py
View file @ e9b624fe
......@@ -106,7 +106,7 @@ def compute_mrr(model, node_emb, src, dst, neg_dst, device, batch_size=500):
h_src = node_emb[src[start:end]][:, None, :].to(device)
h_dst = node_emb[all_dst.view(-1)].view(*all_dst.shape, -1).to(device)
pred = model.predict(h_src, h_dst).squeeze(-1)
relevance = torch.zeros(*pred.shape, dtype=torch.bool)
relevance = torch.zeros(*pred.shape, dtype=torch.bool).to(pred.device)
relevance[:, 0] = True
rr[start:end] = MF.retrieval_reciprocal_rank(pred, relevance)
return rr.mean()
......@@ -117,9 +117,9 @@ def evaluate(model, edge_split, device, num_workers):
node_emb = model.inference(graph, device, 4096, num_workers, 'cpu')
results = []
for split in ['valid', 'test']:
src = edge_split[split]['source_node'].to(device)
dst = edge_split[split]['target_node'].to(device)
neg_dst = edge_split[split]['target_node_neg'].to(device)
src = edge_split[split]['source_node'].to(node_emb.device)
dst = edge_split[split]['target_node'].to(node_emb.device)
neg_dst = edge_split[split]['target_node_neg'].to(node_emb.device)
results.append(compute_mrr(model, node_emb, src, dst, neg_dst, device))
return results
......
examples/pytorch/graphsage/multi_gpu_node_classification.py
View file @ e9b624fe
......@@ -104,7 +104,6 @@ def train(rank, world_size, graph, num_classes, split_idx):
# move ids to GPU
train_idx = train_idx.to('cuda')
valid_idx = valid_idx.to('cuda')
test_idx = test_idx.to('cuda')
# For training, each process/GPU will get a subset of the
# train_idx/valid_idx, and generate mini-batches indepednetly. This allows
......
examples/pytorch/graphsage/node_classification.py
View file @ e9b624fe
......@@ -39,7 +39,7 @@ class SAGE(nn.Module):
g, torch.arange(g.num_nodes()).to(g.device), sampler, device=device,
batch_size=batch_size, shuffle=False, drop_last=False,
num_workers=0)
buffer_device = 'cpu'
buffer_device = torch.device('cpu')
pin_memory = (buffer_device != device)
for l, layer in enumerate(self.layers):
......@@ -72,9 +72,9 @@ def evaluate(model, graph, dataloader):
def layerwise_infer(device, graph, nid, model, batch_size):
model.eval()
with torch.no_grad():
pred = model.inference(graph, device, batch_size)
pred = model.inference(graph, device, batch_size) # pred in buffer_device
pred = pred[nid]
label = graph.ndata['label'][nid]
label = graph.ndata['label'][nid].to(pred.device)
return MF.accuracy(pred, label)
def train(args, device, g, dataset, model):
......@@ -141,5 +141,5 @@ if __name__ == '__main__':
# test the model
print('Testing...')
acc = layerwise_infer(device, g, dataset.test_idx.to(device), model, batch_size=4096)
acc = layerwise_infer(device, g, dataset.test_idx, model, batch_size=4096)
print("Test Accuracy {:.4f}".format(acc.item()))
examples/pytorch/hilander/PSS/README.md 0 → 100644
View file @ e9b624fe
# PSS
Code for the ECCV '22 submission "PSS: Progressive Sample Selection for Open-World Visual Representation Learning".
## Dependencies
We use python 3.7. The CUDA version needs to be 10.2. Besides DGL==0.6.1, we depend on several packages. To install dependencies using conda:
```commandline
conda create -n pss python=3.7 # create env
conda activate pss # activate env
conda install pytorch==1.7.0 torchvision==0.8.0 cudatoolkit=10.2 -c pytorch # install pytorch 1.7 version
conda install -y cudatoolkit=10.2 faiss-gpu=1.6.5 -c pytorch # install faiss gpu version matching cuda 10.2
pip install dgl-cu102 # install dgl for cuda 10.2
pip install tqdm # install tqdm
pip install matplotlib # install matplotlib
pip install pandas # install pandas
pip install pretrainedmodels # install pretrainedmodels
pip install tensorboardX # install tensorboardX
pip install seaborn # install seaborn
pip install scikit-learn
cd ..
git clone https://github.com/yjxiong/clustering-benchmark.git # install clustering-benchmark for evaluation
cd clustering-benchmark
python setup.py install
cd ../PSS
```
## Data
We use the iNaturalist 2018 dataset.
- download link: https://www.kaggle.com/c/inaturalist-2018/data;
- annotations are in `Smooth_AP/data/Inaturalist`;
- annotation txt files for different data splits are in [S3 link]|[[Google Drive](https://drive.google.com/drive/folders/1xrWogJGef4Ex5OGjiImgA06bAnk2MDrK?usp=sharing)]|[[Baidu Netdisk](https://pan.baidu.com/s/14S0Fns29a4o7kFDlNyyPjA?pwd=uwsg)] (password:uwsg).
Download `train_val2018.tar.gz` and the data split txt files to `data/Inaturalist/` folder. Extract the `tar.gz` files.
The data folder has the following structure:
```bash
PSS
|- data
| |- Inaturalist
| |- train2018.json.tar.gz
| |- train_val2018.tar.gz
| |- val2018.json.tar.gz
| |- train_val2018
| | |- Actinopterygii
| | |- ...
| |- lin_train_set1.txt
| |- train_set1.txt
| |- uin_train_set1.txt
| |- uout_train_set1.txt
| |- in_train_set1.txt
| |- Inaturalist_test_set1.txt
|-...
```
## Training
Run `bash train.sh` to train the model.
## Test
Run `bash test.sh` to evaluate on the test set.
\ No newline at end of file
examples/pytorch/hilander/PSS/Smooth_AP/README.md 0 → 100644
View file @ e9b624fe
# Smooth_AP
Referenced from the ECCV '20 paper ["Smooth-AP: Smoothing the Path Towards Large-Scale Image Retrieval"](https://www.robots.ox.ac.uk/~vgg/research/smooth-ap/), reference code is from https://github.com/Andrew-Brown1/Smooth_AP.
![teaser](https://github.com/Andrew-Brown1/Smooth_AP/blob/master/ims/teaser.png)
examples/pytorch/hilander/PSS/Smooth_AP/src/auxiliaries.py 0 → 100644
View file @ e9b624fe
# repo originally forked from https://github.com/Confusezius/Deep-Metric-Learning-Baselines
################## LIBRARIES ##############################
import warnings
warnings.filterwarnings("ignore")
import numpy as np, os, csv, datetime, torch, faiss
from PIL import Image
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn import metrics
import pickle as pkl
from torch import nn
"""============================================================================================================="""
################### TensorBoard Settings ###################
def args2exp_name(args):
exp_name = f"{args.dataset}_{args.loss}_{args.lr}_bs{args.bs}_spc{args.samples_per_class}_embed{args.embed_dim}_arch{args.arch}_decay{args.decay}_fclr{args.fc_lr_mul}_anneal{args.sigmoid_temperature}"
return exp_name
################# ACQUIRE NUMBER OF WEIGHTS #################
def gimme_params(model):
"""
Provide number of trainable parameters (i.e. those requiring gradient computation) for input network.
Args:
model: PyTorch Network
Returns:
int, number of parameters.
"""
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
params = sum([np.prod(p.size()) for p in model_parameters])
return params
################# SAVE TRAINING PARAMETERS IN NICE STRING #################
def gimme_save_string(opt):
"""
Taking the set of parameters and convert it to easy-to-read string, which can be stored later.
Args:
opt: argparse.Namespace, contains all training-specific parameters.
Returns:
string, returns string summary of parameters.
"""
varx = vars(opt)
base_str = ''
for key in varx:
base_str += str(key)
if isinstance(varx[key],dict):
for sub_key, sub_item in varx[key].items():
base_str += '\n\t'+str(sub_key)+': '+str(sub_item)
else:
base_str += '\n\t'+str(varx[key])
base_str+='\n\n'
return base_str
def f1_score(model_generated_cluster_labels, target_labels, feature_coll, computed_centroids):
"""
NOTE: MOSTLY ADAPTED FROM https://github.com/wzzheng/HDML on Hardness-Aware Deep Metric Learning.
Args:
model_generated_cluster_labels: np.ndarray [n_samples x 1], Cluster labels computed on top of data embeddings.
target_labels: np.ndarray [n_samples x 1], ground truth labels for each data sample.
feature_coll: np.ndarray [n_samples x embed_dim], total data embedding made by network.
computed_centroids: np.ndarray [num_cluster=num_classes x embed_dim], cluster coordinates
Returns:
float, F1-score
"""
from scipy.special import comb
d = np.zeros(len(feature_coll))
for i in range(len(feature_coll)):
d[i] = np.linalg.norm(feature_coll[i,:] - computed_centroids[model_generated_cluster_labels[i],:])
labels_pred = np.zeros(len(feature_coll))
for i in np.unique(model_generated_cluster_labels):
index = np.where(model_generated_cluster_labels == i)[0]
ind = np.argmin(d[index])
cid = index[ind]
labels_pred[index] = cid
N = len(target_labels)
#Cluster n_labels
avail_labels = np.unique(target_labels)
n_labels = len(avail_labels)
#Count the number of objects in each cluster
count_cluster = np.zeros(n_labels)
for i in range(n_labels):
count_cluster[i] = len(np.where(target_labels == avail_labels[i])[0])
#Build a mapping from item_id to item index
keys = np.unique(labels_pred)
num_item = len(keys)
values = range(num_item)
item_map = dict()
for i in range(len(keys)):
item_map.update([(keys[i], values[i])])
#Count the number of objects of each item
count_item = np.zeros(num_item)
for i in range(N):
index = item_map[labels_pred[i]]
count_item[index] = count_item[index] + 1
#Compute True Positive (TP) plus False Positive (FP) count
tp_fp = 0
for k in range(n_labels):
if count_cluster[k] > 1:
tp_fp = tp_fp + comb(count_cluster[k], 2)
#Compute True Positive (TP) count
tp = 0
for k in range(n_labels):
member = np.where(target_labels == avail_labels[k])[0]
member_ids = labels_pred[member]
count = np.zeros(num_item)
for j in range(len(member)):
index = item_map[member_ids[j]]
count[index] = count[index] + 1
for i in range(num_item):
if count[i] > 1:
tp = tp + comb(count[i], 2)
#Compute False Positive (FP) count
fp = tp_fp - tp
#Compute False Negative (FN) count
count = 0
for j in range(num_item):
if count_item[j] > 1:
count = count + comb(count_item[j], 2)
fn = count - tp
# compute F measure
beta = 1
P = tp / (tp + fp)
R = tp / (tp + fn)
F1 = (beta*beta + 1) * P * R / (beta*beta * P + R)
return F1
"""============================================================================================================="""
def eval_metrics_one_dataset(model, test_dataloader, device, k_vals, opt):
"""
Compute evaluation metrics on test-dataset, e.g. NMI, F1 and Recall @ k.
Args:
model: PyTorch network, network to compute evaluation metrics for.
test_dataloader: PyTorch Dataloader, dataloader for test dataset, should have no shuffling and correct processing.
device: torch.device, Device to run inference on.
k_vals: list of int, Recall values to compute
opt: argparse.Namespace, contains all training-specific parameters.
Returns:
F1 score (float), NMI score (float), recall_at_k (list of float), data embedding (np.ndarray)
"""
torch.cuda.empty_cache()
_ = model.eval()
n_classes = len(test_dataloader.dataset.avail_classes)
with torch.no_grad():
### For all test images, extract features
target_labels, feature_coll = [],[]
final_iter = tqdm(test_dataloader, desc='Computing Evaluation Metrics...')
image_paths= [x[0] for x in test_dataloader.dataset.image_list]
for idx, inp in enumerate(final_iter):
input_img, target = inp[-1], inp[0]
target_labels.extend(target.numpy().tolist())
out = model(input_img.to(device), feature=True)
feature_coll.extend(out.cpu().detach().numpy().tolist())
#pdb.set_trace()
target_labels = np.hstack(target_labels).reshape(-1,1)
feature_coll = np.vstack(feature_coll).astype('float32')
torch.cuda.empty_cache()
### Set Faiss CPU Cluster index
cpu_cluster_index = faiss.IndexFlatL2(feature_coll.shape[-1])
kmeans = faiss.Clustering(feature_coll.shape[-1], n_classes)
kmeans.niter = 20
kmeans.min_points_per_centroid = 1
kmeans.max_points_per_centroid = 1000000000
### Train Kmeans
kmeans.train(feature_coll, cpu_cluster_index)
computed_centroids = faiss.vector_float_to_array(kmeans.centroids).reshape(n_classes, feature_coll.shape[-1])
### Assign feature points to clusters
faiss_search_index = faiss.IndexFlatL2(computed_centroids.shape[-1])
faiss_search_index.add(computed_centroids)
_, model_generated_cluster_labels = faiss_search_index.search(feature_coll, 1)
### Compute NMI
NMI = metrics.cluster.normalized_mutual_info_score(model_generated_cluster_labels.reshape(-1), target_labels.reshape(-1))
### Recover max(k_vals) nehbours to use for recall computation
faiss_search_index = faiss.IndexFlatL2(feature_coll.shape[-1])
faiss_search_index.add(feature_coll)
_, k_closest_points = faiss_search_index.search(feature_coll, int(np.max(k_vals)+1))
k_closest_classes = target_labels.reshape(-1)[k_closest_points[:,1:]]
print('computing recalls')
### Compute Recall
recall_all_k = []
for k in k_vals:
recall_at_k = np.sum([1 for target, recalled_predictions in zip(target_labels, k_closest_classes) if target in recalled_predictions[:k]])/len(target_labels)
recall_all_k.append(recall_at_k)
print('finished recalls')
print('computing F1')
### Compute F1 Score
F1 = 0
# F1 = f1_score(model_generated_cluster_labels, target_labels, feature_coll, computed_centroids)
print('finished computing f1')
return F1, NMI, recall_all_k, feature_coll
def eval_metrics_query_and_gallery_dataset(model, query_dataloader, gallery_dataloader, device, k_vals, opt):
"""
Compute evaluation metrics on test-dataset, e.g. NMI, F1 and Recall @ k.
Args:
model: PyTorch network, network to compute evaluation metrics for.
query_dataloader: PyTorch Dataloader, dataloader for query dataset, for which nearest neighbours in the gallery dataset are retrieved.
gallery_dataloader: PyTorch Dataloader, dataloader for gallery dataset, provides target samples which are to be retrieved in correspondance to the query dataset.
device: torch.device, Device to run inference on.
k_vals: list of int, Recall values to compute
opt: argparse.Namespace, contains all training-specific parameters.
Returns:
F1 score (float), NMI score (float), recall_at_ks (list of float), query data embedding (np.ndarray), gallery data embedding (np.ndarray)
"""
torch.cuda.empty_cache()
_ = model.eval()
n_classes = len(query_dataloader.dataset.avail_classes)
with torch.no_grad():
### For all query test images, extract features
query_target_labels, query_feature_coll = [],[]
query_image_paths = [x[0] for x in query_dataloader.dataset.image_list]
query_iter = tqdm(query_dataloader, desc='Extraction Query Features')
for idx,inp in enumerate(query_iter):
input_img,target = inp[-1], inp[0]
query_target_labels.extend(target.numpy().tolist())
out = model(input_img.to(device), feature=True)
query_feature_coll.extend(out.cpu().detach().numpy().tolist())
### For all gallery test images, extract features
gallery_target_labels, gallery_feature_coll = [],[]
gallery_image_paths = [x[0] for x in gallery_dataloader.dataset.image_list]
gallery_iter = tqdm(gallery_dataloader, desc='Extraction Gallery Features')
for idx,inp in enumerate(gallery_iter):
input_img,target = inp[-1], inp[0]
gallery_target_labels.extend(target.numpy().tolist())
out = model(input_img.to(device), feature=True)
gallery_feature_coll.extend(out.cpu().detach().numpy().tolist())
query_target_labels, query_feature_coll = np.hstack(query_target_labels).reshape(-1,1), np.vstack(query_feature_coll).astype('float32')
gallery_target_labels, gallery_feature_coll = np.hstack(gallery_target_labels).reshape(-1,1), np.vstack(gallery_feature_coll).astype('float32')
torch.cuda.empty_cache()
### Set CPU Cluster index
stackset = np.concatenate([query_feature_coll, gallery_feature_coll],axis=0)
stacklabels = np.concatenate([query_target_labels, gallery_target_labels],axis=0)
cpu_cluster_index = faiss.IndexFlatL2(stackset.shape[-1])
kmeans = faiss.Clustering(stackset.shape[-1], n_classes)
kmeans.niter = 20
kmeans.min_points_per_centroid = 1
kmeans.max_points_per_centroid = 1000000000
### Train Kmeans
kmeans.train(stackset, cpu_cluster_index)
computed_centroids = faiss.vector_float_to_array(kmeans.centroids).reshape(n_classes, stackset.shape[-1])
### Assign feature points to clusters
faiss_search_index = faiss.IndexFlatL2(computed_centroids.shape[-1])
faiss_search_index.add(computed_centroids)
_, model_generated_cluster_labels = faiss_search_index.search(stackset, 1)
### Compute NMI
NMI = metrics.cluster.normalized_mutual_info_score(model_generated_cluster_labels.reshape(-1), stacklabels.reshape(-1))
### Recover max(k_vals) nearest neighbours to use for recall computation
faiss_search_index = faiss.IndexFlatL2(gallery_feature_coll.shape[-1])
faiss_search_index.add(gallery_feature_coll)
_, k_closest_points = faiss_search_index.search(query_feature_coll, int(np.max(k_vals)))
k_closest_classes = gallery_target_labels.reshape(-1)[k_closest_points]
### Compute Recall
recall_all_k = []
for k in k_vals:
recall_at_k = np.sum([1 for target, recalled_predictions in zip(query_target_labels, k_closest_classes) if target in recalled_predictions[:k]])/len(query_target_labels)
recall_all_k.append(recall_at_k)
recall_str = ', '.join('@{0}: {1:.4f}'.format(k,rec) for k,rec in zip(k_vals, recall_all_k))
### Compute F1 score
F1 = f1_score(model_generated_cluster_labels, stacklabels, stackset, computed_centroids)
return F1, NMI, recall_all_k, query_feature_coll, gallery_feature_coll
"""============================================================================================================="""
####### RECOVER CLOSEST EXAMPLE IMAGES #######
def recover_closest_one_dataset(feature_matrix_all, image_paths, save_path, n_image_samples=10, n_closest=3):
"""
Provide sample recoveries.
Args:
feature_matrix_all: np.ndarray [n_samples x embed_dim], full data embedding of test samples.
image_paths: list [n_samples], list of datapaths corresponding to <feature_matrix_all>
save_path: str, where to store sample image.
n_image_samples: Number of sample recoveries.
n_closest: Number of closest recoveries to show.
Returns:
Nothing!
"""
image_paths = np.array([x[0] for x in image_paths])
sample_idxs = np.random.choice(np.arange(len(feature_matrix_all)), n_image_samples)
faiss_search_index = faiss.IndexFlatL2(feature_matrix_all.shape[-1])
faiss_search_index.add(feature_matrix_all)
_, closest_feature_idxs = faiss_search_index.search(feature_matrix_all, n_closest+1)
sample_paths = image_paths[closest_feature_idxs][sample_idxs]
f,axes = plt.subplots(n_image_samples, n_closest+1)
for i,(ax,plot_path) in enumerate(zip(axes.reshape(-1), sample_paths.reshape(-1))):
ax.imshow(np.array(Image.open(plot_path)))
ax.set_xticks([])
ax.set_yticks([])
if i%(n_closest+1):
ax.axvline(x=0, color='g', linewidth=13)
else:
ax.axvline(x=0, color='r', linewidth=13)
f.set_size_inches(10,20)
f.tight_layout()
f.savefig(save_path)
plt.close()
####### RECOVER CLOSEST EXAMPLE IMAGES #######
def recover_closest_inshop(query_feature_matrix_all, gallery_feature_matrix_all, query_image_paths, gallery_image_paths, save_path, n_image_samples=10, n_closest=3):
"""
Provide sample recoveries.
Args:
query_feature_matrix_all: np.ndarray [n_query_samples x embed_dim], full data embedding of query samples.
gallery_feature_matrix_all: np.ndarray [n_gallery_samples x embed_dim], full data embedding of gallery samples.
query_image_paths: list [n_samples], list of datapaths corresponding to <query_feature_matrix_all>
gallery_image_paths: list [n_samples], list of datapaths corresponding to <gallery_feature_matrix_all>
save_path: str, where to store sample image.
n_image_samples: Number of sample recoveries.
n_closest: Number of closest recoveries to show.
Returns:
Nothing!
"""
query_image_paths, gallery_image_paths = np.array(query_image_paths), np.array(gallery_image_paths)
sample_idxs = np.random.choice(np.arange(len(query_feature_matrix_all)), n_image_samples)
faiss_search_index = faiss.IndexFlatL2(gallery_feature_matrix_all.shape[-1])
faiss_search_index.add(gallery_feature_matrix_all)
_, closest_feature_idxs = faiss_search_index.search(query_feature_matrix_all, n_closest)
image_paths = gallery_image_paths[closest_feature_idxs]
image_paths = np.concatenate([query_image_paths.reshape(-1,1), image_paths],axis=-1)
sample_paths = image_paths[closest_feature_idxs][sample_idxs]
f,axes = plt.subplots(n_image_samples, n_closest+1)
for i,(ax,plot_path) in enumerate(zip(axes.reshape(-1), sample_paths.reshape(-1))):
ax.imshow(np.array(Image.open(plot_path)))
ax.set_xticks([])
ax.set_yticks([])
if i%(n_closest+1):
ax.axvline(x=0, color='g', linewidth=13)
else:
ax.axvline(x=0, color='r', linewidth=13)
f.set_size_inches(10,20)
f.tight_layout()
f.savefig(save_path)
plt.close()
"""============================================================================================================="""
################## SET NETWORK TRAINING CHECKPOINT #####################
def set_checkpoint(model, opt, progress_saver, savepath):
"""
Store relevant parameters (model and progress saver, as well as parameter-namespace).
Can be easily extend for other stuff.
Args:
model: PyTorch network, network whose parameters are to be saved.
opt: argparse.Namespace, includes all training-specific parameters
progress_saver: subclass of LOGGER-class, contains a running memory of all training metrics.
savepath: str, where to save checkpoint.
Returns:
Nothing!
"""
torch.save({'state_dict':model.state_dict(), 'opt':opt,
'progress':progress_saver}, savepath)
"""============================================================================================================="""
################## WRITE TO CSV FILE #####################
class CSV_Writer():
"""
Class to append newly compute training metrics to a csv file
for data logging.
Is used together with the LOGGER class.
"""
def __init__(self, save_path, columns):
"""
Args:
save_path: str, where to store the csv file
columns: list of str, name of csv columns under which the resp. metrics are stored.
Returns:
Nothing!
"""
self.save_path = save_path
self.columns = columns
with open(self.save_path, "a") as csv_file:
writer = csv.writer(csv_file, delimiter=",")
writer.writerow(self.columns)
def log(self, inputs):
"""
log one set of entries to the csv.
Args:
inputs: [list of int/str/float], values to append to the csv. Has to be of the same length as self.columns.
Returns:
Nothing!
"""
with open(self.save_path, "a") as csv_file:
writer = csv.writer(csv_file, delimiter=',')
writer.writerow(inputs)
################## GENERATE LOGGING FOLDER/FILES #######################
def set_logging(opt):
"""
Generate the folder in which everything is saved.
If opt.savename is given, folder will take on said name.
If not, a name based on the start time is provided.
If the folder already exists, it will by iterated until it can be created without
deleting existing data.
The current opt.save_path will be extended to account for the new save_folder name.
Args:
opt: argparse.Namespace, contains all training-specific parameters.
Returns:
Nothing!
"""
checkfolder = opt.save_path+'/'+str(opt.iter)
#Create start-time-based name if opt.savename is not give.
if opt.savename == '':
date = datetime.datetime.now()
checkfolder = opt.save_path+'/'+str(opt.iter)
#If folder already exists, iterate over it until is doesn't.
# counter = 1
# while os.path.exists(checkfolder):
# checkfolder = opt.save_path+'/'+opt.savename+'_'+str(counter)
# counter += 1
#Create Folder
if not os.path.exists(checkfolder):
os.makedirs(checkfolder)
opt.save_path = checkfolder
#Store training parameters as text and pickle in said folder.
with open(opt.save_path+'/Parameter_Info.txt','w') as f:
f.write(gimme_save_string(opt))
pkl.dump(opt,open(opt.save_path+"/hypa.pkl","wb"))
import pdb
class LOGGER():
"""
This class provides a collection of logging properties that are useful for training.
These include setting the save folder, in which progression of training/testing metrics is visualized,
csv log-files are stored, sample recoveries are plotted and an internal data saver.
"""
def __init__(self, opt, metrics_to_log, name='Basic', start_new=True):
"""
Args:
opt: argparse.Namespace, contains all training-specific parameters.
metrics_to_log: dict, dictionary which shows in what structure the data should be saved.
is given as the output of aux.metrics_to_examine. Example:
{'train': ['Epochs', 'Time', 'Train Loss', 'Time'],
'val': ['Epochs','Time','NMI','F1', 'Recall @ 1','Recall @ 2','Recall @ 4','Recall @ 8']}
name: Name of this logger. Will be used to distinguish logged files from other LOGGER instances.
start_new: If set to true, a new save folder will be created initially.
Returns:
Nothing!
"""
self.prop = opt
self.metrics_to_log = metrics_to_log
### Make Logging Directories
if start_new: set_logging(opt)
### Set Progress Saver Dict
self.progress_saver = self.provide_progress_saver(metrics_to_log)
### Set CSV Writters
self.csv_loggers= {mode:CSV_Writer(opt.save_path+'/log_'+mode+'_'+name+'.csv', lognames) for mode, lognames in metrics_to_log.items()}
def provide_progress_saver(self, metrics_to_log):
"""
Provide Progress Saver dictionary.
Args:
metrics_to_log: see __init__(). Describes the structure of Progress_Saver.
"""
Progress_Saver = {key:{sub_key:[] for sub_key in metrics_to_log[key]} for key in metrics_to_log.keys()}
return Progress_Saver
def log(self, main_keys, metric_keys, values):
"""
Actually log new values in csv and Progress Saver dict internally.
Args:
main_keys: Main key in which data will be stored. Normally is either 'train' for training metrics or 'val' for validation metrics.
metric_keys: Needs to follow the list length of self.progress_saver[main_key(s)]. List of metric keys that are extended with new values.
values: Needs to be a list of the same structure as metric_keys. Actual values that are appended.
"""
if not isinstance(main_keys, list): main_keys = [main_keys]
if not isinstance(metric_keys, list): metric_keys = [metric_keys]
if not isinstance(values, list): values = [values]
#Log data to progress saver dict.
for main_key in main_keys:
for value, metric_key in zip(values, metric_keys):
self.progress_saver[main_key][metric_key].append(value)
#Append data to csv.
self.csv_loggers[main_key].log(values)
def update_info_plot(self):
"""
Create a new updated version of training/metric progression plot.
Args:
None
Returns:
Nothing!
"""
t_epochs = self.progress_saver['val']['Epochs']
t_loss_list = [self.progress_saver['train']['Train Loss']]
t_legend_handles = ['Train Loss']
v_epochs = self.progress_saver['val']['Epochs']
#Because Vehicle-ID normally uses three different test sets, a distinction has to be made.
if self.prop.dataset != 'vehicle_id':
title = ' | '.join(key+': {0:3.3f}'.format(np.max(item)) for key,item in self.progress_saver['val'].items() if key not in ['Time', 'Epochs'])
self.info_plot.title = title
v_metric_list = [self.progress_saver['val'][key] for key in self.progress_saver['val'].keys() if key not in ['Time', 'Epochs']]
v_legend_handles = [key for key in self.progress_saver['val'].keys() if key not in ['Time', 'Epochs']]
self.info_plot.make_plot(t_epochs, v_epochs, t_loss_list, v_metric_list, t_legend_handles, v_legend_handles)
else:
#Iterate over all test sets.
for i in range(3):
title = ' | '.join(key+': {0:3.3f}'.format(np.max(item)) for key,item in self.progress_saver['val'].items() if key not in ['Time', 'Epochs'] and 'Set {}'.format(i) in key)
self.info_plot['Set {}'.format(i)].title = title
v_metric_list = [self.progress_saver['val'][key] for key in self.progress_saver['val'].keys() if key not in ['Time', 'Epochs'] and 'Set {}'.format(i) in key]
v_legend_handles = [key for key in self.progress_saver['val'].keys() if key not in ['Time', 'Epochs'] and 'Set {}'.format(i) in key]
self.info_plot['Set {}'.format(i)].make_plot(t_epochs, v_epochs, t_loss_list, v_metric_list, t_legend_handles, v_legend_handles, appendix='set_{}'.format(i))
def metrics_to_examine(dataset, k_vals):
"""
Please only use either of the following keys:
-> Epochs, Time, Train Loss for training
-> Epochs, Time, NMI, F1 & Recall @ k for validation
Args:
dataset: str, dataset for which a storing structure for LOGGER.progress_saver is to be made.
k_vals: list of int, Recall @ k - values.
Returns:
metric_dict: Dictionary representing the storing structure for LOGGER.progress_saver. See LOGGER.__init__() for an example.
"""
metric_dict = {'train':['Epochs','Time','Train Loss']}
if dataset=='vehicle_id':
metric_dict['val'] = ['Epochs','Time']
#Vehicle_ID uses three test sets
for i in range(3):
metric_dict['val'] += ['Set {} NMI'.format(i), 'Set {} F1'.format(i)]
for k in k_vals:
metric_dict['val'] += ['Set {} Recall @ {}'.format(i,k)]
else:
metric_dict['val'] = ['Epochs','Time','NMI', 'F1']
metric_dict['val'] += ['Recall @ {}'.format(k) for k in k_vals]
return metric_dict
def bool_flag(s):
"""
Parse boolean arguments from the command line.
"""
FALSY_STRINGS = {"off", "false", "0"}
TRUTHY_STRINGS = {"on", "true", "1"}
if s.lower() in FALSY_STRINGS:
return False
elif s.lower() in TRUTHY_STRINGS:
return True
else:
raise argparse.ArgumentTypeError("invalid value for a boolean flag")
def vis(model, test_dataloader, device, split, opt):
linsize = opt.linsize
torch.cuda.empty_cache()
if opt.dataset == "Inaturalist":
if opt.iter > 0:
with open(opt.cluster_path, 'rb') as clusterf:
path2idx, global_features, global_pred_labels, gt_labels, masks = pkl.load(clusterf)
gt_labels = gt_labels + len(np.unique(global_pred_labels))
idx2path = {v: k for k, v in path2idx.items()}
else:
with open(os.path.join(opt.source_path, "train_set1.txt")) as f:
filelines = f.readlines()
paths = [x.strip() for x in filelines]
Lin_paths = paths[:linsize]
masks = np.zeros(len(paths))
masks[:len(Lin_paths)] = 0
masks[len(Lin_paths):] = 2
_ = model.eval()
path2ids = {}
with torch.no_grad():
### For all test images, extract features
target_labels, feature_coll = [],[]
final_iter = tqdm(test_dataloader, desc='Computing Evaluation Metrics...')
image_paths = [x[0] for x in test_dataloader.dataset.image_list]
for i in range(len(image_paths)):
path2ids[image_paths[i]] = i
for idx, inp in enumerate(final_iter):
input_img, target = inp[-1], inp[0]
target_labels.extend(target.numpy().tolist())
out = model(input_img.to(device), feature=True)
feature_coll.extend(out.cpu().detach().numpy().tolist())
#pdb.set_trace()
target_labels = np.hstack(target_labels).reshape(-1)
feature_coll = np.vstack(feature_coll).astype('float32')
if (opt.dataset == "Inaturalist") and "all_train" in split:
if opt.iter > 0:
predicted_features = np.zeros_like(feature_coll)
path2ids_new = {}
target_labels_new = np.zeros_like(target_labels)
for i in range(len(idx2path.keys())):
path = idx2path[i]
idxx = path2ids[path]
path2ids_new[path] = i
predicted_features[i] = feature_coll[idxx]
target_labels_new[i] = target_labels[idxx]
path2ids = path2ids_new
feature_coll = predicted_features
target_labels = target_labels_new
gtlabels = target_labels
lastuselected = np.where(masks == 1)
masks[lastuselected] = 0
print(len(np.where(masks == 0)[0]))
else:
predicted_features = np.zeros_like(feature_coll)
path2ids_new = {}
target_labels_new = np.zeros_like(target_labels)
for i in range(len(paths)):
path = paths[i]
idxx = path2ids[opt.source_path+'/'+path]
path2ids_new[opt.source_path+'/'+path] = i
predicted_features[i] = feature_coll[idxx]
target_labels_new[i] = target_labels[idxx]
path2ids = path2ids_new
feature_coll = predicted_features
target_labels = target_labels_new
gtlabels = target_labels
if "all_train" not in split:
print("all_train not in split.")
gtlabels = target_labels
output_feature_path = os.path.join(opt.source_path,split+"_inat_features.pkl")
print("Dump features into {}.".format(output_feature_path))
with open(output_feature_path, "wb") as f:
pkl.dump([path2ids, feature_coll, target_labels, gtlabels, masks], f)
print(target_labels.max())
print("target_labels:", target_labels.shape)
print("feature_coll:", feature_coll.shape)
\ No newline at end of file
examples/pytorch/hilander/PSS/Smooth_AP/src/datasets.py 0 → 100644
View file @ e9b624fe
# repo originally forked from https://github.com/Confusezius/Deep-Metric-Learning-Baselines
################# LIBRARIES ###############################
import pickle
import warnings
from numpy.core.arrayprint import IntegerFormat
warnings.filterwarnings("ignore")
import numpy as np, pandas as pd, copy, torch, random, os
from torch.utils.data import Dataset
from PIL import Image
from torchvision import transforms
"""============================================================================"""
################ FUNCTION TO RETURN ALL DATALOADERS NECESSARY ####################
def give_dataloaders(dataset, trainset, testset, opt, cluster_path=""):
"""
Args:
dataset: string, name of dataset for which the dataloaders should be returned.
opt: argparse.Namespace, contains all training-specific parameters.
Returns:
dataloaders: dict of dataloaders for training, testing and evaluation on training.
"""
#Dataset selection
if opt.dataset=='Inaturalist':
if opt.finetune:
datasets = give_inat_datasets_finetune_1head(testset, cluster_path, opt)
else:
if opt.get_features:
datasets = give_inaturalist_datasets_for_features(opt)
else:
datasets = give_inaturalist_datasets(opt)
else:
raise Exception('No Dataset >{}< available!'.format(dataset))
#Move datasets to dataloaders.
dataloaders = {}
for key, dataset in datasets.items():
if (isinstance(dataset, TrainDatasetsmoothap) or isinstance(dataset, TrainDatasetsmoothap1Head))\
and key in ['training', 'clustering']:
dataloaders[key] = torch.utils.data.DataLoader(dataset, batch_size=opt.bs,
num_workers=opt.kernels, sampler=torch.utils.data.SequentialSampler(dataset),
pin_memory=True, drop_last=True)
else:
is_val = dataset.is_validation
if key == 'training' or key == 'clustering':
dataloaders[key] = torch.utils.data.DataLoader(dataset, batch_size=opt.bs,
num_workers=opt.kernels, shuffle=not is_val, pin_memory=True, drop_last=not is_val)
else:
dataloaders[key] = torch.utils.data.DataLoader(dataset, batch_size=opt.bs,
num_workers=6, shuffle=not is_val, pin_memory=True, drop_last=not is_val)
return dataloaders
def give_inaturalist_datasets(opt):
"""
This function generates a training, testing and evaluation dataloader for Metric Learning on the Inaturalist 2018 dataset.
For Metric Learning, training and test sets are provided by given json files. Will define a train and test split
So no random shuffling of classes.
Args:
opt: argparse.Namespace, contains all traininig-specific parameters.
Returns:
dict of PyTorch datasets for training, testing and evaluation.
"""
#Load text-files containing classes and imagepaths.
#Generate image_dicts of shape {class_idx:[list of paths to images belong to this class] ...}
train_image_dict, val_image_dict, test_image_dict = {},{},{}
with open(os.path.join(opt.source_path, opt.trainset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3],info[-2]]) not in train_image_dict:
train_image_dict['/'.join([info[-3],info[-2]])] = []
train_image_dict['/'.join([info[-3],info[-2]])].append(os.path.join(opt.source_path,entry))
with open(os.path.join(opt.source_path, opt.testset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3],info[-2]]) not in val_image_dict:
val_image_dict['/'.join([info[-3], info[-2]])] = []
val_image_dict['/'.join([info[-3], info[-2]])].append(os.path.join(opt.source_path,entry))
with open(os.path.join(opt.source_path, opt.testset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3],info[-2]]) not in test_image_dict:
test_image_dict['/'.join([info[-3],info[-2]])] = []
test_image_dict['/'.join([info[-3],info[-2]])].append(os.path.join(opt.source_path,entry))
new_train_dict = {}
class_ind_ind = 0
for cate in train_image_dict:
new_train_dict["te/%d"%class_ind_ind] = train_image_dict[cate]
class_ind_ind += 1
train_image_dict = new_train_dict
train_dataset = TrainDatasetsmoothap(train_image_dict, opt)
val_dataset = BaseTripletDataset(val_image_dict, opt, is_validation=True)
eval_dataset = BaseTripletDataset(test_image_dict, opt, is_validation=True)
#train_dataset.conversion = conversion
#val_dataset.conversion = conversion
#eval_dataset.conversion = conversion
return {'training':train_dataset, 'testing':val_dataset, 'evaluation':eval_dataset}
def give_inaturalist_datasets_for_features(opt):
"""
This function generates a training, testing and evaluation dataloader for Metric Learning on the Inaturalist 2018 dataset.
For Metric Learning, training and test sets are provided by given json files. Will define a train and test split
So no random shuffling of classes.
Args:
opt: argparse.Namespace, contains all traininig-specific parameters.
Returns:
dict of PyTorch datasets for training, testing and evaluation.
"""
# Load text-files containing classes and imagepaths.
# Generate image_dicts of shape {class_idx:[list of paths to images belong to this class] ...}
train_image_dict, test_image_dict, eval_image_dict = {}, {}, {}
if opt.iter > 0:
with open(os.path.join(opt.cluster_path), 'rb') as clusterf:
path2idx, global_features, global_pred_labels, gt_labels, masks = pickle.load(clusterf)
gt_labels = gt_labels + len(np.unique(global_pred_labels))
for path, idx in path2idx.items():
if global_pred_labels[idx] == -1:
if "te/%d" % gt_labels[idx] not in test_image_dict:
test_image_dict["te/%d" % gt_labels[idx]] = []
test_image_dict["te/%d" % gt_labels[idx]].append(path)
else:
if "te/%d" % global_pred_labels[idx] not in train_image_dict:
train_image_dict["te/%d" % global_pred_labels[idx]] = []
train_image_dict["te/%d" % global_pred_labels[idx]].append(path)
if "te/%d" % global_pred_labels[idx] not in test_image_dict:
test_image_dict["te/%d" % global_pred_labels[idx]] = []
test_image_dict["te/%d" % global_pred_labels[idx]].append(path)
else:
with open(os.path.join(opt.source_path, opt.trainset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3], info[-2]]) not in train_image_dict:
train_image_dict['/'.join([info[-3], info[-2]])] = []
train_image_dict['/'.join([info[-3], info[-2]])].append(os.path.join(opt.source_path, entry))
with open(os.path.join(opt.source_path, opt.all_trainset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3], info[-2]]) not in test_image_dict:
test_image_dict['/'.join([info[-3], info[-2]])] = []
test_image_dict['/'.join([info[-3], info[-2]])].append(os.path.join(opt.source_path, entry))
with open(os.path.join(opt.source_path, opt.testset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3], info[-2]]) not in eval_image_dict:
eval_image_dict['/'.join([info[-3], info[-2]])] = []
eval_image_dict['/'.join([info[-3], info[-2]])].append(os.path.join(opt.source_path, entry))
new_train_dict = {}
class_ind_ind = 0
for cate in train_image_dict:
new_train_dict["te/%d" % class_ind_ind] = train_image_dict[cate]
class_ind_ind += 1
train_image_dict = new_train_dict
new_test_dict = {}
class_ind_ind = 0
for cate in test_image_dict:
new_test_dict["te/%d" % class_ind_ind] = test_image_dict[cate]
class_ind_ind += 1
test_image_dict = new_test_dict
new_eval_dict = {}
class_ind_ind = 0
for cate in eval_image_dict:
new_eval_dict["te/%d" % class_ind_ind] = eval_image_dict[cate]
class_ind_ind += 1
eval_image_dict = new_eval_dict
train_dataset = BaseTripletDataset(train_image_dict, opt, is_validation=True)
test_dataset = BaseTripletDataset(test_image_dict, opt, is_validation=True)
eval_dataset = BaseTripletDataset(eval_image_dict, opt, is_validation=True)
# train_dataset.conversion = conversion
# val_dataset.conversion = conversion
# eval_dataset.conversion = conversion
return {'training': train_dataset, 'testing': test_dataset, 'eval': eval_dataset}
def give_inat_datasets_finetune_1head(testset, cluster_label_path, opt):
"""
This function generates a training, testing and evaluation dataloader for Metric Learning on the Inaturalist 2018 dataset.
For Metric Learning, training and test sets are provided by given json files. Will define a train and test split
So no random shuffling of classes.
Args:
opt: argparse.Namespace, contains all traininig-specific parameters.
Returns:
dict of PyTorch datasets for training, testing and evaluation.
"""
# Load cluster labels from hilander results.
import pickle
train_image_dict, val_image_dict, cluster_image_dict = {}, {}, {}
with open(cluster_label_path, 'rb') as clusterf:
path2idx, global_features, global_pred_labels, gt_labels, masks = pickle.load(clusterf)
for path, idx in path2idx.items():
if global_pred_labels[idx] == -1:
continue
else:
if "te/%d" % global_pred_labels[idx] not in train_image_dict:
train_image_dict["te/%d" % global_pred_labels[idx]] = []
train_image_dict["te/%d" % global_pred_labels[idx]].append(path)
with open(os.path.join(opt.source_path, testset)) as f:
FileLines = f.readlines()
FileLines = [x.strip() for x in FileLines]
for entry in FileLines:
info = entry.split('/')
if '/'.join([info[-3], info[-2]]) not in val_image_dict:
val_image_dict['/'.join([info[-3], info[-2]])] = []
val_image_dict['/'.join([info[-3], info[-2]])].append(os.path.join(opt.source_path, entry))
train_dataset = TrainDatasetsmoothap(train_image_dict, opt)
val_dataset = BaseTripletDataset(val_image_dict, opt, is_validation=True)
# train_dataset.conversion = conversion
# val_dataset.conversion = conversion
# eval_dataset.conversion = conversion
return {'training': train_dataset, 'testing': val_dataset, 'evaluation': val_dataset}
################## BASIC PYTORCH DATASET USED FOR ALL DATASETS ##################################
class BaseTripletDataset(Dataset):
"""
Dataset class to provide (augmented) correctly prepared training samples corresponding to standard DML literature.
This includes normalizing to ImageNet-standards, and Random & Resized cropping of shapes 224 for ResNet50 and 227 for
GoogLeNet during Training. During validation, only resizing to 256 or center cropping to 224/227 is performed.
"""
def __init__(self, image_dict, opt, samples_per_class=8, is_validation=False):
"""
Dataset Init-Function.
Args:
image_dict: dict, Dictionary of shape {class_idx:[list of paths to images belong to this class] ...} providing all the training paths and classes.
opt: argparse.Namespace, contains all training-specific parameters.
samples_per_class: Number of samples to draw from one class before moving to the next when filling the batch.
is_validation: If is true, dataset properties for validation/testing are used instead of ones for training.
Returns:
Nothing!
"""
#Define length of dataset
self.n_files = np.sum([len(image_dict[key]) for key in image_dict.keys()])
self.is_validation = is_validation
self.pars = opt
self.image_dict = image_dict
self.avail_classes = sorted(list(self.image_dict.keys()))
#Convert image dictionary from classname:content to class_idx:content, because the initial indices are not necessarily from 0 - <n_classes>.
self.image_dict = {i:self.image_dict[key] for i,key in enumerate(self.avail_classes)}
self.avail_classes = sorted(list(self.image_dict.keys()))
#Init. properties that are used when filling up batches.
if not self.is_validation:
self.samples_per_class = samples_per_class
#Select current class to sample images from up to <samples_per_class>
self.current_class = np.random.randint(len(self.avail_classes))
self.classes_visited = [self.current_class, self.current_class]
self.n_samples_drawn = 0
#Data augmentation/processing methods.
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
transf_list = []
if not self.is_validation:
transf_list.extend([transforms.RandomResizedCrop(size=224) if opt.arch=='resnet50' else transforms.RandomResizedCrop(size=227),
transforms.RandomHorizontalFlip(0.5)])
else:
transf_list.extend([transforms.Resize(256),
transforms.CenterCrop(224) if opt.arch=='resnet50' else transforms.CenterCrop(227)])
transf_list.extend([transforms.ToTensor(), normalize])
self.transform = transforms.Compose(transf_list)
#Convert Image-Dict to list of (image_path, image_class). Allows for easier direct sampling.
self.image_list = [[(x,key) for x in self.image_dict[key]] for key in self.image_dict.keys()]
self.image_list = [x for y in self.image_list for x in y]
#Flag that denotes if dataset is called for the first time.
self.is_init = True
def ensure_3dim(self, img):
"""
Function that ensures that the input img is three-dimensional.
Args:
img: PIL.Image, image which is to be checked for three-dimensionality (i.e. if some images are black-and-white in an otherwise coloured dataset).
Returns:
Checked PIL.Image img.
"""
if len(img.size)==2:
img = img.convert('RGB')
return img
def __getitem__(self, idx):
"""
Args:
idx: Sample idx for training sample
Returns:
tuple of form (sample_class, torch.Tensor() of input image)
"""
if self.pars.loss == 'smoothap' or self.pars.loss == 'smoothap_element':
if self.is_init:
#self.current_class = self.avail_classes[idx%len(self.avail_classes)]
self.is_init = False
if not self.is_validation:
if self.samples_per_class==1:
return self.image_list[idx][-1], self.transform(self.ensure_3dim(Image.open(self.image_list[idx][0])))
if self.n_samples_drawn==self.samples_per_class:
#Once enough samples per class have been drawn, we choose another class to draw samples from.
#Note that we ensure with self.classes_visited that no class is chosen if it had been chosen
#previously or one before that.
counter = copy.deepcopy(self.avail_classes)
for prev_class in self.classes_visited:
if prev_class in counter: counter.remove(prev_class)
self.current_class = counter[idx%len(counter)]
#self.classes_visited = self.classes_visited[1:]+[self.current_class]
# EDIT -> there can be no class repeats
self.classes_visited = self.classes_visited+[self.current_class]
self.n_samples_drawn = 0
class_sample_idx = idx%len(self.image_dict[self.current_class])
self.n_samples_drawn += 1
out_img = self.transform(self.ensure_3dim(Image.open(self.image_dict[self.current_class][class_sample_idx])))
return self.current_class,out_img
else:
return self.image_list[idx][-1], self.transform(self.ensure_3dim(Image.open(self.image_list[idx][0])))
else:
if self.is_init:
self.current_class = self.avail_classes[idx%len(self.avail_classes)]
self.is_init = False
if not self.is_validation:
if self.samples_per_class==1:
return self.image_list[idx][-1], self.transform(self.ensure_3dim(Image.open(self.image_list[idx][0])))
if self.n_samples_drawn==self.samples_per_class:
#Once enough samples per class have been drawn, we choose another class to draw samples from.
#Note that we ensure with self.classes_visited that no class is chosen if it had been chosen
#previously or one before that.
counter = copy.deepcopy(self.avail_classes)
for prev_class in self.classes_visited:
if prev_class in counter: counter.remove(prev_class)
self.current_class = counter[idx%len(counter)]
self.classes_visited = self.classes_visited[1:]+[self.current_class]
self.n_samples_drawn = 0
class_sample_idx = idx%len(self.image_dict[self.current_class])
self.n_samples_drawn += 1
out_img = self.transform(self.ensure_3dim(Image.open(self.image_dict[self.current_class][class_sample_idx])))
return self.current_class, out_img
else:
return self.image_list[idx][-1], self.transform(self.ensure_3dim(Image.open(self.image_list[idx][0])))
def __len__(self):
return self.n_files
flatten = lambda l: [item for sublist in l for item in sublist]
######################## dataset for SmoothAP regular training ##################################
class TrainDatasetsmoothap(Dataset):
"""
This dataset class allows mini-batch formation pre-epoch, for greater speed
"""
def __init__(self, image_dict, opt):
"""
Args:
image_dict: two-level dict, `super_dict[super_class_id][class_id]` gives the list of
image paths having the same super-label and class label
"""
self.image_dict = image_dict
self.dataset_name = opt.dataset
self.batch_size = opt.bs
self.samples_per_class = opt.samples_per_class
for sub in self.image_dict:
newsub = []
for instance in self.image_dict[sub]:
newsub.append((sub, instance))
self.image_dict[sub] = newsub
# checks
# provide avail_classes
self.avail_classes = [*self.image_dict]
# Data augmentation/processing methods.
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
transf_list = []
transf_list.extend([
transforms.RandomResizedCrop(size=224) if opt.arch in ['resnet50', 'resnet50_mcn'] else transforms.RandomResizedCrop(size=227),
transforms.RandomHorizontalFlip(0.5)])
transf_list.extend([transforms.ToTensor(), normalize])
self.transform = transforms.Compose(transf_list)
self.reshuffle()
def ensure_3dim(self, img):
if len(img.size) == 2:
img = img.convert('RGB')
return img
def reshuffle(self):
image_dict = copy.deepcopy(self.image_dict)
print('shuffling data')
for sub in image_dict:
random.shuffle(image_dict[sub])
classes = [*image_dict]
random.shuffle(classes)
total_batches = []
batch = []
finished = 0
while finished == 0:
for sub_class in classes:
if (len(image_dict[sub_class]) >=self.samples_per_class) and (len(batch) < self.batch_size/self.samples_per_class) :
batch.append(image_dict[sub_class][:self.samples_per_class])
image_dict[sub_class] = image_dict[sub_class][self.samples_per_class:]
if len(batch) == self.batch_size/self.samples_per_class:
total_batches.append(batch)
batch = []
else:
finished = 1
random.shuffle(total_batches)
self.dataset = flatten(flatten(total_batches))
def __getitem__(self, idx):
# we use SequentialSampler together with SuperLabelTrainDataset,
# so idx==0 indicates the start of a new epoch
batch_item = self.dataset[idx]
if self.dataset_name == 'Inaturalist':
cls = int(batch_item[0].split('/')[1])
else:
cls = batch_item[0]
img = Image.open(batch_item[1])
return cls, self.transform(self.ensure_3dim(img))
def __len__(self):
return len(self.dataset)
class TrainDatasetsmoothap1Head(Dataset):
"""
This dataset class allows mini-batch formation pre-epoch, for greater speed
"""
def __init__(self, image_dict_L, image_dict_U, opt):
"""
Args:
image_dict: two-level dict, `super_dict[super_class_id][class_id]` gives the list of
image paths having the same super-label and class label
"""
self.image_dict_L = image_dict_L
self.image_dict_U = image_dict_U
self.dataset_name = opt.dataset
self.batch_size = opt.bs
self.samples_per_class = opt.samples_per_class
for sub_L in self.image_dict_L:
newsub_L = []
for instance in self.image_dict_L[sub_L]:
newsub_L.append((sub_L, instance))
self.image_dict_L[sub_L] = newsub_L
for sub_U in self.image_dict_U:
newsub_U = []
for instance in self.image_dict_U[sub_U]:
newsub_U.append((sub_U, instance))
self.image_dict_U[sub_U] = newsub_U
# checks
# provide avail_classes
self.avail_classes = [*self.image_dict_L] + [*self.image_dict_U]
# Data augmentation/processing methods.
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
transf_list = []
transf_list.extend([
transforms.RandomResizedCrop(size=224) if opt.arch in ['resnet50', 'resnet50_mcn'] else transforms.RandomResizedCrop(size=227),
transforms.RandomHorizontalFlip(0.5)])
transf_list.extend([transforms.ToTensor(), normalize])
self.transform = transforms.Compose(transf_list)
self.reshuffle()
def sample_same_size(self):
image_dict = copy.deepcopy(self.image_dict_L)
L_size = 0
for sub_L in self.image_dict_L:
L_size += len(self.image_dict_L[sub_L])
U_size = 0
classes_U = [*self.image_dict_U]
# while U_size < len(list(self.image_dict_U)) and U_size < L_size:
while len(classes_U) != 0:
sub_U = random.choice(classes_U)
classes_U.remove(sub_U)
sub_U_size = len(self.image_dict_U[sub_U])
if sub_U in [*image_dict]:
image_dict[sub_U].extend(self.image_dict_U[sub_U])
else:
image_dict[sub_U] = self.image_dict_U[sub_U]
U_size += sub_U_size
return image_dict
def ensure_3dim(self, img):
if len(img.size) == 2:
img = img.convert('RGB')
return img
def reshuffle(self):
image_dict = self.sample_same_size()
print('shuffling data')
for sub in image_dict:
random.shuffle(image_dict[sub])
classes = [*image_dict]
random.shuffle(classes)
total_batches = []
batch = []
finished = 0
while finished == 0:
for sub_class in classes:
if (len(image_dict[sub_class]) >=self.samples_per_class) and (len(batch) < self.batch_size/self.samples_per_class) :
batch.append(image_dict[sub_class][:self.samples_per_class])
image_dict[sub_class] = image_dict[sub_class][self.samples_per_class:]
if len(batch) == self.batch_size/self.samples_per_class:
total_batches.append(batch)
batch = []
else:
finished = 1
random.shuffle(total_batches)
self.dataset = flatten(flatten(total_batches))
def __getitem__(self, idx):
# we use SequentialSampler together with SuperLabelTrainDataset,
# so idx==0 indicates the start of a new epoch
batch_item = self.dataset[idx]
if self.dataset_name == 'Inaturalist':
cls = int(batch_item[0].split('/')[1])
else:
cls = batch_item[0]
img = Image.open(str(batch_item[1]))
return cls, self.transform(self.ensure_3dim(img))
def __len__(self):
return len(self.dataset)
examples/pytorch/hilander/PSS/Smooth_AP/src/evaluate.py 0 → 100644
View file @ e9b624fe
# repo originally forked from https://github.com/Confusezius/Deep-Metric-Learning-Baselines
##################################### LIBRARIES ###########################################
import warnings
warnings.filterwarnings("ignore")
import numpy as np, time, pickle as pkl, csv
import matplotlib.pyplot as plt
from scipy.spatial import distance
from sklearn.preprocessing import normalize
from tqdm import tqdm
import torch, torch.nn as nn
import auxiliaries as aux
import torch.multiprocessing
torch.multiprocessing.set_sharing_strategy('file_system')
"""=================================================================================================================="""
"""=================================================================================================================="""
"""========================================================="""
def evaluate(dataset, LOG, **kwargs):
"""
Given a dataset name, applies the correct evaluation function.
Args:
dataset: str, name of dataset.
LOG: aux.LOGGER instance, main logging class.
**kwargs: Input Argument Dict, depends on dataset.
Returns:
(optional) Computed metrics. Are normally written directly to LOG and printed.
"""
if dataset in ['Inaturalist', 'semi_fungi']:
ret = evaluate_one_dataset(LOG, **kwargs)
elif dataset in ['vehicle_id']:
ret = evaluate_multiple_datasets(LOG, **kwargs)
else:
raise Exception('No implementation for dataset {} available!')
return ret
"""========================================================="""
class DistanceMeasure():
"""
Container class to run and log the change of distance ratios
between intra-class distances and inter-class distances.
"""
def __init__(self, checkdata, opt, name='Train', update_epochs=1):
"""
Args:
checkdata: PyTorch DataLoader, data to check distance progression.
opt: argparse.Namespace, contains all training-specific parameters.
name: str, Name of instance. Important for savenames.
update_epochs: int, Only compute distance ratios every said epoch.
Returns:
Nothing!
"""
self.update_epochs = update_epochs
self.pars = opt
self.save_path = opt.save_path
self.name = name
self.csv_file = opt.save_path+'/distance_measures_{}.csv'.format(self.name)
with open(self.csv_file,'a') as csv_file:
writer = csv.writer(csv_file, delimiter=',')
writer.writerow(['Rel. Intra/Inter Distance'])
self.checkdata = checkdata
self.mean_class_dists = []
self.epochs = []
def measure(self, model, epoch):
"""
Compute distance ratios of intra- and interclass distance.
Args:
model: PyTorch Network, network that produces the resp. embeddings.
epoch: Current epoch.
Returns:
Nothing!
"""
if epoch%self.update_epochs: return
self.epochs.append(epoch)
torch.cuda.empty_cache()
_ = model.eval()
#Compute Embeddings
with torch.no_grad():
feature_coll, target_coll = [],[]
data_iter = tqdm(self.checkdata, desc='Estimating Data Distances...')
for idx, data in enumerate(data_iter):
input_img, target = data[1], data[0]
features = model(input_img.to(self.pars.device))
feature_coll.extend(features.cpu().detach().numpy().tolist())
target_coll.extend(target.numpy().tolist())
feature_coll = np.vstack(feature_coll).astype('float32')
target_coll = np.hstack(target_coll).reshape(-1)
avail_labels = np.unique(target_coll)
#Compute indixes of embeddings for each class.
class_positions = []
for lab in avail_labels:
class_positions.append(np.where(target_coll==lab)[0])
#Compute average intra-class distance and center of mass.
com_class, dists_class = [],[]
for class_pos in class_positions:
dists = distance.cdist(feature_coll[class_pos],feature_coll[class_pos],'cosine')
dists = np.sum(dists)/(len(dists)**2-len(dists))
# dists = np.linalg.norm(np.std(feature_coll_aux[class_pos],axis=0).reshape(1,-1)).reshape(-1)
com = normalize(np.mean(feature_coll[class_pos],axis=0).reshape(1,-1)).reshape(-1)
dists_class.append(dists)
com_class.append(com)
#Compute mean inter-class distances by the class-coms.
mean_inter_dist = distance.cdist(np.array(com_class), np.array(com_class), 'cosine')
mean_inter_dist = np.sum(mean_inter_dist)/(len(mean_inter_dist)**2-len(mean_inter_dist))
#Compute distance ratio
mean_class_dist = np.mean(np.array(dists_class)/mean_inter_dist)
self.mean_class_dists.append(mean_class_dist)
self.update(mean_class_dist)
def update(self, mean_class_dist):
"""
Update Loggers.
Args:
mean_class_dist: float, Distance Ratio
Returns:
Nothing!
"""
self.update_csv(mean_class_dist)
self.update_plot()
def update_csv(self, mean_class_dist):
"""
Update CSV.
Args:
mean_class_dist: float, Distance Ratio
Returns:
Nothing!
"""
with open(self.csv_file, 'a') as csv_file:
writer = csv.writer(csv_file, delimiter=',')
writer.writerow([mean_class_dist])
def update_plot(self):
"""
Update progression plot.
Args:
None.
Returns:
Nothing!
"""
plt.style.use('ggplot')
f,ax = plt.subplots(1)
ax.set_title('Mean Intra- over Interclassdistances')
ax.plot(self.epochs, self.mean_class_dists, label='Class')
f.legend()
f.set_size_inches(15,8)
f.savefig(self.save_path+'/distance_measures_{}.svg'.format(self.name))
class GradientMeasure():
"""
Container for gradient measure functionalities.
Measure the gradients coming from the embedding layer to the final conv. layer
to examine learning signal.
"""
def __init__(self, opt, name='class-it'):
"""
Args:
opt: argparse.Namespace, contains all training-specific parameters.
name: Name of class instance. Important for the savename.
Returns:
Nothing!
"""
self.pars = opt
self.name = name
self.saver = {'grad_normal_mean':[], 'grad_normal_std':[], 'grad_abs_mean':[], 'grad_abs_std':[]}
def include(self, params):
"""
Include the gradients for a set of parameters, normally the final embedding layer.
Args:
params: PyTorch Network layer after .backward() was called.
Returns:
Nothing!
"""
gradients = [params.weight.grad.detach().cpu().numpy()]
for grad in gradients:
### Shape: 128 x 2048
self.saver['grad_normal_mean'].append(np.mean(grad,axis=0))
self.saver['grad_normal_std'].append(np.std(grad,axis=0))
self.saver['grad_abs_mean'].append(np.mean(np.abs(grad),axis=0))
self.saver['grad_abs_std'].append(np.std(np.abs(grad),axis=0))
def dump(self, epoch):
"""
Append all gradients to a pickle file.
Args:
epoch: Current epoch
Returns:
Nothing!
"""
with open(self.pars.save_path+'/grad_dict_{}.pkl'.format(self.name),'ab') as f:
pkl.dump([self.saver], f)
self.saver = {'grad_normal_mean':[], 'grad_normal_std':[], 'grad_abs_mean':[], 'grad_abs_std':[]}
"""========================================================="""
def evaluate_one_dataset(LOG, dataloader, model, opt, save=True, give_return=True, epoch=0):
"""
Compute evaluation metrics, update LOGGER and print results.
Args:
LOG: aux.LOGGER-instance. Main Logging Functionality.
dataloader: PyTorch Dataloader, Testdata to be evaluated.
model: PyTorch Network, Network to evaluate.
opt: argparse.Namespace, contains all training-specific parameters.
save: bool, if True, Checkpoints are saved when testing metrics (specifically Recall @ 1) improve.
give_return: bool, if True, return computed metrics.
epoch: int, current epoch, required for logger.
Returns:
(optional) Computed metrics. Are normally written directly to LOG and printed.
"""
start = time.time()
image_paths = np.array(dataloader.dataset.image_list)
with torch.no_grad():
#Compute Metrics
F1, NMI, recall_at_ks, feature_matrix_all = aux.eval_metrics_one_dataset(model, dataloader, device=opt.device, k_vals=opt.k_vals, opt=opt)
#Make printable summary string.
result_str = ', '.join('@{0}: {1:.4f}'.format(k,rec) for k,rec in zip(opt.k_vals, recall_at_ks))
result_str = 'Epoch (Test) {0}: NMI [{1:.4f}] | F1 [{2:.4f}] | Recall [{3}]'.format(epoch, NMI, F1, result_str)
if LOG is not None:
if save:
if not len(LOG.progress_saver['val']['Recall @ 1']) or recall_at_ks[0]>np.max(LOG.progress_saver['val']['Recall @ 1']):
#Save Checkpoint
print("Set checkpoint at {}.".format(LOG.prop.save_path+'/checkpoint_{}.pth.tar'.format(opt.iter)))
aux.set_checkpoint(model, opt, LOG.progress_saver, LOG.prop.save_path+'/checkpoint_{}.pth.tar'.format(opt.iter))
# aux.recover_closest_one_dataset(feature_matrix_all, image_paths, LOG.prop.save_path+'/sample_recoveries.png')
#Update logs.
LOG.log('val', LOG.metrics_to_log['val'], [epoch, np.round(time.time()-start), NMI, F1]+recall_at_ks)
print(result_str)
if give_return:
return recall_at_ks, NMI, F1
else:
None
"""========================================================="""
def evaluate_query_and_gallery_dataset(LOG, query_dataloader, gallery_dataloader, model, opt, save=True, give_return=True, epoch=0):
"""
Compute evaluation metrics, update LOGGER and print results, specifically for In-Shop Clothes.
Args:
LOG: aux.LOGGER-instance. Main Logging Functionality.
query_dataloader: PyTorch Dataloader, Query-testdata to be evaluated.
gallery_dataloader: PyTorch Dataloader, Gallery-testdata to be evaluated.
model: PyTorch Network, Network to evaluate.
opt: argparse.Namespace, contains all training-specific parameters.
save: bool, if True, Checkpoints are saved when testing metrics (specifically Recall @ 1) improve.
give_return: bool, if True, return computed metrics.
epoch: int, current epoch, required for logger.
Returns:
(optional) Computed metrics. Are normally written directly to LOG and printed.
"""
start = time.time()
query_image_paths = np.array([x[0] for x in query_dataloader.dataset.image_list])
gallery_image_paths = np.array([x[0] for x in gallery_dataloader.dataset.image_list])
with torch.no_grad():
#Compute Metri cs.
F1, NMI, recall_at_ks, query_feature_matrix_all, gallery_feature_matrix_all = aux.eval_metrics_query_and_gallery_dataset(model, query_dataloader, gallery_dataloader, device=opt.device, k_vals = opt.k_vals, opt=opt)
#Generate printable summary string.
result_str = ', '.join('@{0}: {1:.4f}'.format(k,rec) for k,rec in zip(opt.k_vals, recall_at_ks))
result_str = 'Epoch (Test) {0}: NMI [{1:.4f}] | F1 [{2:.4f}] | Recall [{3}]'.format(epoch, NMI, F1, result_str)
if LOG is not None:
if save:
if not len(LOG.progress_saver['val']['Recall @ 1']) or recall_at_ks[0]>np.max(LOG.progress_saver['val']['Recall @ 1']):
#Save Checkpoint
aux.set_checkpoint(model, opt, LOG.progress_saver, LOG.prop.save_path+'/checkpoint.pth.tar')
aux.recover_closest_inshop(query_feature_matrix_all, gallery_feature_matrix_all, query_image_paths, gallery_image_paths, LOG.prop.save_path+'/sample_recoveries.png')
#Update logs.
LOG.log('val', LOG.metrics_to_log['val'], [epoch, np.round(time.time()-start), NMI, F1]+recall_at_ks)
print(result_str)
if give_return:
return recall_at_ks, NMI, F1
else:
None
"""========================================================="""
def evaluate_multiple_datasets(LOG, dataloaders, model, opt, save=True, give_return=True, epoch=0):
"""
Compute evaluation metrics, update LOGGER and print results, specifically for Multi-test datasets s.a. PKU Vehicle ID.
Args:
LOG: aux.LOGGER-instance. Main Logging Functionality.
dataloaders: List of PyTorch Dataloaders, test-dataloaders to evaluate.
model: PyTorch Network, Network to evaluate.
opt: argparse.Namespace, contains all training-specific parameters.
sa ve: bool, if True, Checkpoints are saved when testing metrics (specifically Recall @ 1) improve.
give_return: bool, i f True, return computed metrics.
epoch: int, current epoch, required for logger.
Returns :
(optional) Computed metrics. Are normally written directly to LOG and printed.
"""
start = time.time()
csv_data = [epoch]
with torch.no_grad():
for i,dataloader in enumerate(dataloaders):
print('Working on Set {}/{}'.format(i+1, len(dataloaders)))
image_paths = np.array(dataloader.dataset.image_list)
#Compute Metrics for specific testset.
F1, NMI, recall_at_ks, feature_matrix_all = aux.eval_metrics_one_dataset(model, dataloader, device=opt.device, k_vals=opt.k_vals, opt=opt)
#Generate printable summary string.
result_str = ', '.join('@{0}: {1:.4f}'.format(k,rec) for k,rec in zip(opt.k_vals, recall_at_ks))
result_str = 'SET {0}: Epoch (Test) {1}: NMI [{2:.4f}] | F1 {3:.4f}| Recall [{4}]'.format(i+1, epoch, NMI, F1, result_str)
if LOG is not None:
if save:
if not len(LOG.progress_saver['val']['Set {} Recall @ 1'.format(i)]) or recall_at_ks[0]>np.max(LOG.progress_saver['val']['Set {} Recall @ 1'.format(i)]):
#Save Checkpoint for specific test set.
aux.set_checkpoint(model, opt, LOG.progress_saver, LOG.prop.save_path+'/checkpoint_set{}.pth.tar'.format(i+1))
aux.recover_closest_one_dataset(feature_matrix_all, image_paths, LOG.prop.save_path+'/sample_recoveries_set{}.png'.format(i+1))
csv_data += [NMI, F1]+recall_at_ks
print(result_str)
csv_data.insert(0, np.round(time.time()-start))
#Update logs.
LOG.log('val', LOG.metrics_to_log['val'], csv_data)
#if give_return:
return csv_data[2:]
#else:
# None
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment