gat.py

import networkx as nx
from dgl.graph import DGLGraph
import torch
import torch.nn as nn
import torch.nn.functional as F
import argparse
from dataset import load_data, preprocess_features
import numpy as np

class NodeUpdateModule(nn.Module):
    def __init__(self, input_dim, num_hidden, aggregator, num_heads=3, act=None,
            attention_dropout=None, input_dropout=None, residual=False):
        super(NodeUpdateModule, self).__init__()
        self.num_hidden = num_hidden
        self.num_heads = num_heads
        self.fc = nn.ModuleList(
                [nn.Linear(input_dim, num_hidden, bias=False)
                    for _ in range(num_heads)])
        self.attention = nn.ModuleList(
                [nn.Linear(num_hidden * 2, 1, bias=False) for _ in range(num_heads)])
        self.act = act
        self.attention_dropout = attention_dropout
        self.input_dropout = input_dropout
        self.aggregator = aggregator
        self.residual = residual

    def forward(self, node, msgs):
        hv = node['h']
        hu = torch.cat(msgs, dim=0)

        # number of neighbors, including itself
        n = len(msgs) + 1

        out = []
        for i in range(self.num_heads):
            hvv = hv
            huu = hu
            if self.input_dropout is not None:
                hvv = F.dropout(hvv, self.input_dropout)
                huu = F.dropout(huu, self.input_dropout)
            # calc W*hself and W*hneigh
            hvv = self.fc[i](hv)
            huu = self.fc[i](hu)
            # concat itself with neighbors to make self-attention
            huu = torch.cat((hvv, huu), dim=0)
            # calculate W*hself||W*hneigh
            h = torch.cat((hvv.expand(n, -1), huu), dim=1)
            a = F.leaky_relu(self.attention[i](h))
            a = F.softmax(a, dim=0)
            if self.attention_dropout is not None:
                a = F.dropout(a, self.attention_dropout)
            if self.input_dropout is not None:
                hvv = F.dropout(hvv, self.input_dropout)
            h = torch.sum(a * hvv, 0, keepdim=True)
            # add residual connection
            if self.residual:
                h += hvv
            if self.act is not None:
                h = self.act(h)
            out.append(h)

        # aggregate multi-head results
        h = self.aggregator(out)
        return {'h': h}


class GAT(nn.Module):
    def __init__(self, num_layers, in_dim, num_hidden, num_classes, num_heads,
            activation, attention_dropout, input_dropout, use_residual=False):
        super(GAT, self).__init__()
        self.layers = nn.ModuleList()
        # update layers
        aggregator = lambda x: torch.cat(x, 1)
        for i in range(num_layers):
            if i == 0:
                last_dim = in_dim
                residual = False
            else:
                last_dim = num_hidden * num_heads # because of concat heads
                residual = use_residual
            self.layers.append(
                    NodeUpdateModule(last_dim, num_hidden, aggregator, num_heads,
                        activation, attention_dropout, input_dropout, residual))
        # projection layer
        # FIXME: does pytorch has something similar to tf.add_n which sum over a list?
        aggregator = lambda x: reduce(lambda a, b: a+b, x)
        self.layers.append(NodeUpdateModule(num_hidden * 3, num_classes, aggregator,
            1, None, attention_dropout, input_dropout, False))

    def forward(self, g):
        g.register_message_func(lambda src, dst, edge: src['h'])
        for layer in self.layers:
            g.register_update_func(layer)
            g.update_all()
        logits = [g.node[n]['h'] for n in g.nodes()]
        logits = torch.cat(logits, dim=0)
        return logits


def main(args):
    # dropout parameters
    input_dropout = 0.2
    attention_dropout = 0.2

    # load and preprocess dataset
    adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(args.dataset)
    features = preprocess_features(features)

    # initialize graph
    g = DGLGraph(adj)

    # create model
    model = GAT(args.num_layers,
                features.shape[1],
                args.num_hidden,
                y_train.shape[1],
                args.num_heads,
                F.elu,
                attention_dropout,
                input_dropout,
                args.residual)

    # use optimizer
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)

    # convert labels and masks to tensor
    labels = torch.FloatTensor(y_train)
    mask = torch.FloatTensor(train_mask.astype(np.float32))
    n_train = torch.sum(mask)

    for epoch in range(args.epochs):
        # reset grad
        optimizer.zero_grad()

        # reset graph states
        for n in g.nodes():
            g.node[n]['h'] = torch.FloatTensor(features[n].toarray())

        # forward
        logits = model.forward(g)

        # masked cross entropy loss
        # TODO: (lingfan) use gather to speed up
        logp = F.log_softmax(logits, 1)
        loss = -torch.sum(logp * labels * mask.view(-1, 1)) / n_train
        print("epoch {} loss: {}".format(epoch, loss.item()))

        loss.backward()
        optimizer.step()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='GAT')
    parser.add_argument("--dataset", type=str, required=True,
            help="dataset name")
    parser.add_argument("--epochs", type=int, default=10,
            help="training epoch")
    parser.add_argument("--num-heads", type=int, default=3,
            help="number of attentional heads to use")
    parser.add_argument("--num-layers", type=int, default=1,
            help="number of hidden layers")
    parser.add_argument("--num-hidden", type=int, default=8,
            help="size of hidden units")
    parser.add_argument("--residual", action="store_true",
            help="use residual connection")
    parser.add_argument("--lr", type=float, default=0.001,
            help="learning rate")
    args = parser.parse_args()
    print(args)

    main(args)