lsun.py

import torch.utils.data as data
from PIL import Image
import os
import os.path
import StringIO
import string
import sys
if sys.version_info[0] == 2:
    import cPickle as pickle
else:
    import pickle

class LSUNClassDataset(data.Dataset):
    def __init__(self, db_path, transform=None, target_transform=None):
        import lmdb
        self.db_path = db_path
        self.env = lmdb.open(db_path, map_size=1099511627776,
                        max_readers=100, readonly=True)
        with self.env.begin(write=False) as txn:
            self.length = txn.stat()['entries']
        cache_file = '_cache_' + db_path.replace('/', '_')
        if os.path.isfile(cache_file):
            self.keys = pickle.load( open( cache_file, "rb" ) )
        else:
            with self.env.begin(write=False) as txn:
                self.keys = [ key for key, _ in txn.cursor() ]
            pickle.dump( self.keys, open( cache_file, "wb" ) )
        self.transform = transform
        self.target_transform = target_transform

    def __getitem__(self, index):
        img, target = None, None
        env = self.env
        with env.begin(write=False) as txn:
            imgbuf = txn.get(self.keys[index])

        buf = StringIO.StringIO()
        buf.write(imgbuf)
        buf.seek(0)
        img = Image.open(buf).convert('RGB')

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target

    def __len__(self):
        return self.length

    def __repr__(self):
        return self.__class__.__name__ + ' (' + self.db_path + ')'

class LSUNDataset(data.Dataset):
    """
    db_path = root directory for the database files
    classes = 'train' | 'val' | 'test' | ['bedroom_train', 'church_train', ...]
    """
    def __init__(self, db_path, classes='train',
                 transform=None, target_transform=None):
        categories = ['bedroom', 'bridge', 'church_outdoor', 'classroom',
                      'conference_room', 'dining_room', 'kitchen',
                      'living_room', 'restaurant', 'tower']
        dset_opts = ['train', 'val', 'test']
        self.db_path = db_path
        if type(classes) == str and classes in dset_opts:
            classes = [c + '_' + classes for c in categories]
        if type(classes) == list:
            for c in classes:
                c_short = c.split('_')
                c_short.pop(len(c_short) - 1)
                c_short = string.join(c_short, '_')
                if c_short not in categories:
                    raise(ValueError('Unknown LSUN class: ' + c_short + '.'\
                          'Options are: ' + str(categories)))
                c_short = c.split('_')
                c_short = c_short.pop(len(c_short) - 1)
                if c_short not in dset_opts:
                    raise(ValueError('Unknown postfix: ' + c_short + '.'\
                          'Options are: ' + str(dset_opts)))
        else:
            raise(ValueError('Unknown option for classes'))
        self.classes = classes

        # for each class, create an LSUNClassDataset
        self.dbs = []
        for c in self.classes:
            self.dbs.append(LSUNClassDataset(
                db_path = db_path + '/' + c + '_lmdb',
                transform = transform))

        self.indices = []
        count = 0
        for db in self.dbs:
            count += len(db)
            self.indices.append(count)

        self.length = count
        self.target_transform = target_transform

    def __getitem__(self, index):
        target = 0
        sub = 0
        for ind in self.indices:
            if index < ind:
                break
            target += 1
            sub += ind

        db = self.dbs[target]
        index = index - sub

        if self.target_transform is not None:
            target = self.target_transform(target)

        return db[index], target

    def __len__(self):
        return self.length

    def __repr__(self):
        return self.__class__.__name__ + ' (' + self.db_path + ')'

if __name__ == '__main__':
    #lsun = LSUNClassDataset(db_path='/home/soumith/local/lsun/train/bedroom_train_lmdb')
    #a = lsun[0]
    lsun = LSUNDataset(db_path='/home/soumith/local/lsun/train',
                       classes=['bedroom_train', 'church_outdoor_train'])
    print(lsun.classes)
    print(lsun.dbs)
    a, t = lsun[len(lsun)-1]
    print(a)
    print(t)