Initial commit

tensorlayer/activation.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+
+
+
+import tensorflow as tf
+
+def identity(x, name=None):
+    """The identity activation function, Shortcut is ``linear``.
+
+    Parameters
+    ----------
+    x : a tensor input
+        input(s)
+
+
+    Returns
+    --------
+    A `Tensor` with the same type as `x`.
+    """
+    return x
+
+# Shortcut
+linear = identity
+
+def ramp(x=None, v_min=0, v_max=1, name=None):
+    """The ramp activation function.
+
+    Parameters
+    ----------
+    x : a tensor input
+        input(s)
+    v_min : float
+        if input(s) smaller than v_min, change inputs to v_min
+    v_max : float
+        if input(s) greater than v_max, change inputs to v_max
+    name : a string or None
+        An optional name to attach to this activation function.
+
+
+    Returns
+    --------
+    A `Tensor` with the same type as `x`.
+    """
+    return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)
+
+def leaky_relu(x=None, alpha=0.1, name="LeakyReLU"):
+    """The LeakyReLU, Shortcut is ``lrelu``.
+
+    Modified version of ReLU, introducing a nonzero gradient for negative
+    input.
+
+    Parameters
+    ----------
+    x : A `Tensor` with type `float`, `double`, `int32`, `int64`, `uint8`,
+        `int16`, or `int8`.
+    alpha : `float`. slope.
+    name : a string or None
+        An optional name to attach to this activation function.
+
+    Examples
+    ---------
+    >>> network = tl.layers.DenseLayer(network, n_units=100, name = 'dense_lrelu',
+    ...                 act= lambda x : tl.act.lrelu(x, 0.2))
+
+    References
+    ------------
+    - `Rectifier Nonlinearities Improve Neural Network Acoustic Models, Maas et al. (2013) <http://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf>`_
+    """
+    with tf.name_scope(name) as scope:
+        # x = tf.nn.relu(x)
+        # m_x = tf.nn.relu(-x)
+        # x -= alpha * m_x
+        x = tf.maximum(x, alpha * x)
+    return x
+
+#Shortcut
+lrelu = leaky_relu
+
+def pixel_wise_softmax(output, name='pixel_wise_softmax'):
+    """Return the softmax outputs of images, every pixels have multiple label, the sum of a pixel is 1.
+    Usually be used for image segmentation.
+
+    Parameters
+    ------------
+    output : tensor
+        - For 2d image, 4D tensor [batch_size, height, weight, channel], channel >= 2.
+        - For 3d image, 5D tensor [batch_size, depth, height, weight, channel], channel >= 2.
+
+    Examples
+    ---------
+    >>> outputs = pixel_wise_softmax(network.outputs)
+    >>> dice_loss = 1 - dice_coe(outputs, y_, epsilon=1e-5)
+
+    References
+    -----------
+    - `tf.reverse <https://www.tensorflow.org/versions/master/api_docs/python/array_ops.html#reverse>`_
+    """
+    with tf.name_scope(name) as scope:
+        return tf.nn.softmax(output)
+        ## old implementation
+        # exp_map = tf.exp(output)
+        # if output.get_shape().ndims == 4:   # 2d image
+        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, True]))
+        # elif output.get_shape().ndims == 5: # 3d image
+        #     evidence = tf.add(exp_map, tf.reverse(exp_map, [False, False, False, False, True]))
+        # else:
+        #     raise Exception("output parameters should be 2d or 3d image, not %s" % str(output._shape))
+        # return tf.div(exp_map, evidence)

tensorlayer/db.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+"""
+Experimental Database Management System.
+
+Latest Version
+"""
+
+
+import tensorflow as tf
+import tensorlayer as tl
+import numpy as np
+import time
+import math
+
+
+import uuid
+
+import pymongo
+import gridfs
+import pickle
+from pymongo import MongoClient
+from datetime import datetime
+
+import inspect
+
+def AutoFill(func):
+    def func_wrapper(self,*args,**kwargs):
+        d=inspect.getcallargs(func,self,*args,**kwargs)
+        d['args'].update({"studyID":self.studyID})
+        return  func(**d)
+    return func_wrapper
+
+
+
+
+
+
+class TensorDB(object):
+    """TensorDB is a MongoDB based manager that help you to manage data, network topology, parameters and logging.
+
+    Parameters
+    -------------
+    ip : string, localhost or IP address.
+    port : int, port number.
+    db_name : string, database name.
+    user_name : string, set to None if it donnot need authentication.
+    password : string.
+
+    Properties
+    ------------
+    db : ``pymongo.MongoClient[db_name]``, xxxxxx
+    datafs : ``gridfs.GridFS(self.db, collection="datafs")``, xxxxxxxxxx
+    modelfs : ``gridfs.GridFS(self.db, collection="modelfs")``,
+    paramsfs : ``gridfs.GridFS(self.db, collection="paramsfs")``,
+    db.Params : Collection for
+    db.TrainLog : Collection for
+    db.ValidLog : Collection for
+    db.TestLog : Collection for
+    studyID : string, unique ID, if None random generate one.
+
+    Dependencies
+    -------------
+    1 : MongoDB, as TensorDB is based on MongoDB, you need to install it in your
+       local machine or remote machine.
+    2 : pip install pymongo, for MongoDB python API.
+
+    Optional Tools
+    ----------------
+    1 : You may like to install MongoChef or Mongo Management Studo APP for
+       visualizing or testing your MongoDB.
+    """
+    def __init__(
+        self,
+        ip = 'localhost',
+        port = 27017,
+        db_name = 'db_name',
+        user_name = None,
+        password = 'password',
+        studyID=None
+    ):
+        ## connect mongodb
+        client = MongoClient(ip, port)
+        self.db = client[db_name]
+        if user_name != None:
+            self.db.authenticate(user_name, password)
+
+
+        if studyID is None:
+            self.studyID=str(uuid.uuid1())
+        else:
+            self.studyID=studyID
+
+        ## define file system (Buckets)
+        self.datafs = gridfs.GridFS(self.db, collection="datafs")
+        self.modelfs = gridfs.GridFS(self.db, collection="modelfs")
+        self.paramsfs = gridfs.GridFS(self.db, collection="paramsfs")
+        self.archfs=gridfs.GridFS(self.db,collection="ModelArchitecture")
+        ##
+        print("[TensorDB] Connect SUCCESS {}:{} {} {} {}".format(ip, port, db_name, user_name, studyID))
+
+        self.ip = ip
+        self.port = port
+        self.db_name = db_name
+        self.user_name = user_name
+
+    def __autofill(self,args):
+        return args.update({'studyID':self.studyID})
+
+    def __serialization(self,ps):
+        return pickle.dumps(ps, protocol=2)
+
+    def __deserialization(self,ps):
+        return pickle.loads(ps)
+
+    def save_params(self, params=[], args={}):#, file_name='parameters'):
+        """ Save parameters into MongoDB Buckets, and save the file ID into Params Collections.
+
+        Parameters
+        ----------
+        params : a list of parameters
+        args : dictionary, item meta data.
+
+        Returns
+        ---------
+        f_id : the Buckets ID of the parameters.
+        """
+        self.__autofill(args)
+        s = time.time()
+        f_id = self.paramsfs.put(self.__serialization(params))#, file_name=file_name)
+        args.update({'f_id': f_id, 'time': datetime.utcnow()})
+        self.db.Params.insert_one(args)
+        # print("[TensorDB] Save params: {} SUCCESS, took: {}s".format(file_name, round(time.time()-s, 2)))
+        print("[TensorDB] Save params: SUCCESS, took: {}s".format(round(time.time()-s, 2)))
+        return f_id
+
+    @AutoFill
+    def find_one_params(self, args={},sort=None):
+        """ Find one parameter from MongoDB Buckets.
+
+        Parameters
+        ----------
+        args : dictionary, find items.
+
+        Returns
+        --------
+        params : the parameters, return False if nothing found.
+        f_id : the Buckets ID of the parameters, return False if nothing found.
+        """
+
+        s = time.time()
+        # print(args)
+        d = self.db.Params.find_one(filter=args,sort=sort)
+
+        if d is not None:
+            f_id = d['f_id']
+        else:
+            print("[TensorDB] FAIL! Cannot find: {}".format(args))
+            return False, False
+        try:
+            params = self.__deserialization(self.paramsfs.get(f_id).read())
+            print("[TensorDB] Find one params SUCCESS, {} took: {}s".format(args, round(time.time()-s, 2)))
+            return params, f_id
+        except:
+            return False, False
+
+    @AutoFill
+    def find_all_params(self, args={}):
+        """ Find all parameter from MongoDB Buckets
+
+        Parameters
+        ----------
+        args : dictionary, find items
+
+        Returns
+        --------
+        params : the parameters, return False if nothing found.
+
+        """
+
+        s = time.time()
+        pc = self.db.Params.find(args)
+
+        if pc is not None:
+            f_id_list = pc.distinct('f_id')
+            params = []
+            for f_id in f_id_list: # you may have multiple Buckets files
+                tmp = self.paramsfs.get(f_id).read()
+                params.append(self.__deserialization(tmp))
+        else:
+            print("[TensorDB] FAIL! Cannot find any: {}".format(args))
+            return False
+
+        print("[TensorDB] Find all params SUCCESS, took: {}s".format(round(time.time()-s, 2)))
+        return params
+
+    @AutoFill
+    def del_params(self, args={}):
+        """ Delete params in MongoDB uckets.
+
+        Parameters
+        -----------
+        args : dictionary, find items to delete, leave it empty to delete all parameters.
+        """
+
+        pc = self.db.Params.find(args)
+        f_id_list = pc.distinct('f_id')
+        # remove from Buckets
+        for f in f_id_list:
+            self.paramsfs.delete(f)
+        # remove from Collections
+        self.db.Params.remove(args)
+
+        print("[TensorDB] Delete params SUCCESS: {}".format(args))
+
+    def _print_dict(self, args):
+        # return " / ".join(str(key) + ": "+ str(value) for key, value in args.items())
+
+        string = ''
+        for key, value in args.items():
+            if key is not '_id':
+                string += str(key) + ": "+ str(value) + " / "
+        return string
+
+    ## =========================== LOG =================================== ##
+    @AutoFill
+    def train_log(self, args={}):
+        """Save the training log.
+
+        Parameters
+        -----------
+        args : dictionary, items to save.
+
+        Examples
+        ---------
+        >>> db.train_log(time=time.time(), {'loss': loss, 'acc': acc})
+        """
+
+        _result = self.db.TrainLog.insert_one(args)
+        _log = self._print_dict(args)
+        #print("[TensorDB] TrainLog: " +_log)
+        return _result
+
+    @AutoFill
+    def del_train_log(self, args={}):
+        """ Delete train log.
+
+        Parameters
+        -----------
+        args : dictionary, find items to delete, leave it empty to delete all log.
+        """
+
+        self.db.TrainLog.delete_many(args)
+        print("[TensorDB] Delete TrainLog SUCCESS")
+
+    @AutoFill
+    def valid_log(self, args={}):
+        """Save the validating log.
+
+        Parameters
+        -----------
+        args : dictionary, items to save.
+
+        Examples
+        ---------
+        >>> db.valid_log(time=time.time(), {'loss': loss, 'acc': acc})
+        """
+
+        _result = self.db.ValidLog.insert_one(args)
+        # _log = "".join(str(key) + ": " + str(value) for key, value in args.items())
+        _log = self._print_dict(args)
+        print("[TensorDB] ValidLog: " +_log)
+        return _result
+
+    @AutoFill
+    def del_valid_log(self, args={}):
+        """ Delete validation log.
+
+        Parameters
+        -----------
+        args : dictionary, find items to delete, leave it empty to delete all log.
+        """
+        self.db.ValidLog.delete_many(args)
+        print("[TensorDB] Delete ValidLog SUCCESS")
+
+    @AutoFill
+    def test_log(self, args={}):
+        """Save the testing log.
+
+        Parameters
+        -----------
+        args : dictionary, items to save.
+
+        Examples
+        ---------
+        >>> db.test_log(time=time.time(), {'loss': loss, 'acc': acc})
+        """
+
+        _result = self.db.TestLog.insert_one(args)
+        # _log = "".join(str(key) + str(value) for key, value in args.items())
+        _log = self._print_dict(args)
+        print("[TensorDB] TestLog: " +_log)
+        return _result
+
+    @AutoFill
+    def del_test_log(self, args={}):
+        """ Delete test log.
+
+        Parameters
+        -----------
+        args : dictionary, find items to delete, leave it empty to delete all log.
+        """
+
+        self.db.TestLog.delete_many(args)
+        print("[TensorDB] Delete TestLog SUCCESS")
+
+    ## =========================== Network Architecture ================== ##
+    @AutoFill
+    def save_model_architecture(self,s,args={}):
+        self.__autofill(args)
+        fid=self.archfs.put(s,filename="modelarchitecture")
+        args.update({"fid":fid})
+        self.db.march.insert_one(args)
+
+    @AutoFill
+    def load_model_architecture(self,args={}):
+
+        d = self.db.march.find_one(args)
+        if d is not None:
+            fid = d['fid']
+            print(d)
+            print(fid)
+            # "print find"
+        else:
+            print("[TensorDB] FAIL! Cannot find: {}".format(args))
+            print ("no idtem")
+            return False, False
+        try:
+            archs = self.archfs.get(fid).read()
+            '''print("[TensorDB] Find one params SUCCESS, {} took: {}s".format(args, round(time.time()-s, 2)))'''
+            return archs, fid
+        except Exception as e:
+            print("exception")
+            print(e)
+            return False, False
+
+    @AutoFill
+    def save_job(self, script=None, args={}):
+        """Save the job.
+
+        Parameters
+        -----------
+        script : a script file name or None.
+        args : dictionary, items to save.
+
+        Examples
+        ---------
+        >>> # Save your job
+        >>> db.save_job('your_script.py', {'job_id': 1, 'learning_rate': 0.01, 'n_units': 100})
+        >>> # Run your job
+        >>> temp = db.find_one_job(args={'job_id': 1})
+        >>> print(temp['learning_rate'])
+        ... 0.01
+        >>> import _your_script
+        ... running your script
+        """
+        self.__autofill(args)
+        if script is not None:
+            _script = open(script, 'rb').read()
+            args.update({'script': _script, 'script_name': script})
+        # _result = self.db.Job.insert_one(args)
+        _result = self.db.Job.replace_one(args, args, upsert=True)
+        _log = self._print_dict(args)
+        print("[TensorDB] Save Job: script={}, args={}".format(script, args))
+        return _result
+
+    @AutoFill
+    def find_one_job(self, args={}):
+        """ Find one job from MongoDB Job Collections.
+
+        Parameters
+        ----------
+        args : dictionary, find items.
+
+        Returns
+        --------
+        dictionary : contains all meta data and script.
+        """
+
+
+        temp = self.db.Job.find_one(args)
+
+        if temp is not None:
+            if 'script_name' in temp.keys():
+                f = open('_' + temp['script_name'], 'wb')
+                f.write(temp['script'])
+                f.close()
+            print("[TensorDB] Find Job: {}".format(args))
+        else:
+            print("[TensorDB] FAIL! Cannot find any: {}".format(args))
+            return False
+
+        return temp
+
+    def push_job(self,margs, wargs,dargs,epoch):
+
+        ms,mid=self.load_model_architecture(margs)
+        weight,wid=self.find_one_params(wargs)
+        args={"weight":wid,"model":mid,"dargs":dargs,"epoch":epoch,"time":datetime.utcnow(),"Running":False}
+        self.__autofill(args)
+        self.db.JOBS.insert_one(args)
+
+    def peek_job(self):
+        args={'Running':False}
+        self.__autofill(args)
+        m=self.db.JOBS.find_one(args)
+        print(m)
+        if m is None:
+            return False
+
+        s=self.paramsfs.get(m['weight']).read()
+        w=self.__deserialization(s)
+
+        ach=self.archfs.get(m['model']).read()
+
+        return m['_id'], ach,w,m["dargs"],m['epoch']
+
+    def run_job(self,jid):
+        self.db.JOBS.find_one_and_update({'_id':jid},{'$set': {'Running': True,"Since":datetime.utcnow()}})
+
+    def del_job(self,jid):
+        self.db.JOBS.find_one_and_update({'_id':jid},{'$set': {'Running': True,"Finished":datetime.utcnow()}})
+
+    def __str__(self):
+        _s = "[TensorDB] Info:\n"
+        _t = _s + "    " + str(self.db)
+        return _t
+
+    # def save_bulk_data(self, data=None, filename='filename'):
+    #     """ Put bulk data into TensorDB.datafs, return file ID.
+    #     When you have a very large data, you may like to save it into GridFS Buckets
+    #     instead of Collections, then when you want to load it, XXXX
+    #
+    #     Parameters
+    #     -----------
+    #     data : serialized data.
+    #     filename : string, GridFS Buckets.
+    #
+    #     References
+    #     -----------
+    #     - MongoDB find, xxxxx
+    #     """
+    #     s = time.time()
+    #     f_id = self.datafs.put(data, filename=filename)
+    #     print("[TensorDB] save_bulk_data: {} took: {}s".format(filename, round(time.time()-s, 2)))
+    #     return f_id
+    #
+    # def save_collection(self, data=None, collect_name='collect_name'):
+    #     """ Insert data into MongoDB Collections, return xx.
+    #
+    #     Parameters
+    #     -----------
+    #     data : serialized data.
+    #     collect_name : string, MongoDB collection name.
+    #
+    #     References
+    #     -----------
+    #     - MongoDB find, xxxxx
+    #     """
+    #     s = time.time()
+    #     rl = self.db[collect_name].insert_many(data)
+    #     print("[TensorDB] save_collection: {} took: {}s".format(collect_name, round(time.time()-s, 2)))
+    #     return rl
+    #
+    # def find(self, args={}, collect_name='collect_name'):
+    #     """ Find data from MongoDB Collections.
+    #
+    #     Parameters
+    #     -----------
+    #     args : dictionary, arguments for finding.
+    #     collect_name : string, MongoDB collection name.
+    #
+    #     References
+    #     -----------
+    #     - MongoDB find, xxxxx
+    #     """
+    #     s = time.time()
+    #
+    #     pc = self.db[collect_name].find(args)  # pymongo.cursor.Cursor object
+    #     flist = pc.distinct('f_id')
+    #     fldict = {}
+    #     for f in flist: # you may have multiple Buckets files
+    #         # fldict[f] = pickle.loads(self.datafs.get(f).read())
+    #         # s2 = time.time()
+    #         tmp = self.datafs.get(f).read()
+    #         # print(time.time()-s2)
+    #         fldict[f] = pickle.loads(tmp)
+    #         # print(time.time()-s2)
+    #         # exit()
+    #     # print(round(time.time()-s, 2))
+    #     data = [fldict[x['f_id']][x['id']] for x in pc]
+    #     data = np.asarray(data)
+    #     print("[TensorDB] find: {} get: {} took: {}s".format(collect_name, pc.count(), round(time.time()-s, 2)))
+    #     return data
+
+
+
+class DBLogger:
+    """ """
+    def __init__(self,db,model):
+        self.db=db
+        self.model=model
+
+    def on_train_begin(self,logs={}):
+        print("start")
+
+    def on_train_end(self,logs={}):
+        print("end")
+
+    def on_epoch_begin(self,epoch,logs={}):
+        self.epoch=epoch
+        self.et=time.time()
+        return
+
+    def on_epoch_end(self, epoch, logs={}):
+        self.et=time.time()-self.et
+        print("ending")
+        print(epoch)
+        logs['epoch']=epoch
+        logs['time']=datetime.utcnow()
+        logs['stepTime']=self.et
+        logs['acc']=np.asscalar(logs['acc'])
+        print(logs)
+
+        w=self.model.Params
+        fid=self.db.save_params(w,logs)
+        logs.update({'params':fid})
+        self.db.valid_log(logs)
+    def on_batch_begin(self, batch,logs={}):
+        self.t=time.time()
+        self.losses = []
+        self.batch=batch
+
+    def on_batch_end(self, batch, logs={}):
+        self.t2=time.time()-self.t
+        logs['acc']=np.asscalar(logs['acc'])
+        #logs['loss']=np.asscalar(logs['loss'])
+        logs['step_time']=self.t2
+        logs['time']=datetime.utcnow()
+        logs['epoch']=self.epoch
+        logs['batch']=self.batch
+        self.db.train_log(logs)

tensorlayer/iterate.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+
+
+
+import numpy as np
+from six.moves import xrange
+
+def minibatches(inputs=None, targets=None, batch_size=None, shuffle=False):
+    """Generate a generator that input a group of example in numpy.array and
+    their labels, return the examples and labels by the given batchsize.
+
+    Parameters
+    ----------
+    inputs : numpy.array
+        (X) The input features, every row is a example.
+    targets : numpy.array
+        (y) The labels of inputs, every row is a example.
+    batch_size : int
+        The batch size.
+    shuffle : boolean
+        Indicating whether to use a shuffling queue, shuffle the dataset before return.
+
+    Hints
+    -------
+    - If you have two inputs, e.g. X1 (1000, 100) and X2 (1000, 80), you can ``np.hstack((X1, X2))
+    into (1000, 180) and feed into ``inputs``, then you can split a batch of X1 and X2.
+
+    Examples
+    --------
+    >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+    >>> y = np.asarray([0,1,2,3,4,5])
+    >>> for batch in tl.iterate.minibatches(inputs=X, targets=y, batch_size=2, shuffle=False):
+    >>>     print(batch)
+    ... (array([['a', 'a'],
+    ...        ['b', 'b']],
+    ...         dtype='<U1'), array([0, 1]))
+    ... (array([['c', 'c'],
+    ...        ['d', 'd']],
+    ...         dtype='<U1'), array([2, 3]))
+    ... (array([['e', 'e'],
+    ...        ['f', 'f']],
+    ...         dtype='<U1'), array([4, 5]))
+    """
+    assert len(inputs) == len(targets)
+    if shuffle:
+        indices = np.arange(len(inputs))
+        np.random.shuffle(indices)
+    for start_idx in range(0, len(inputs) - batch_size + 1, batch_size):
+        if shuffle:
+            excerpt = indices[start_idx:start_idx + batch_size]
+        else:
+            excerpt = slice(start_idx, start_idx + batch_size)
+        yield inputs[excerpt], targets[excerpt]
+
+def seq_minibatches(inputs, targets, batch_size, seq_length, stride=1):
+    """Generate a generator that return a batch of sequence inputs and targets.
+    If ``batch_size = 100, seq_length = 5``, one return will have ``500`` rows (examples).
+
+    Examples
+    --------
+    - Synced sequence input and output.
+    >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+    >>> y = np.asarray([0, 1, 2, 3, 4, 5])
+    >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=y, batch_size=2, seq_length=2, stride=1):
+    >>>     print(batch)
+    ... (array([['a', 'a'],
+    ...        ['b', 'b'],
+    ...         ['b', 'b'],
+    ...         ['c', 'c']],
+    ...         dtype='<U1'), array([0, 1, 1, 2]))
+    ... (array([['c', 'c'],
+    ...         ['d', 'd'],
+    ...         ['d', 'd'],
+    ...         ['e', 'e']],
+    ...         dtype='<U1'), array([2, 3, 3, 4]))
+    ...
+    ...
+
+    - Many to One
+    >>> return_last = True
+    >>> num_steps = 2
+    >>> X = np.asarray([['a','a'], ['b','b'], ['c','c'], ['d','d'], ['e','e'], ['f','f']])
+    >>> Y = np.asarray([0,1,2,3,4,5])
+    >>> for batch in tl.iterate.seq_minibatches(inputs=X, targets=Y, batch_size=2, seq_length=num_steps, stride=1):
+    >>>     x, y = batch
+    >>>     if return_last:
+    >>>         tmp_y = y.reshape((-1, num_steps) + y.shape[1:])
+    >>>     y = tmp_y[:, -1]
+    >>>     print(x, y)
+    ... [['a' 'a']
+    ... ['b' 'b']
+    ... ['b' 'b']
+    ... ['c' 'c']] [1 2]
+    ... [['c' 'c']
+    ... ['d' 'd']
+    ... ['d' 'd']
+    ... ['e' 'e']] [3 4]
+    """
+    assert len(inputs) == len(targets)
+    n_loads = (batch_size * stride) + (seq_length - stride)
+    for start_idx in range(0, len(inputs) - n_loads + 1, (batch_size * stride)):
+        seq_inputs = np.zeros((batch_size, seq_length) + inputs.shape[1:],
+                              dtype=inputs.dtype)
+        seq_targets = np.zeros((batch_size, seq_length) + targets.shape[1:],
+                               dtype=targets.dtype)
+        for b_idx in xrange(batch_size):
+            start_seq_idx = start_idx + (b_idx * stride)
+            end_seq_idx = start_seq_idx + seq_length
+            seq_inputs[b_idx] = inputs[start_seq_idx:end_seq_idx]
+            seq_targets[b_idx] = targets[start_seq_idx:end_seq_idx]
+        flatten_inputs = seq_inputs.reshape((-1,) + inputs.shape[1:])
+        flatten_targets = seq_targets.reshape((-1,) + targets.shape[1:])
+        yield flatten_inputs, flatten_targets
+
+def seq_minibatches2(inputs, targets, batch_size, num_steps):
+    """Generate a generator that iterates on two list of words. Yields (Returns) the source contexts and
+    the target context by the given batch_size and num_steps (sequence_length),
+    see ``PTB tutorial``. In TensorFlow's tutorial, this generates the batch_size pointers into the raw
+    PTB data, and allows minibatch iteration along these pointers.
+
+    - Hint, if the input data are images, you can modify the code as follow.
+
+    .. code-block:: python
+
+        from
+        data = np.zeros([batch_size, batch_len)
+        to
+        data = np.zeros([batch_size, batch_len, inputs.shape[1], inputs.shape[2], inputs.shape[3]])
+
+    Parameters
+    ----------
+    inputs : a list
+            the context in list format; note that context usually be
+            represented by splitting by space, and then convert to unique
+            word IDs.
+    targets : a list
+            the context in list format; note that context usually be
+            represented by splitting by space, and then convert to unique
+            word IDs.
+    batch_size : int
+            the batch size.
+    num_steps : int
+            the number of unrolls. i.e. sequence_length
+
+    Yields
+    ------
+    Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
+
+    Raises
+    ------
+    ValueError : if batch_size or num_steps are too high.
+
+    Examples
+    --------
+    >>> X = [i for i in range(20)]
+    >>> Y = [i for i in range(20,40)]
+    >>> for batch in tl.iterate.seq_minibatches2(X, Y, batch_size=2, num_steps=3):
+    ...     x, y = batch
+    ...     print(x, y)
+    ...
+    ... [[  0.   1.   2.]
+    ... [ 10.  11.  12.]]
+    ... [[ 20.  21.  22.]
+    ... [ 30.  31.  32.]]
+    ...
+    ... [[  3.   4.   5.]
+    ... [ 13.  14.  15.]]
+    ... [[ 23.  24.  25.]
+    ... [ 33.  34.  35.]]
+    ...
+    ... [[  6.   7.   8.]
+    ... [ 16.  17.  18.]]
+    ... [[ 26.  27.  28.]
+    ... [ 36.  37.  38.]]
+
+    Code References
+    ---------------
+    - ``tensorflow/models/rnn/ptb/reader.py``
+    """
+    assert len(inputs) == len(targets)
+    data_len = len(inputs)
+    batch_len = data_len // batch_size
+    # data = np.zeros([batch_size, batch_len])
+    data = np.zeros((batch_size, batch_len) + inputs.shape[1:],
+                          dtype=inputs.dtype)
+    data2 = np.zeros([batch_size, batch_len])
+
+    for i in range(batch_size):
+        data[i] = inputs[batch_len * i:batch_len * (i + 1)]
+        data2[i] = targets[batch_len * i:batch_len * (i + 1)]
+
+    epoch_size = (batch_len - 1) // num_steps
+
+    if epoch_size == 0:
+        raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
+
+    for i in range(epoch_size):
+        x = data[:, i*num_steps:(i+1)*num_steps]
+        x2 = data2[:, i*num_steps:(i+1)*num_steps]
+        yield (x, x2)
+
+
+def ptb_iterator(raw_data, batch_size, num_steps):
+    """
+    Generate a generator that iterates on a list of words, see PTB tutorial. Yields (Returns) the source contexts and
+    the target context by the given batch_size and num_steps (sequence_length).\n
+    see ``PTB tutorial``.
+
+    e.g. x = [0, 1, 2]  y = [1, 2, 3] , when batch_size = 1, num_steps = 3,
+    raw_data = [i for i in range(100)]
+
+    In TensorFlow's tutorial, this generates batch_size pointers into the raw
+    PTB data, and allows minibatch iteration along these pointers.
+
+    Parameters
+    ----------
+    raw_data : a list
+            the context in list format; note that context usually be
+            represented by splitting by space, and then convert to unique
+            word IDs.
+    batch_size : int
+            the batch size.
+    num_steps : int
+            the number of unrolls. i.e. sequence_length
+
+    Yields
+    ------
+    Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
+    The second element of the tuple is the same data time-shifted to the
+    right by one.
+
+    Raises
+    ------
+    ValueError : if batch_size or num_steps are too high.
+
+    Examples
+    --------
+    >>> train_data = [i for i in range(20)]
+    >>> for batch in tl.iterate.ptb_iterator(train_data, batch_size=2, num_steps=3):
+    >>>     x, y = batch
+    >>>     print(x, y)
+    ... [[ 0  1  2] <---x                       1st subset/ iteration
+    ...  [10 11 12]]
+    ... [[ 1  2  3] <---y
+    ...  [11 12 13]]
+    ...
+    ... [[ 3  4  5]  <--- 1st batch input       2nd subset/ iteration
+    ...  [13 14 15]] <--- 2nd batch input
+    ... [[ 4  5  6]  <--- 1st batch target
+    ...  [14 15 16]] <--- 2nd batch target
+    ...
+    ... [[ 6  7  8]                             3rd subset/ iteration
+    ...  [16 17 18]]
+    ... [[ 7  8  9]
+    ...  [17 18 19]]
+
+    Code References
+    ----------------
+    - ``tensorflow/models/rnn/ptb/reader.py``
+    """
+    raw_data = np.array(raw_data, dtype=np.int32)
+
+    data_len = len(raw_data)
+    batch_len = data_len // batch_size
+    data = np.zeros([batch_size, batch_len], dtype=np.int32)
+    for i in range(batch_size):
+        data[i] = raw_data[batch_len * i:batch_len * (i + 1)]
+
+    epoch_size = (batch_len - 1) // num_steps
+
+    if epoch_size == 0:
+        raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
+
+    for i in range(epoch_size):
+        x = data[:, i*num_steps:(i+1)*num_steps]
+        y = data[:, i*num_steps+1:(i+1)*num_steps+1]
+        yield (x, y)
+
+
+
+# def minibatches_for_sequence2D(inputs, targets, batch_size, sequence_length, stride=1):
+#     """
+#     Input a group of example in 2D numpy.array and their labels.
+#     Return the examples and labels by the given batchsize, sequence_length.
+#     Use for RNN.
+#
+#     Parameters
+#     ----------
+#     inputs : numpy.array
+#         (X) The input features, every row is a example.
+#     targets : numpy.array
+#         (y) The labels of inputs, every row is a example.
+#     batchsize : int
+#         The batch size must be a multiple of sequence_length: int(batch_size % sequence_length) == 0
+#     sequence_length : int
+#         The sequence length
+#     stride : int
+#         The stride step
+#
+#     Examples
+#     --------
+#     >>> sequence_length = 2
+#     >>> batch_size = 4
+#     >>> stride = 1
+#     >>> X_train = np.asarray([[1,2,3],[4,5,6],[7,8,9],[10,11,12],[13,14,15],[16,17,18],[19,20,21],[22,23,24]])
+#     >>> y_train = np.asarray(['0','1','2','3','4','5','6','7'])
+#     >>> print('X_train = %s' % X_train)
+#     >>> print('y_train = %s' % y_train)
+#     >>> for batch in minibatches_for_sequence2D(X_train, y_train, batch_size=batch_size, sequence_length=sequence_length, stride=stride):
+#     >>>     inputs, targets = batch
+#     >>>     print(inputs)
+#     >>>     print(targets)
+#     ... [[ 1.  2.  3.]
+#     ... [ 4.  5.  6.]
+#     ... [ 4.  5.  6.]
+#     ... [ 7.  8.  9.]]
+#     ... [1 2]
+#     ... [[  4.   5.   6.]
+#     ... [  7.   8.   9.]
+#     ... [  7.   8.   9.]
+#     ... [ 10.  11.  12.]]
+#     ... [2 3]
+#     ... ...
+#     ... [[ 16.  17.  18.]
+#     ... [ 19.  20.  21.]
+#     ... [ 19.  20.  21.]
+#     ... [ 22.  23.  24.]]
+#     ... [6 7]
+#     """
+#     print('len(targets)=%d batch_size=%d sequence_length=%d stride=%d' % (len(targets), batch_size, sequence_length, stride))
+#     assert len(inputs) == len(targets), '1 feature vector have 1 target vector/value' #* sequence_length
+#     # assert int(batch_size % sequence_length) == 0, 'batch_size % sequence_length must == 0\
+#     # batch_size is number of examples rather than number of targets'
+#
+#     # print(inputs.shape, len(inputs), len(inputs[0]))
+#
+#     n_targets = int(batch_size/sequence_length)
+#     # n_targets = int(np.ceil(batch_size/sequence_length))
+#     X = np.empty(shape=(0,len(inputs[0])), dtype=np.float32)
+#     y = np.zeros(shape=(1, n_targets), dtype=np.int32)
+#
+#     for idx in range(sequence_length, len(inputs), stride):  # go through all example during 1 epoch
+#         for n in range(n_targets):   # for num of target
+#             X = np.concatenate((X, inputs[idx-sequence_length+n:idx+n]))
+#             y[0][n] = targets[idx-1+n]
+#             # y = np.vstack((y, targets[idx-1+n]))
+#         yield X, y[0]
+#         X = np.empty(shape=(0,len(inputs[0])))
+#         # y = np.empty(shape=(1,0))
+#
+#
+# def minibatches_for_sequence4D(inputs, targets, batch_size, sequence_length, stride=1): #
+#     """
+#     Input a group of example in 4D numpy.array and their labels.
+#     Return the examples and labels by the given batchsize, sequence_length.
+#     Use for RNN.
+#
+#     Parameters
+#     ----------
+#     inputs : numpy.array
+#         (X) The input features, every row is a example.
+#     targets : numpy.array
+#         (y) The labels of inputs, every row is a example.
+#     batchsize : int
+#         The batch size must be a multiple of sequence_length: int(batch_size % sequence_length) == 0
+#     sequence_length : int
+#         The sequence length
+#     stride : int
+#         The stride step
+#
+#     Examples
+#     --------
+#     >>> sequence_length = 2
+#     >>> batch_size = 2
+#     >>> stride = 1
+#     >>> X_train = np.asarray([[1,2,3],[4,5,6],[7,8,9],[10,11,12],[13,14,15],[16,17,18],[19,20,21],[22,23,24]])
+#     >>> y_train = np.asarray(['0','1','2','3','4','5','6','7'])
+#     >>> X_train = np.expand_dims(X_train, axis=1)
+#     >>> X_train = np.expand_dims(X_train, axis=3)
+#     >>> for batch in minibatches_for_sequence4D(X_train, y_train, batch_size=batch_size, sequence_length=sequence_length, stride=stride):
+#     >>>     inputs, targets = batch
+#     >>>     print(inputs)
+#     >>>     print(targets)
+#     ... [[[[ 1.]
+#     ...    [ 2.]
+#     ...    [ 3.]]]
+#     ... [[[ 4.]
+#     ...   [ 5.]
+#     ...   [ 6.]]]]
+#     ... [1]
+#     ... [[[[ 4.]
+#     ...    [ 5.]
+#     ...    [ 6.]]]
+#     ... [[[ 7.]
+#     ...   [ 8.]
+#     ...   [ 9.]]]]
+#     ... [2]
+#     ... ...
+#     ... [[[[ 19.]
+#     ...    [ 20.]
+#     ...    [ 21.]]]
+#     ... [[[ 22.]
+#     ...   [ 23.]
+#     ...   [ 24.]]]]
+#     ... [7]
+#     """
+#     print('len(targets)=%d batch_size=%d sequence_length=%d stride=%d' % (len(targets), batch_size, sequence_length, stride))
+#     assert len(inputs) == len(targets), '1 feature vector have 1 target vector/value' #* sequence_length
+#     # assert int(batch_size % sequence_length) == 0, 'in LSTM, batch_size % sequence_length must == 0\
+#     # batch_size is number of X_train rather than number of targets'
+#     assert stride >= 1, 'stride must be >=1, at least move 1 step for each iternation'
+#
+#     n_example, n_channels, width, height = inputs.shape
+#     print('n_example=%d n_channels=%d width=%d height=%d' % (n_example, n_channels, width, height))
+#
+#     n_targets = int(np.ceil(batch_size/sequence_length)) # 实际为 batchsize/sequence_length + 1
+#     print(n_targets)
+#     X = np.zeros(shape=(batch_size, n_channels, width, height), dtype=np.float32)
+#     # X = np.zeros(shape=(n_targets, sequence_length, n_channels, width, height), dtype=np.float32)
+#     y = np.zeros(shape=(1,n_targets), dtype=np.int32)
+#     # y = np.empty(shape=(0,1), dtype=np.float32)
+#     # time.sleep(2)
+#     for idx in range(sequence_length, n_example-n_targets+2, stride):  # go through all example during 1 epoch
+#         for n in range(n_targets):   # for num of target
+#             # print(idx+n, inputs[idx-sequence_length+n : idx+n].shape)
+#             X[n*sequence_length : (n+1)*sequence_length] = inputs[idx+n-sequence_length : idx+n]
+#             # X[n] = inputs[idx-sequence_length+n:idx+n]
+#             y[0][n] = targets[idx+n-1]
+#             # y = np.vstack((y, targets[idx-1+n]))
+#         # y = targets[idx: idx+n_targets]
+#         yield X, y[0]

tensorlayer/ops.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+
+
+
+
+import tensorflow as tf
+import os
+import sys
+from sys import platform as _platform
+
+
+def exit_tf(sess=None):
+    """Close tensorboard and nvidia-process if available
+
+    Parameters
+    ----------
+    sess : a session instance of TensorFlow
+        TensorFlow session
+    """
+    text = "[tl] Close tensorboard and nvidia-process if available"
+    sess.close()
+    # import time
+    # time.sleep(2)
+    if _platform == "linux" or _platform == "linux2":
+        print('linux: %s' % text)
+        os.system('nvidia-smi')
+        os.system('fuser 6006/tcp -k')  # kill tensorboard 6006
+        os.system("nvidia-smi | grep python |awk '{print $3}'|xargs kill") # kill all nvidia-smi python process
+    elif _platform == "darwin":
+        print('OS X: %s' % text)
+        os.system("lsof -i tcp:6006 | grep -v PID | awk '{print $2}' | xargs kill") # kill tensorboard 6006
+    elif _platform == "win32":
+        print('Windows: %s' % text)
+    else:
+        print(_platform)
+    exit()
+
+def clear_all(printable=True):
+    """Clears all the placeholder variables of keep prob,
+    including keeping probabilities of all dropout, denoising, dropconnect etc.
+
+    Parameters
+    ----------
+    printable : boolean
+        If True, print all deleted variables.
+    """
+    print('clear all .....................................')
+    gl = globals().copy()
+    for var in gl:
+        if var[0] == '_': continue
+        if 'func' in str(globals()[var]): continue
+        if 'module' in str(globals()[var]): continue
+        if 'class' in str(globals()[var]): continue
+
+        if printable:
+            print(" clear_all ------- %s" % str(globals()[var]))
+
+        del globals()[var]
+
+# def clear_all2(vars, printable=True):
+#     """
+#     The :function:`clear_all()` Clears all the placeholder variables of keep prob,
+#     including keeping probabilities of all dropout, denoising, dropconnect
+#     Parameters
+#     ----------
+#     printable : if True, print all deleted variables.
+#     """
+#     print('clear all .....................................')
+#     for var in vars:
+#         if var[0] == '_': continue
+#         if 'func' in str(var): continue
+#         if 'module' in str(var): continue
+#         if 'class' in str(var): continue
+#
+#         if printable:
+#             print(" clear_all ------- %s" % str(var))
+#
+#         del var
+
+def set_gpu_fraction(sess=None, gpu_fraction=0.3):
+    """Set the GPU memory fraction for the application.
+
+    Parameters
+    ----------
+    sess : a session instance of TensorFlow
+        TensorFlow session
+    gpu_fraction : a float
+        Fraction of GPU memory, (0 ~ 1]
+
+    References
+    ----------
+    - `TensorFlow using GPU <https://www.tensorflow.org/versions/r0.9/how_tos/using_gpu/index.html>`_
+    """
+    print("  tensorlayer: GPU MEM Fraction %f" % gpu_fraction)
+    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)
+    sess = tf.Session(config = tf.ConfigProto(gpu_options = gpu_options))
+    return sess
+
+
+
+
+
+def disable_print():
+    """Disable console output, ``suppress_stdout`` is recommended.
+
+    Examples
+    ---------
+    >>> print("You can see me")
+    >>> tl.ops.disable_print()
+    >>> print(" You can't see me")
+    >>> tl.ops.enable_print()
+    >>> print("You can see me")
+    """
+    # sys.stdout = os.devnull   # this one kill the process
+    sys.stdout = None
+    sys.stderr = os.devnull
+
+def enable_print():
+    """Enable console output, ``suppress_stdout`` is recommended.
+
+    Examples
+    --------
+    - see tl.ops.disable_print()
+    """
+    sys.stdout = sys.__stdout__
+    sys.stderr = sys.__stderr__
+
+
+# class temporary_disable_print:
+#     """Temporarily disable console output.
+#
+#     Examples
+#     ---------
+#     >>> print("You can see me")
+#     >>> with tl.ops.temporary_disable_print() as t:
+#     >>>     print("You can't see me")
+#     >>> print("You can see me")
+#     """
+#     def __init__(self):
+#         pass
+#     def __enter__(self):
+#         sys.stdout = None
+#         sys.stderr = os.devnull
+#     def __exit__(self, type, value, traceback):
+#         sys.stdout = sys.__stdout__
+#         sys.stderr = sys.__stderr__
+#         return isinstance(value, TypeError)
+
+
+from contextlib import contextmanager
+@contextmanager
+def suppress_stdout():
+    """Temporarily disable console output.
+
+    Examples
+    ---------
+    >>> print("You can see me")
+    >>> with tl.ops.suppress_stdout():
+    >>>     print("You can't see me")
+    >>> print("You can see me")
+
+    References
+    -----------
+    - `stackoverflow <http://stackoverflow.com/questions/2125702/how-to-suppress-console-output-in-python>`_
+    """
+    with open(os.devnull, "w") as devnull:
+        old_stdout = sys.stdout
+        sys.stdout = devnull
+        try:
+            yield
+        finally:
+            sys.stdout = old_stdout
+
+
+
+def get_site_packages_directory():
+    """Print and return the site-packages directory.
+
+    Examples
+    ---------
+    >>> loc = tl.ops.get_site_packages_directory()
+    """
+    import site
+    try:
+        loc = site.getsitepackages()
+        print("  tl.ops : site-packages in ", loc)
+        return loc
+    except:
+        print("  tl.ops : Cannot find package dir from virtual environment")
+        return False
+
+
+
+def empty_trash():
+    """Empty trash folder.
+
+    """
+    text = "[tl] Empty the trash"
+    if _platform == "linux" or _platform == "linux2":
+        print('linux: %s' % text)
+        os.system("rm -rf ~/.local/share/Trash/*")
+    elif _platform == "darwin":
+        print('OS X: %s' % text)
+        os.system("sudo rm -rf ~/.Trash/*")
+    elif _platform == "win32":
+        print('Windows: %s' % text)
+        try:
+            os.system("rd /s c:\$Recycle.Bin")  # Windows 7 or Server 2008
+        except:
+            pass
+        try:
+            os.system("rd /s c:\recycler")  #  Windows XP, Vista, or Server 2003
+        except:
+            pass
+    else:
+        print(_platform)
+
+#

tensorlayer/rein.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+
+
+
+import tensorflow as tf
+import numpy as np
+from six.moves import xrange
+
+def discount_episode_rewards(rewards=[], gamma=0.99, mode=0):
+    """ Take 1D float array of rewards and compute discounted rewards for an
+    episode. When encount a non-zero value, consider as the end a of an episode.
+
+    Parameters
+    ----------
+    rewards : numpy list
+        a list of rewards
+    gamma : float
+        discounted factor
+    mode : int
+        if mode == 0, reset the discount process when encount a non-zero reward (Ping-pong game).
+        if mode == 1, would not reset the discount process.
+
+    Examples
+    ----------
+    >>> rewards = np.asarray([0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1])
+    >>> gamma = 0.9
+    >>> discount_rewards = tl.rein.discount_episode_rewards(rewards, gamma)
+    >>> print(discount_rewards)
+    ... [ 0.72899997  0.81        0.89999998  1.          0.72899997  0.81
+    ... 0.89999998  1.          0.72899997  0.81        0.89999998  1.        ]
+    >>> discount_rewards = tl.rein.discount_episode_rewards(rewards, gamma, mode=1)
+    >>> print(discount_rewards)
+    ... [ 1.52110755  1.69011939  1.87791049  2.08656716  1.20729685  1.34144104
+    ... 1.49048996  1.65610003  0.72899997  0.81        0.89999998  1.        ]
+    """
+    discounted_r = np.zeros_like(rewards, dtype=np.float32)
+    running_add = 0
+    for t in reversed(xrange(0, rewards.size)):
+        if mode == 0:
+            if rewards[t] != 0: running_add = 0
+
+        running_add = running_add * gamma + rewards[t]
+        discounted_r[t] = running_add
+    return discounted_r
+
+
+def cross_entropy_reward_loss(logits, actions, rewards, name=None):
+    """ Calculate the loss for Policy Gradient Network.
+
+    Parameters
+    ----------
+    logits : tensor
+        The network outputs without softmax. This function implements softmax
+        inside.
+    actions : tensor/ placeholder
+        The agent actions.
+    rewards : tensor/ placeholder
+        The rewards.
+
+    Examples
+    ----------
+    >>> states_batch_pl = tf.placeholder(tf.float32, shape=[None, D])   # observation for training
+    >>> network = tl.layers.InputLayer(states_batch_pl, name='input_layer')
+    >>> network = tl.layers.DenseLayer(network, n_units=H, act = tf.nn.relu, name='relu1')
+    >>> network = tl.layers.DenseLayer(network, n_units=3, act = tl.activation.identity, name='output_layer')
+    >>> probs = network.outputs
+    >>> sampling_prob = tf.nn.softmax(probs)
+    >>> actions_batch_pl = tf.placeholder(tf.int32, shape=[None])
+    >>> discount_rewards_batch_pl = tf.placeholder(tf.float32, shape=[None])
+    >>> loss = cross_entropy_reward_loss(probs, actions_batch_pl, discount_rewards_batch_pl)
+    >>> train_op = tf.train.RMSPropOptimizer(learning_rate, decay_rate).minimize(loss)
+    """
+
+    try: # TF 1.0
+        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=actions, logits=logits, name=name)
+    except:
+        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, targets=actions)
+        # cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, actions)
+
+    try: ## TF1.0
+        loss = tf.reduce_sum(tf.multiply(cross_entropy, rewards))
+    except: ## TF0.12
+        loss = tf.reduce_sum(tf.mul(cross_entropy, rewards))   # element-wise mul
+    return loss

tensorlayer/visualize.py

+#! /usr/bin/python
+# -*- coding: utf8 -*-
+
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+# import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+
+## Save images
+import scipy.misc
+
+def save_image(image, image_path):
+    """Save one image.
+
+    Parameters
+    -----------
+    images : numpy array [w, h, c]
+    image_path : string.
+    """
+    scipy.misc.imsave(image_path, image)
+
+def save_images(images, size, image_path):
+    """Save mutiple images into one single image.
+
+    Parameters
+    -----------
+    images : numpy array [batch, w, h, c]
+    size : list of two int, row and column number.
+        number of images should be equal or less than size[0] * size[1]
+    image_path : string.
+
+    Examples
+    ---------
+    >>> images = np.random.rand(64, 100, 100, 3)
+    >>> tl.visualize.save_images(images, [8, 8], 'temp.png')
+    """
+    def merge(images, size):
+        h, w = images.shape[1], images.shape[2]
+        img = np.zeros((h * size[0], w * size[1], 3))
+        for idx, image in enumerate(images):
+            i = idx % size[1]
+            j = idx // size[1]
+            img[j*h:j*h+h, i*w:i*w+w, :] = image
+        return img
+
+    def imsave(images, size, path):
+        return scipy.misc.imsave(path, merge(images, size))
+
+    assert len(images) <= size[0] * size[1], "number of images should be equal or less than size[0] * size[1] {}".format(len(images))
+    return imsave(images, size, image_path)
+
+def W(W=None, second=10, saveable=True, shape=[28,28], name='mnist', fig_idx=2396512):
+    """Visualize every columns of the weight matrix to a group of Greyscale img.
+
+    Parameters
+    ----------
+    W : numpy.array
+        The weight matrix
+    second : int
+        The display second(s) for the image(s), if saveable is False.
+    saveable : boolean
+        Save or plot the figure.
+    shape : a list with 2 int
+        The shape of feature image, MNIST is [28, 80].
+    name : a string
+        A name to save the image, if saveable is True.
+    fig_idx : int
+        matplotlib figure index.
+
+    Examples
+    --------
+    >>> tl.visualize.W(network.all_params[0].eval(), second=10, saveable=True, name='weight_of_1st_layer', fig_idx=2012)
+    """
+    if saveable is False:
+        plt.ion()
+    fig = plt.figure(fig_idx)      # show all feature images
+    size = W.shape[0]
+    n_units = W.shape[1]
+
+    num_r = int(np.sqrt(n_units))  # 每行显示的个数   若25个hidden unit -> 每行显示5个
+    num_c = int(np.ceil(n_units/num_r))
+    count = int(1)
+    for row in range(1, num_r+1):
+        for col in range(1, num_c+1):
+            if count > n_units:
+                break
+            a = fig.add_subplot(num_r, num_c, count)
+            # ------------------------------------------------------------
+            # plt.imshow(np.reshape(W[:,count-1],(28,28)), cmap='gray')
+            # ------------------------------------------------------------
+            feature = W[:,count-1] / np.sqrt( (W[:,count-1]**2).sum())
+            # feature[feature<0.0001] = 0   # value threshold
+            # if count == 1 or count == 2:
+            #     print(np.mean(feature))
+            # if np.std(feature) < 0.03:      # condition threshold
+            #     feature = np.zeros_like(feature)
+            # if np.mean(feature) < -0.015:      # condition threshold
+            #     feature = np.zeros_like(feature)
+            plt.imshow(np.reshape(feature ,(shape[0],shape[1])),
+                    cmap='gray', interpolation="nearest")#, vmin=np.min(feature), vmax=np.max(feature))
+            # plt.title(name)
+            # ------------------------------------------------------------
+            # plt.imshow(np.reshape(W[:,count-1] ,(np.sqrt(size),np.sqrt(size))), cmap='gray', interpolation="nearest")
+            plt.gca().xaxis.set_major_locator(plt.NullLocator())    # distable tick
+            plt.gca().yaxis.set_major_locator(plt.NullLocator())
+            count = count + 1
+    if saveable:
+        plt.savefig(name+'.pdf',format='pdf')
+    else:
+        plt.draw()
+        plt.pause(second)
+
+def frame(I=None, second=5, saveable=True, name='frame', cmap=None, fig_idx=12836):
+    """Display a frame(image). Make sure OpenAI Gym render() is disable before using it.
+
+    Parameters
+    ----------
+    I : numpy.array
+        The image
+    second : int
+        The display second(s) for the image(s), if saveable is False.
+    saveable : boolean
+        Save or plot the figure.
+    name : a string
+        A name to save the image, if saveable is True.
+    cmap : None or string
+        'gray' for greyscale, None for default, etc.
+    fig_idx : int
+        matplotlib figure index.
+
+    Examples
+    --------
+    >>> env = gym.make("Pong-v0")
+    >>> observation = env.reset()
+    >>> tl.visualize.frame(observation)
+    """
+    if saveable is False:
+        plt.ion()
+    fig = plt.figure(fig_idx)      # show all feature images
+
+    if len(I.shape) and I.shape[-1]==1:     # (10,10,1) --> (10,10)
+        I = I[:,:,0]
+
+    plt.imshow(I, cmap)
+    plt.title(name)
+    # plt.gca().xaxis.set_major_locator(plt.NullLocator())    # distable tick
+    # plt.gca().yaxis.set_major_locator(plt.NullLocator())
+
+    if saveable:
+        plt.savefig(name+'.pdf',format='pdf')
+    else:
+        plt.draw()
+        plt.pause(second)
+
+def CNN2d(CNN=None, second=10, saveable=True, name='cnn', fig_idx=3119362):
+    """Display a group of RGB or Greyscale CNN masks.
+
+    Parameters
+    ----------
+    CNN : numpy.array
+        The image. e.g: 64 5x5 RGB images can be (5, 5, 3, 64).
+    second : int
+        The display second(s) for the image(s), if saveable is False.
+    saveable : boolean
+        Save or plot the figure.
+    name : a string
+        A name to save the image, if saveable is True.
+    fig_idx : int
+        matplotlib figure index.
+
+    Examples
+    --------
+    >>> tl.visualize.CNN2d(network.all_params[0].eval(), second=10, saveable=True, name='cnn1_mnist', fig_idx=2012)
+    """
+    # print(CNN.shape)    # (5, 5, 3, 64)
+    # exit()
+    n_mask = CNN.shape[3]
+    n_row = CNN.shape[0]
+    n_col = CNN.shape[1]
+    n_color = CNN.shape[2]
+    row = int(np.sqrt(n_mask))
+    col = int(np.ceil(n_mask/row))
+    plt.ion()   # active mode
+    fig = plt.figure(fig_idx)
+    count = 1
+    for ir in range(1, row+1):
+        for ic in range(1, col+1):
+            if count > n_mask:
+                break
+            a = fig.add_subplot(col, row, count)
+            # print(CNN[:,:,:,count-1].shape, n_row, n_col)   # (5, 1, 32) 5 5
+            # exit()
+            # plt.imshow(
+            #         np.reshape(CNN[count-1,:,:,:], (n_row, n_col)),
+            #         cmap='gray', interpolation="nearest")     # theano
+            if n_color == 1:
+                plt.imshow(
+                        np.reshape(CNN[:,:,:,count-1], (n_row, n_col)),
+                        cmap='gray', interpolation="nearest")
+            elif n_color == 3:
+                plt.imshow(
+                        np.reshape(CNN[:,:,:,count-1], (n_row, n_col, n_color)),
+                        cmap='gray', interpolation="nearest")
+            else:
+                raise Exception("Unknown n_color")
+            plt.gca().xaxis.set_major_locator(plt.NullLocator())    # distable tick
+            plt.gca().yaxis.set_major_locator(plt.NullLocator())
+            count = count + 1
+    if saveable:
+        plt.savefig(name+'.pdf',format='pdf')
+    else:
+        plt.draw()
+        plt.pause(second)
+
+
+def images2d(images=None, second=10, saveable=True, name='images', dtype=None,
+                                                            fig_idx=3119362):
+    """Display a group of RGB or Greyscale images.
+
+    Parameters
+    ----------
+    images : numpy.array
+        The images.
+    second : int
+        The display second(s) for the image(s), if saveable is False.
+    saveable : boolean
+        Save or plot the figure.
+    name : a string
+        A name to save the image, if saveable is True.
+    dtype : None or numpy data type
+        The data type for displaying the images.
+    fig_idx : int
+        matplotlib figure index.
+
+    Examples
+    --------
+    >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3), plotable=False)
+    >>> tl.visualize.images2d(X_train[0:100,:,:,:], second=10, saveable=False, name='cifar10', dtype=np.uint8, fig_idx=20212)
+    """
+    # print(images.shape)    # (50000, 32, 32, 3)
+    # exit()
+    if dtype:
+        images = np.asarray(images, dtype=dtype)
+    n_mask = images.shape[0]
+    n_row = images.shape[1]
+    n_col = images.shape[2]
+    n_color = images.shape[3]
+    row = int(np.sqrt(n_mask))
+    col = int(np.ceil(n_mask/row))
+    plt.ion()   # active mode
+    fig = plt.figure(fig_idx)
+    count = 1
+    for ir in range(1, row+1):
+        for ic in range(1, col+1):
+            if count > n_mask:
+                break
+            a = fig.add_subplot(col, row, count)
+            # print(images[:,:,:,count-1].shape, n_row, n_col)   # (5, 1, 32) 5 5
+            # plt.imshow(
+            #         np.reshape(images[count-1,:,:,:], (n_row, n_col)),
+            #         cmap='gray', interpolation="nearest")     # theano
+            if n_color == 1:
+                plt.imshow(
+                        np.reshape(images[count-1,:,:], (n_row, n_col)),
+                        cmap='gray', interpolation="nearest")
+                # plt.title(name)
+            elif n_color == 3:
+                plt.imshow(images[count-1,:,:],
+                        cmap='gray', interpolation="nearest")
+                # plt.title(name)
+            else:
+                raise Exception("Unknown n_color")
+            plt.gca().xaxis.set_major_locator(plt.NullLocator())    # distable tick
+            plt.gca().yaxis.set_major_locator(plt.NullLocator())
+            count = count + 1
+    if saveable:
+        plt.savefig(name+'.pdf',format='pdf')
+    else:
+        plt.draw()
+        plt.pause(second)
+
+def tsne_embedding(embeddings, reverse_dictionary, plot_only=500,
+                        second=5, saveable=False, name='tsne', fig_idx=9862):
+    """Visualize the embeddings by using t-SNE.
+
+    Parameters
+    ----------
+    embeddings : a matrix
+        The images.
+    reverse_dictionary : a dictionary
+        id_to_word, mapping id to unique word.
+    plot_only : int
+        The number of examples to plot, choice the most common words.
+    second : int
+        The display second(s) for the image(s), if saveable is False.
+    saveable : boolean
+        Save or plot the figure.
+    name : a string
+        A name to save the image, if saveable is True.
+    fig_idx : int
+        matplotlib figure index.
+
+    Examples
+    --------
+    >>> see 'tutorial_word2vec_basic.py'
+    >>> final_embeddings = normalized_embeddings.eval()
+    >>> tl.visualize.tsne_embedding(final_embeddings, labels, reverse_dictionary,
+    ...                   plot_only=500, second=5, saveable=False, name='tsne')
+    """
+    def plot_with_labels(low_dim_embs, labels, figsize=(18, 18), second=5,
+                                    saveable=True, name='tsne', fig_idx=9862):
+        assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
+        if saveable is False:
+            plt.ion()
+            plt.figure(fig_idx)
+        plt.figure(figsize=figsize)  #in inches
+        for i, label in enumerate(labels):
+            x, y = low_dim_embs[i,:]
+            plt.scatter(x, y)
+            plt.annotate(label,
+                     xy=(x, y),
+                     xytext=(5, 2),
+                     textcoords='offset points',
+                     ha='right',
+                     va='bottom')
+        if saveable:
+            plt.savefig(name+'.pdf',format='pdf')
+        else:
+            plt.draw()
+            plt.pause(second)
+
+    try:
+        from sklearn.manifold import TSNE
+        import matplotlib.pyplot as plt
+        from six.moves import xrange
+
+        tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
+        # plot_only = 500
+        low_dim_embs = tsne.fit_transform(embeddings[:plot_only,:])
+        labels = [reverse_dictionary[i] for i in xrange(plot_only)]
+        plot_with_labels(low_dim_embs, labels, second=second, saveable=saveable, \
+                                                    name=name, fig_idx=fig_idx)
+    except ImportError:
+        print("Please install sklearn and matplotlib to visualize embeddings.")
+
+
+#

utils.py

+import scipy
+import numpy as np
+
+def get_imgs_fn(file_name):
+	return scipy.misc.imread(file_name, mode='RGB')
+
+def augment_imgs_fn(x, add_noise=True):
+	return x+0.1*x.std()*np.random.random(x.shape)
+
+def normalize_imgs_fn(x):
+    x = x * (2./ 255.) - 1.
+    # x = x * (1./255.)
+    return x
+
+def truncate_imgs_fn(x):
+	x = np.where(x > -1., x, -1.)
+	x = np.where(x < 1., x, 1.)
+	return x
\ No newline at end of file