#       Email:      example@nnthink.com                                                           %

from opfunu.cec_basic.cec2014_nobias import *
from mealpy.bio_based import SMA
from mealpy.problem import Problem
from mealpy.utils.termination import Termination

# Setting parameters

# A - Different way to provide lower bound and upper bound. Here are some examples:

## A1. When you have different lower bound and upper bound for each parameters
problem_dict1 = {
    "obj_func": F5,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
}
problem_obj1 = Problem(problem_dict1)

## A2. When you have same lower bound and upper bound for each parameters, then you can use:
##      + int or float: then you need to specify your problem size / number of dimensions (n_dims)
problem_dict2 = {
    "obj_func": F5,
    "lb": -10,
    "ub": 30,
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj2 = Problem(problem_dict2)

problem_dict3 = {
    "obj_func": F5,
    "lb": [-5],
    "ub": [10],
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj3 = Problem(problem_dict3)

n_dims = 100
problem_dict4 = {
    "obj_func": F5,
    "lb": [-5] * n_dims,
    "ub": [10] * n_dims,
    "minmax": "min",
    "verbose": True,
}

## Run the algorithm

### Your parameter problem can be an instane of Problem class or just dict like above
model1 = SMA.BaseSMA(problem_obj1, epoch=100, pop_size=50, pr=0.03)
model1.solve()

model2 = SMA.BaseSMA(problem_dict4, epoch=100, pop_size=50, pr=0.03)
model2.solve()

# B - Test with different Stopping Condition (Termination) by creating an Termination object

## There are 4 termination cases:
### 1. FE (Number of Function Evaluation)
### 3. ES (Early Stopping): Same idea in training neural network (If the global best solution not better an epsilon
###     after K epoch then stop the program
### 4. TB (Time Bound): You just want your algorithm run in K seconds. Especially when comparing different algorithms.

termination_dict1 = {
    "mode": "FE",
    "quantity": 100000  # 100000 number of function evaluation
}
termination_dict2 = {  # When creating this object, it will override the default epoch you define in your model
    "mode": "MG",
    "quantity": 1000  # 1000 epochs
}
termination_dict3 = {
    "mode": "ES",
    "quantity": 30  # after 30 epochs, if the global best doesn't improve then we stop the program
}
termination_dict4 = {
    "mode": "ES",
    "quantity": 60  # 60 seconds = 1 minute to run this algorithm only
}
termination_obj1 = Termination(termination_dict1)
termination_obj2 = Termination(termination_dict2)
termination_obj3 = Termination(termination_dict3)
termination_obj4 = Termination(termination_dict4)

### Pass your termination object into your model as a addtional parameter with the keyword "termination"
model3 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03, termination=termination_obj1)
model3.solve()
### Remember you can't pass termination dict, it only accept the Termination object

# C - Test with different training mode (sequential, threading parallelization, processing parallelization)

## + thread: create multiple threading depend on your chip
## + process: create multiple cores to run your algorithm.
## Note: For windows, your program need the if __nam__ == "__main__" condition to avoid creating infinite processors

model5 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model5.solve(mode='sequential')  # Default

model6 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model6.solve(mode='thread')

if __name__ == "__main__":
    model7 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
    model7.solve(mode='process')

# D - Drawing all available figures

## There are 8 different figures for each algorithm.
## D.1: Based on fitness value:
##      1. Global best fitness chart
##      2. Local best fitness chart
## D.2: Based on objective value:
##      3. Global objective chart
##      4. Local objective chart
## D.3: Based on runtime value (runtime for each epoch)
## D.4: Based on exploration verse exploration value
##      6. Exploration vs Exploitation chart
## D.5: Based on diversity of population
## D.6: Based on trajectory value (1D, 2D only)

model8 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model8.solve()

## You can access them all via object "history" like this:
model8.history.save_global_objectives_chart(filename="hello/goc")
model8.history.save_local_objectives_chart(filename="hello/loc")
model8.history.save_global_best_fitness_chart(filename="hello/gbfc")
model8.history.save_local_best_fitness_chart(filename="hello/lbfc")
model8.history.save_runtime_chart(filename="hello/rtc")
model8.history.save_exploration_exploitation_chart(filename="hello/eec")
model8.history.save_diversity_chart(filename="hello/dc")
model8.history.save_trajectory_chart(list_agent_idx=[3, 5], list_dimensions=[3], filename="hello/tc")

# E - Handling Multi-Objective function and Constraint Method

## To handling Multi-Objective, mealpy is using weighting method which converting multiple objectives to a single target (fitness value)

## Define your objective function, your constraint
def obj_function(solution):
    t1 = solution[0] ** 2
    t2 = ((2 * solution[1]) / 5) **

    t3 = 0
    for i in range(3, len(solution)):
        t3 += (1 + solution[i] ** 2) ** 0.5
    return [t1, t2, t3]

## Define your objective weights. For example:

problem_dict9 = {
    "obj_func": obj_function,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
    "obj_weight": [0.5, 0.2, 0.3]  # Remember the keyword "obj_weight"
}
problem_obj9 = Problem(problem_dict9)
model9 = SMA.BaseSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model9.solve()

## To access the results, you can get the results by solve() method
position, fitness_value = model9.solve()

## To get all fitness value and all objective values, get it via "solution" attribute
position = model9.solution[0]
fitness_value = model9.solution[1][0]
objective_values = model9.solution[1][1]

# F - Test with different variants of this algorithm

model10 = SMA.OriginalSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model10.solve()


if __name__ == "__main__":
    model7 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
    model7.solve(mode='process')

#       Email:      example@nnthink.com                                                           %

from opfunu.cec_basic.cec2014_nobias import *
from mealpy.bio_based import SMA
from mealpy.problem import Problem
from mealpy.utils.termination import Termination

# Setting parameters

# A - Different way to provide lower bound and upper bound. Here are some examples:

## A1. When you have different lower bound and upper bound for each parameters
problem_dict1 = {
    "obj_func": F5,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
}
problem_obj1 = Problem(problem_dict1)

## A2. When you have same lower bound and upper bound for each parameters, then you can use:
##      + int or float: then you need to specify your problem size / number of dimensions (n_dims)
problem_dict2 = {
    "obj_func": F5,
    "lb": -10,
    "ub": 30,
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj2 = Problem(problem_dict2)

problem_dict3 = {
    "obj_func": F5,
    "lb": [-5],
    "ub": [10],
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj3 = Problem(problem_dict3)

n_dims = 100
problem_dict4 = {
    "obj_func": F5,
    "lb": [-5] * n_dims,
    "ub": [10] * n_dims,
    "minmax": "min",
    "verbose": True,
}

## Run the algorithm

### Your parameter problem can be an instane of Problem class or just dict like above
model1 = SMA.BaseSMA(problem_obj1, epoch=100, pop_size=50, pr=0.03)
model1.solve()

model2 = SMA.BaseSMA(problem_dict4, epoch=100, pop_size=50, pr=0.03)
model2.solve()

# B - Test with different Stopping Condition (Termination) by creating an Termination object

## There are 4 termination cases:
### 1. FE (Number of Function Evaluation)
### 3. ES (Early Stopping): Same idea in training neural network (If the global best solution not better an epsilon
###     after K epoch then stop the program
### 4. TB (Time Bound): You just want your algorithm run in K seconds. Especially when comparing different algorithms.

termination_dict1 = {
    "mode": "FE",
    "quantity": 100000  # 100000 number of function evaluation
}
termination_dict2 = {  # When creating this object, it will override the default epoch you define in your model
    "mode": "MG",
    "quantity": 1000  # 1000 epochs
}
termination_dict3 = {
    "mode": "ES",
    "quantity": 30  # after 30 epochs, if the global best doesn't improve then we stop the program
}
termination_dict4 = {
    "mode": "ES",
    "quantity": 60  # 60 seconds = 1 minute to run this algorithm only
}
termination_obj1 = Termination(termination_dict1)
termination_obj2 = Termination(termination_dict2)
termination_obj3 = Termination(termination_dict3)
termination_obj4 = Termination(termination_dict4)

### Pass your termination object into your model as a addtional parameter with the keyword "termination"
model3 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03, termination=termination_obj1)
model3.solve()
### Remember you can't pass termination dict, it only accept the Termination object

# C - Test with different training mode (sequential, threading parallelization, processing parallelization)

## + thread: create multiple threading depend on your chip
## + process: create multiple cores to run your algorithm.
## Note: For windows, your program need the if __nam__ == "__main__" condition to avoid creating infinite processors

model5 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model5.solve(mode='sequential')  # Default

model6 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model6.solve(mode='thread')

# D - Drawing all available figures

## There are 8 different figures for each algorithm.
## D.1: Based on fitness value:
##      1. Global best fitness chart
##      2. Local best fitness chart
## D.2: Based on objective value:
##      3. Global objective chart
##      4. Local objective chart
## D.3: Based on runtime value (runtime for each epoch)
## D.4: Based on exploration verse exploration value
##      6. Exploration vs Exploitation chart
## D.5: Based on diversity of population
## D.6: Based on trajectory value (1D, 2D only)

model8 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model8.solve()

## You can access them all via object "history" like this:
model8.history.save_global_objectives_chart(filename="hello/goc")
model8.history.save_local_objectives_chart(filename="hello/loc")
model8.history.save_global_best_fitness_chart(filename="hello/gbfc")
model8.history.save_local_best_fitness_chart(filename="hello/lbfc")
model8.history.save_runtime_chart(filename="hello/rtc")
model8.history.save_exploration_exploitation_chart(filename="hello/eec")
model8.history.save_diversity_chart(filename="hello/dc")
model8.history.save_trajectory_chart(list_agent_idx=[3, 5], list_dimensions=[3], filename="hello/tc")

# E - Handling Multi-Objective function and Constraint Method

## To handling Multi-Objective, mealpy is using weighting method which converting multiple objectives to a single target (fitness value)

## Define your objective function, your constraint
def obj_function(solution):
    t1 = solution[0] ** 2
    t2 = ((2 * solution[1]) / 5) ** 2
    t3 = 0
    for i in range(3, len(solution)):
        t3 += (1 + solution[i] ** 2) ** 0.5
    return [t1, t2, t3]

## Define your objective weights. For example:

problem_dict9 = {
    "obj_func": obj_function,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
    "obj_weight": [0.5, 0.2, 0.3]  # Remember the keyword "obj_weight"
}
problem_obj9 = Problem(problem_dict9)
model9 = SMA.BaseSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model9.solve()

## To access the results, you can get the results by solve() method
position, fitness_value = model9.solve()

## To get all fitness value and all objective values, get it via "solution" attribute
position = model9.solution[0]
fitness_value = model9.solution[1][0]
objective_values = model9.solution[1][1]

# F - Test with different variants of this algorithm

model10 = SMA.OriginalSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model10.solve()


for i in range(3, len(solution)):
        t3 += (1 + solution[i] ** 2) ** 0.5

#       Email:      example@nnthink.com                                                           %

from opfunu.cec_basic.cec2014_nobias import *
from mealpy.bio_based import SMA
from mealpy.problem import Problem
from mealpy.utils.termination import Termination

# Setting parameters

# A - Different way to provide lower bound and upper bound. Here are some examples:

## A1. When you have different lower bound and upper bound for each parameters
problem_dict1 = {
    "obj_func": F5,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
}
problem_obj1 = Problem(problem_dict1)

## A2. When you have same lower bound and upper bound for each parameters, then you can use:
##      + int or float: then you need to specify your problem size / number of dimensions (n_dims)
problem_dict2 = {
    "obj_func": F5,
    "lb": -10,
    "ub": 30,
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj2 = Problem(problem_dict2)

problem_dict3 = {
    "obj_func": F5,
    "lb": [-5],
    "ub": [10],
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj3 = Problem(problem_dict3)

n_dims = 100
problem_dict4 = {
    "obj_func": F5,
    "lb": [-5] * n_dims,
    "ub": [10] * n_dims,
    "minmax": "min",
    "verbose": True,
}

## Run the algorithm

### Your parameter problem can be an instane of Problem class or just dict like above
model1 = SMA.BaseSMA(problem_obj1, epoch=100, pop_size=50, pr=0.03)
model1.solve()

model2 = SMA.BaseSMA(problem_dict4, epoch=100, pop_size=50, pr=0.03)
model2.solve()

# B - Test with different Stopping Condition (Termination) by creating an Termination object

## There are 4 termination cases:
### 1. FE (Number of Function Evaluation)
### 3. ES (Early Stopping): Same idea in training neural network (If the global best solution not better an epsilon
###     after K epoch then stop the program
### 4. TB (Time Bound): You just want your algorithm run in K seconds. Especially when comparing different algorithms.

termination_dict1 = {
    "mode": "FE",
    "quantity": 100000  # 100000 number of function evaluation
}
termination_dict2 = {  # When creating this object, it will override the default epoch you define in your model
    "mode": "MG",
    "quantity": 1000  # 1000 epochs
}
termination_dict3 = {
    "mode": "ES",
    "quantity": 30  # after 30 epochs, if the global best doesn't improve then we stop the program
}
termination_dict4 = {
    "mode": "ES",
    "quantity": 60  # 60 seconds = 1 minute to run this algorithm only
}
termination_obj1 = Termination(termination_dict1)
termination_obj2 = Termination(termination_dict2)
termination_obj3 = Termination(termination_dict3)
termination_obj4 = Termination(termination_dict4)

### Pass your termination object into your model as a addtional parameter with the keyword "termination"
model3 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03, termination=termination_obj1)
model3.solve()
### Remember you can't pass termination dict, it only accept the Termination object

# C - Test with different training mode (sequential, threading parallelization, processing parallelization)

## + thread: create multiple threading depend on your chip
## + process: create multiple cores to run your algorithm.
## Note: For windows, your program need the if __nam__ == "__main__" condition to avoid creating infinite processors

model5 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model5.solve(mode='sequential')  # Default

model6 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model6.solve(mode='thread')

if __name__ == "__main__":
    model7 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
    model7.solve(mode='process')

# D - Drawing all available figures

## There are 8 different figures for each algorithm.
## D.1: Based on fitness value:
##      1. Global best fitness chart
##      2. Local best fitness chart
## D.2: Based on objective value:
##      3. Global objective chart
##      4. Local objective chart
## D.3: Based on runtime value (runtime for each epoch)
## D.4: Based on exploration verse exploration value
##      6. Exploration vs Exploitation chart
## D.5: Based on diversity of population
## D.6: Based on trajectory value (1D, 2D only)

model8 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model8.solve()

## You can access them all via object "history" like this:
model8.history.save_global_objectives_chart(filename="hello/goc")
model8.history.save_local_objectives_chart(filename="hello/loc")
model8.history.save_global_best_fitness_chart(filename="hello/gbfc")
model8.history.save_local_best_fitness_chart(filename="hello/lbfc")
model8.history.save_runtime_chart(filename="hello/rtc")
model8.history.save_exploration_exploitation_chart(filename="hello/eec")
model8.history.save_diversity_chart(filename="hello/dc")
model8.history.save_trajectory_chart(list_agent_idx=[3, 5], list_dimensions=[3], filename="hello/tc")

# E - Handling Multi-Objective function and Constraint Method

## To handling Multi-Objective, mealpy is using weighting method which converting multiple objectives to a single target (fitness value)

## Define your objective function, your constraint
def obj_function(solution):
    t1 = solution[0] ** 2
    t2 = ((2 * solution[1]) / 5) ** 2
    t3 = 0

    return [t1, t2, t3]

## Define your objective weights. For example:

problem_dict9 = {
    "obj_func": obj_function,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
    "obj_weight": [0.5, 0.2, 0.3]  # Remember the keyword "obj_weight"
}
problem_obj9 = Problem(problem_dict9)
model9 = SMA.BaseSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model9.solve()

## To access the results, you can get the results by solve() method
position, fitness_value = model9.solve()

## To get all fitness value and all objective values, get it via "solution" attribute
position = model9.solution[0]
fitness_value = model9.solution[1][0]
objective_values = model9.solution[1][1]

# F - Test with different variants of this algorithm

model10 = SMA.OriginalSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model10.solve()


#       Email:      example@nnthink.com                                                           %

from opfunu.cec_basic.cec2014_nobias import *
from mealpy.bio_based import SMA
from mealpy.problem import Problem
from mealpy.utils.termination import Termination

# Setting parameters

# A - Different way to provide lower bound and upper bound. Here are some examples:

## A1. When you have different lower bound and upper bound for each parameters
problem_dict1 = {
    "obj_func": F5,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
}
problem_obj1 = Problem(problem_dict1)

## A2. When you have same lower bound and upper bound for each parameters, then you can use:
##      + int or float: then you need to specify your problem size / number of dimensions (n_dims)
problem_dict2 = {
    "obj_func": F5,
    "lb": -10,
    "ub": 30,
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj2 = Problem(problem_dict2)

problem_dict3 = {
    "obj_func": F5,
    "lb": [-5],
    "ub": [10],
    "minmax": "min",
    "verbose": True,
    "n_dims": 30,  # Remember the keyword "n_dims"
}
problem_obj3 = Problem(problem_dict3)

n_dims = 100
problem_dict4 = {
    "obj_func": F5,
    "lb": [-5] * n_dims,
    "ub": [10] * n_dims,
    "minmax": "min",
    "verbose": True,
}

## Run the algorithm

### Your parameter problem can be an instane of Problem class or just dict like above
model1 = SMA.BaseSMA(problem_obj1, epoch=100, pop_size=50, pr=0.03)
model1.solve()

model2 = SMA.BaseSMA(problem_dict4, epoch=100, pop_size=50, pr=0.03)
model2.solve()

# B - Test with different Stopping Condition (Termination) by creating an Termination object

## There are 4 termination cases:
### 1. FE (Number of Function Evaluation)
### 3. ES (Early Stopping): Same idea in training neural network (If the global best solution not better an epsilon
###

after K epoch then stop the program
### 4. TB (Time Bound): You just want your algorithm run in K seconds. Especially when comparing different algorithms.

termination_dict1 = {
    "mode": "FE",
    "quantity": 100000  # 100000 number of function evaluation
}
termination_dict2 = {  # When creating this object, it will override the default epoch you define in your model
    "mode": "MG",
    "quantity": 1000  # 1000 epochs
}
termination_dict3 = {
    "mode": "ES",
    "quantity": 30  # after 30 epochs, if the global best doesn't improve then we stop the program
}
termination_dict4 = {
    "mode": "ES",
    "quantity": 60  # 60 seconds = 1 minute to run this algorithm only
}
termination_obj1 = Termination(termination_dict1)
termination_obj2 = Termination(termination_dict2)
termination_obj3 = Termination(termination_dict3)
termination_obj4 = Termination(termination_dict4)

### Pass your termination object into your model as a addtional parameter with the keyword "termination"
model3 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03, termination=termination_obj1)
model3.solve()
### Remember you can't pass termination dict, it only accept the Termination object

# C - Test with different training mode (sequential, threading parallelization, processing parallelization)

## + thread: create multiple threading depend on your chip
## + process: create multiple cores to run your algorithm.
## Note: For windows, your program need the if __nam__ == "__main__" condition to avoid creating infinite processors

model5 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model5.solve(mode='sequential')  # Default

model6 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model6.solve(mode='thread')

if __name__ == "__main__":
    model7 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
    model7.solve(mode='process')

# D - Drawing all available figures

## There are 8 different figures for each algorithm.
## D.1: Based on fitness value:
##      1. Global best fitness chart
##      2. Local best fitness chart
## D.2: Based on objective value:
##      3. Global objective chart
##      4. Local objective chart
## D.3: Based on runtime value (runtime for each epoch)
## D.4: Based on exploration verse exploration value
##      6. Exploration vs Exploitation chart
## D.5: Based on diversity of population
## D.6: Based on trajectory value (1D, 2D only)

model8 = SMA.BaseSMA(problem_dict1, epoch=100, pop_size=50, pr=0.03)
model8.solve()

## You can access them all via object "history" like this:
model8.history.save_global_objectives_chart(filename="hello/goc")
model8.history.save_local_objectives_chart(filename="hello/loc")
model8.history.save_global_best_fitness_chart(filename="hello/gbfc")
model8.history.save_local_best_fitness_chart(filename="hello/lbfc")
model8.history.save_runtime_chart(filename="hello/rtc")
model8.history.save_exploration_exploitation_chart(filename="hello/eec")
model8.history.save_diversity_chart(filename="hello/dc")
model8.history.save_trajectory_chart(list_agent_idx=[3, 5], list_dimensions=[3], filename="hello/tc")

# E - Handling Multi-Objective function and Constraint Method

## To handling Multi-Objective, mealpy is using weighting method which converting multiple objectives to a single target (fitness value)

## Define your objective function, your constraint
def obj_function(solution):
    t1 = solution[0] ** 2
    t2 = ((2 * solution[1]) / 5) ** 2
    t3 = 0
    for i in range(3, len(solution)):
        t3 += (1 + solution[i] ** 2) ** 0.5
    return [t1, t2, t3]

## Define your objective weights. For example:

problem_dict9 = {
    "obj_func": obj_function,
    "lb": [-3, -5, 1, -10, ],
    "ub": [5, 10, 100, 30, ],
    "minmax": "min",
    "verbose": True,
    "obj_weight": [0.5, 0.2, 0.3]  # Remember the keyword "obj_weight"
}
problem_obj9 = Problem(problem_dict9)
model9 = SMA.BaseSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model9.solve()

## To access the results, you can get the results by solve() method
position, fitness_value = model9.solve()

## To get all fitness value and all objective values, get it via "solution" attribute
position = model9.solution[0]
fitness_value = model9.solution[1][0]
objective_values = model9.solution[1][1]

# F - Test with different variants of this algorithm

model10 = SMA.OriginalSMA(problem_obj9, epoch=100, pop_size=50, pr=0.03)
model10.solve()


try:
    someid = sys.argv[1]
except IndexError:
    print('Supply a PMCID or DOI (example: ./blah.py PMC3479421)')
    sys.exit()

from __future__ import absolute_import, print_function, unicode_literals

import sys, logging
from metapub import PubMedFetcher

logging.loglevel =  logging.INFO

fetch = PubMedFetcher()
article = fetch.article_by_pmcid(someid)

print(article.title)
print(', '.join(article.authors))
print()

print(article.journal, article.volume_issue)
print()
print('Pubmed ID: ' + article.pmid)

"""
<pmcids status="ok">
<request idtype="pmcid" pmcids="" versions="yes" showaiid="no">
<echo>tool=medgen-python;email=naomi.most%40invitae.com;ids=PMC3479421</echo>
</request>
<record requested-id="PMC3479421" pmcid="PMC3479421" pmid="22676651" doi="10.1186/1750-1172-7-35"><versions><version pmcid="PMC3479421.1" current="true"/></versions></record>
</pmcids>
"""


from __future__ import absolute_import, print_function, unicode_literals

import sys, logging
from metapub import PubMedFetcher

logging.loglevel =  logging.IN

FO

try:
    someid = sys.argv[1]
except IndexError:
    print('Supply a PMCID or DOI (example: ./blah.py PMC3479421)')
    sys.exit()

fetch = PubMedFetcher()
article = fetch.article_by_pmcid(someid)

print(article.title)
print(', '.join(article.authors))
print()

print(article.journal, article.volume_issue)
print()
print('Pubmed ID: ' + article.pmid)

"""
<pmcids status="ok">
<request idtype="pmcid" pmcids="" versions="yes" showaiid="no">
<echo>tool=medgen-python;email=naomi.most%40invitae.com;ids=PMC3479421</echo>
</request>
<record requested-id="PMC3479421" pmcid="PMC3479421" pmid="22676651" doi="10.1186/1750-1172-7-35"><versions><version pmcid="PMC3479421.1" current="true"/></versions></record>
</pmcids>
"""


def pase_local(ckpt, model_config, **kwargs):

"""
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


"""
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


"""
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]

"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):

        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)

"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]

        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return


    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """


for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


"""*********************************************************************************************"""
#   Synopsis     [ the pase torch hubconf ]
"""*********************************************************************************************"""

import os
from typing import Tuple, List

import torch
from torch import Tensor

from s3prl.utility.download import _urls_to_filepaths
from .expert import UpstreamExpert as _UpstreamExpert

def pase_local(ckpt, model_config, **kwargs):
    """
    The model from local ckpt
        ckpt (str): PATH
        model_config (str): PATH
    """
    assert os.path.isfile(ckpt)
    assert os.path.isfile(model_config)
    return _UpstreamExpert(ckpt, model_config, **kwargs)

def pase_url(ckpt, model_config, refresh=False, **kwargs):
    """
    The model from URL
        ckpt (str): URL
        model_config (str): URL
    """
    ckpt = _urls_to_filepaths(ckpt, refresh=refresh)
    model_config = _urls_to_filepaths(model_config, refresh=refresh)
    return pase_local(ckpt, model_config, **kwargs)

def pase_plus(refresh=False, **kwargs):
    """
    The default model
        refresh (bool): whether to download ckpt/config again if existed
    """
    kwargs["ckpt"] = "http://example.com"
    kwargs[
        "model_config"
    ] = "https://www.dropbox.com/s/2p3ouod1k0ekfxn/PASE%2B.cfg?dl=0"

    def align_

skip(input_, skip):
        """
        Ref: http://example.com
        """
        dfactor = skip.shape[2] // input_.shape[2]
        if dfactor > 1:
            maxlen = input_.shape[2] * dfactor
            skip = skip[:, :, :maxlen]
            bsz, feats, slen = skip.shape
            skip_re = skip.view(bsz, feats, slen // dfactor, dfactor)
            skip = torch.mean(skip_re, dim=3)
        return skip

    def hook_postprocess(hiddens: List[Tuple[str, Tensor]]):
        remained_hiddens = [x for x in hiddens if x[0] != "self.model"]
        final_hidden = [x for x in hiddens if x[0] == "self.model"]
        assert len(final_hidden) == 1
        final_hidden = final_hidden[0]

        updated_hiddens = []
        for identifier, tensor in remained_hiddens:
            updated_hiddens.append(
                (identifier, align_skip(final_hidden[1], tensor).transpose(1, 2))
            )

        updated_hiddens.append((final_hidden[0], final_hidden[1].transpose(1, 2)))
        return updated_hiddens

    hooks = [
        (f"self.model.W", lambda input, output: output),
        (f"self.model", lambda input, output: output),
    ]
    for i in range(7):
        hooks.append((f"self.model.denseskips[{i}]", lambda input, output: output))

    kwargs["hooks"] = hooks
    kwargs["hook_postprocess"] = hook_postprocess
    return pase_url(refresh=refresh, **kwargs)


def predict_unlabeled(model,
                      model_path,
                      nr_visualize=100,
                      output_dir='output_patch_samples'):

"""Inference entrance."""
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-builtin
# pylint: disable=unused-variable
import argparse
import collections
import itertools
import numpy as np
import os
import shutil
import cv2
import tqdm
import deepdish as dd
import json

import tensorflow as tf
from tensorflow.python.framework import test_util

import tensorpack.utils.viz as tpviz
from tensorpack.predict import MultiTowerOfflinePredictor, OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit, get_tf_version_tuple
from tensorpack.tfutils.export import ModelExporter
from tensorpack.utils import fs, logger

from dataset import register_coco, register_voc
from config import config as cfg
from config import finalize_configs

from data import get_eval_unlabeled_dataflow

from FasterRCNN.dataset import DatasetRegistry
from FasterRCNN.data import get_eval_dataflow
from FasterRCNN.eval import DetectionResult, multithread_predict_dataflow, predict_image
from FasterRCNN.modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model
from FasterRCNN.viz import draw_final_outputs, draw_predictions
from FasterRCNN.utils import custom
from FasterRCNN.predict import do_evaluate, do_predict, do_visualize

"""Predict the pseudo label information of unlabeled data."""

  assert cfg.EVAL.PSEUDO_INFERENCE, 'set cfg.EVAL.PSEUDO_INFERENCE=True'
  df, dataset_size = get_eval_unlabeled_dataflow(
      cfg.DATA.TRAIN, return_size=True)
  df.reset_state()
  predcfg = PredictConfig(
      model=model,
      session_init=SmartInit(model_path),
      input_names=['image'],  # ['image', 'gt_boxes', 'gt_labels'],
      output_names=[
          'generate_{}_proposals/boxes'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'generate_{}_proposals/scores'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'fastrcnn_all_scores',
          'output/boxes',
          'output/scores',  # score of the labels
          'output/labels',
      ])
  pred = OfflinePredictor(predcfg)

  if os.path.isdir(output_dir):
    if os.path.isfile(os.path.join(output_dir, 'pseudo_data.npy')):
      os.remove(os.path.join(output_dir, 'pseudo_data.npy'))
    if not os.path.isdir(os.path.join(output_dir, 'vis')):
      os.makedirs(os.path.join(output_dir, 'vis'))
    else:
      shutil.rmtree(os.path.join(output_dir, 'vis'))
      fs.mkdir_p(output_dir + '/vis')
  else:
    fs.mkdir_p(output_dir)
    fs.mkdir_p(output_dir + '/vis')
  logger.warning('-' * 100)
  logger.warning('Write to {}'.format(output_dir))
  logger.warning('-' * 100)

  with tqdm.tqdm(total=nr_visualize) as pbar:
    for idx, dp in itertools.islice(enumerate(df), nr_visualize):
      img, img_id = dp  # dp['image'], dp['img_id']
      rpn_boxes, rpn_scores, all_scores, \
          final_boxes, final_scores, final_labels = pred(img)
      outs = {
          'proposals_boxes': rpn_boxes,  # (?,4)
          'proposals_scores': rpn_scores,  # (?,)
          'boxes': final_boxes,
          'scores': final_scores,
          'labels': final_labels
      }
      ratios = [10, 10]  # [top 20% as background, bottom 20% as background]
      bg_ind, fg_ind = custom.find_bg_and_fg_proposals(
          all_scores, ratios=ratios)

      bg_viz = draw_predictions(img, rpn_boxes[bg_ind], all_scores[bg_ind])

      fg_viz = draw_predictions(img, rpn_boxes[fg_ind], all_scores[fg_ind])

      results = [
          DetectionResult(*args)
          for args in zip(final_boxes, final_scores, final_labels, [None] *
                          len(final_labels))
      ]
      final_viz = draw_final_outputs(img, results)

      viz = tpviz.stack_patches([bg_viz, fg_viz, final_viz], 2, 2)

      if os.environ.get('DISPLAY', None):
        tpviz.interactive_imshow(viz)
      assert cv2.imwrite('{}/vis/{:03d}.png'.format(output_dir, idx), viz)
      pbar.update()
  logger.info('Write {} samples to {}'.format(nr_visualize, output_dir))

  ## Parallel inference the whole unlabled data
  pseudo_preds = collections.defaultdict(list)

  num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
  graph_funcs = MultiTowerOfflinePredictor(predcfg, list(
      range(num_tower))).get_predictors()
  dataflows = [
      get_eval_unlabeled_dataflow(
          cfg.DATA.TRAIN, shard=k, num_shards=num_tower)
      for k in range(num_tower)
  ]

  all_results = multithread_predict_dataflow(dataflows, graph_funcs)

  for id, result in tqdm.tqdm(enumerate(all_results)):
    img_id = result['image_id']
    outs = {
        'proposals_boxes': result['proposal_box'].astype(np.float16),  # (?,4)
        'proposals_scores': result['proposal_score'].astype(np.float16),  # (?,)
        'boxes': result['bbox'].astype(np.float16),  # (?,4)
        'scores': result['score'].astype(np.float16),  # (?,)
        'labels':
            result['category_id'].astype(np.float16)  # (?,)
    }
    pseudo_preds[img_id] = outs
  logger.warn('Writing to {}'.format(
      os.path.join(output_dir, 'pseudo_data.npy')))
  try:
    dd.io.save(os.path.join(output_dir, 'pseudo_data.npy'), pseudo_preds)
  except RuntimeError:
    logger.error('Save failed. Check reasons manually...')

def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
  """Evaluate unlabled data."""

  for i, dataset in enumerate(cfg.DATA.VAL):
    output = output_file + '-' + dataset
    if not os.path.isfile(output) or not reuse:
      if i == 0:
        num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
        graph_funcs = MultiTowerOfflinePredictor(
            pred_config, list(range(num_tower))).get_predictors()
      logger.info('Evaluating {} ...'.format(dataset))
      dataflows = [
          get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
          for k in range(num_tower)
      ]
      all_results = multithread_predict_dataflow(dataflows, graph_funcs)
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
    else:
      all_results = json.load(open(output, 'r'))
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE, metric_only=True)

    with open(output + '_cocometric.json', 'w') as f:
      json.dump(eval_metrics, f)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--load', help='load a model for evaluation.', required=True)
  parser.add_argument(
      '--visualize', action='store_true', help='visualize intermediate results')
  parser.add_argument(
      '--predict_unlabeled', help='visualize intermediate results')
  parser.add_argument('--eval_unlabeled', help='visualize intermediate results')
  parser.add_argument(
      '--evaluate',
      help='Run evaluation. '
      'This argument is the path to the output json evaluation file')
  parser.add_argument(
      '--predict',
      help='Run prediction on a given image. '
      'This argument is the path to the input image file',
      nargs='+')
  parser.add_argument(
      '--benchmark',
      action='store_true',
      help='Benchmark the speed of the model + postprocessing')
  parser.add_argument(
      '--config',
      help='A list of KEY=VALUE to overwrite those defined in config.py',
      nargs='+')
  parser.add_argument('--output-pb', help='Save a model to .pb')
  parser.add_argument('--output-serving', help='Save a model to serving file')

  args = parser.parse_args()
  if args.config:
    cfg.update_args(args.config)
  try:
    register_voc(cfg.DATA.BASEDIR)  # add VOC datasets to the registry
  except NotImplementedError:
    logger.warning('VOC does not find!')
  register_coco(cfg.DATA.BASEDIR)  # add COCO datasets to the registry
  MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

  if not tf.test.is_gpu_available():
    assert get_tf_version_tuple() >= (1, 7) and test_util.IsMklEnabled(), \
        'Inference requires either GPU support or MKL support!'
  assert args.load
  finalize_configs(is_training=False)

  if args.predict or args.visualize:
    cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

  # let the output has the same path logic as checkpoint
  if args.predict_unlabeled:
    output_dir = args.predict_unlabeled
    predict_unlabeled(MODEL, args.load, output_dir=output_dir)

  if args.visualize:
    do_visualize(MODEL, args.load, output_dir=output_dir)
  else:
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(args.load),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1])

    if args.output_pb:
      ModelExporter(predcfg).export_compact(args.output_pb, optimize=False)
    elif args.output_serving:
      ModelExporter(predcfg).export_serving(args.output_serving, optimize=False)

    if args.predict:
      predictor = OfflinePredictor(predcfg)
      for image_file in args.predict:
        do_predict(predictor, image_file)
    elif args.evaluate:
      assert args.evaluate.endswith('.json'), args.evaluate
      do_evaluate(predcfg, args.evaluate)
    elif args.eval_unlabeled:
      assert args.eval_unlabeled.endswith('.json'), args.eval_unlabeled
      do_evaluate_unlabeled(predcfg, args.eval_unlabeled)
    elif args.benchmark:
      df = get_eval_dataflow(cfg.DATA.VAL[0])
      df.reset_state()
      predictor = OfflinePredictor(predcfg)
      for _, img in enumerate(tqdm.tqdm(df, total=len(df), smoothing=0.5)):
        predict_image(img[0], predictor)


"""Predict the pseudo label information of unlabeled data."""

  assert cfg.EVAL.PSEUDO_INFERENCE, 'set cfg.EVAL.PSEUDO_INFERENCE=True'
  df, dataset_size = get_eval_unlabeled_dataflow(
      cfg.DATA.TRAIN, return_size=True)
  df.reset_state()
  predcfg = PredictConfig(
      model=model,
      session_init=SmartInit(model_path),
      input_names=['image'],  # ['image', 'gt_boxes', 'gt_labels'],
      output_names=[
          'generate_{}_proposals/boxes'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'generate_{}_proposals/scores'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'fastrcnn_all_scores',
          'output/boxes',
          'output/scores',  # score of the labels
          'output/labels',
      ])
  pred = OfflinePredictor(predcfg)

  if os.path.isdir(output_dir):
    if os.path.isfile(os.path.join(output_dir, 'pseudo_data.npy')):
      os.remove(os.path.join(output_dir, 'pseudo_data.npy'))
    if not os.path.isdir(os.path.join(output_dir, 'vis')):
      os.makedirs(os.path.join(output_dir, 'vis'))
    else:
      shutil.rmtree(os.path.join(output_dir, 'vis'))
      fs.mkdir_p(output_dir + '/vis')
  else:
    fs.mkdir_p(output_dir)
    fs.mkdir_p(output_dir + '/vis')

"""Inference entrance."""
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-builtin
# pylint: disable=unused-variable
import argparse
import collections
import itertools
import numpy as np
import os
import shutil
import cv2
import tqdm
import deepdish as dd
import json

import tensorflow as tf
from tensorflow.python.framework import test_util

import tensorpack.utils.viz as tpviz
from tensorpack.predict import MultiTowerOfflinePredictor, OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit, get_tf_version_tuple
from tensorpack.tfutils.export import ModelExporter
from tensorpack.utils import fs, logger

from dataset import register_coco, register_voc
from config import config as cfg
from config import finalize_configs

from data import get_eval_unlabeled_dataflow

from FasterRCNN.dataset import DatasetRegistry
from FasterRCNN.data import get_eval_dataflow
from FasterRCNN.eval import DetectionResult, multithread_predict_dataflow, predict_image
from FasterRCNN.modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model
from FasterRCNN.viz import draw_final_outputs, draw_predictions
from FasterRCNN.utils import custom
from FasterRCNN.predict import do_evaluate, do_predict, do_visualize

def predict_unlabeled(model,
                      model_path,
                      nr_visualize=100,
                      output_dir='output_patch_samples'):

  logger.warning('-' * 100)
  logger.warning('Write to {}'.format(output_dir))
  logger.warning('-' * 100)

  with tqdm.tqdm(total=nr_visualize) as pbar:
    for idx, dp in itertools.islice(enumerate(df), nr_visualize):
      img, img_id = dp  # dp['image'], dp['img_id']
      rpn_boxes, rpn_scores, all_scores, \
          final_boxes, final_scores, final_labels = pred(img)
      outs = {
          'proposals_boxes': rpn_boxes,  # (?,4)
          'proposals_scores': rpn_scores,  # (?,)
          'boxes': final_boxes,
          'scores': final_scores,
          'labels': final_labels
      }
      ratios = [10, 10]  # [top 20% as background, bottom 20% as background]
      bg_ind, fg_ind = custom.find_bg_and_fg_proposals(
          all_scores, ratios=ratios)

      bg_viz = draw_predictions(img, rpn_boxes[bg_ind], all_scores[bg_ind])

      fg_viz = draw_predictions(img, rpn_boxes[fg_ind], all_scores[fg_ind])

      results = [
          DetectionResult(*args)
          for args in zip(final_boxes, final_scores, final_labels, [None] *
                          len(final_labels))
      ]
      final_viz = draw_final_outputs(img, results)

      viz = tpviz.stack_patches([bg_viz, fg_viz, final_viz], 2, 2)

      if os.environ.get('DISPLAY', None):
        tpviz.interactive_imshow(viz)
      assert cv2.imwrite('{}/vis/{:03d}.png'.format(output_dir, idx), viz)
      pbar.update()
  logger.info('Write {} samples to {}'.format(nr_visualize, output_dir))

  ## Parallel inference the whole unlabled data
  pseudo_preds = collections.defaultdict(list)

  num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
  graph_funcs = MultiTowerOfflinePredictor(predcfg, list(
      range(num_tower))).get_predictors()
  dataflows = [
      get_eval_unlabeled_dataflow(
          cfg.DATA.TRAIN, shard=k, num_shards=num_tower)
      for k in range(num_tower)
  ]

  all_results = multithread_predict_dataflow(dataflows, graph_funcs)

  for id, result in tqdm.tqdm(enumerate(all_results)):
    img_id = result['image_id']
    outs = {
        'proposals_boxes': result['proposal_box'].astype(np.float16),  # (?,4)
        'proposals_scores': result['proposal_score'].astype(np.float16),  # (?,)
        'boxes': result['bbox'].astype(np.float16),  # (?,4)
        'scores': result['score'].astype(np.float16),  # (?,)
        'labels':
            result['category_id'].astype(np.float16)  # (?,)
    }
    pseudo_preds[img_id] = outs
  logger.warn('Writing to {}'.format(
      os.path.join(output_dir, 'pseudo_data.npy')))
  try:
    dd.io.save(os.path.join(output_dir, 'pseudo_data.npy'), pseudo_preds)
  except RuntimeError:
    logger.error('Save failed. Check reasons manually...')

def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
  """Evaluate unlabled data."""

  for i, dataset in enumerate(cfg.DATA.VAL):
    output = output_file + '-' + dataset
    if not os.path.isfile(output) or not reuse:
      if i == 0:
        num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
        graph_funcs = MultiTowerOfflinePredictor(
            pred_config, list(range(num_tower))).get_predictors()
      logger.info('Evaluating {} ...'.format(dataset))
      dataflows = [
          get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
          for k in range(num_tower)
      ]
      all_results = multithread_predict_dataflow(dataflows, graph_funcs)
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
    else:
      all_results = json.load(open(output, 'r'))
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE, metric_only=True)

    with open(output + '_cocometric.json', 'w') as f:
      json.dump(eval_metrics, f)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--load', help='load a model for evaluation.', required=True)
  parser.add_argument(
      '--visualize', action='store_true', help='visualize intermediate results')
  parser.add_argument(
      '--predict_unlabeled', help='visualize intermediate results')
  parser.add_argument('--eval_unlabeled', help='visualize intermediate results')
  parser.add_argument(
      '--evaluate',
      help='Run evaluation. '
      'This argument is the path to the output json evaluation file')
  parser.add_argument(
      '--predict',
      help='Run prediction on a given image. '
      'This argument is the path to the input image file',
      nargs='+')
  parser.add_argument(
      '--benchmark',
      action='store_true',
      help='Benchmark the speed of the model + postprocessing')
  parser.add_argument(
      '--config',
      help='A list of KEY=VALUE to overwrite those defined in config.py',
      nargs='+')
  parser.add_argument('--output-pb', help='Save a model to .pb')
  parser.add_argument('--output-serving', help='Save a model to serving file')

  args = parser.parse_args()
  if args.config:
    cfg.update_args(args.config)
  try:
    register_voc(cfg.DATA.BASEDIR)  # add VOC datasets to the registry
  except NotImplementedError:
    logger.warning('VOC does not find!')
  register_coco(cfg.DATA.BASEDIR)  # add COCO datasets to the registry
  MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

  if not tf.test.is_gpu_available():
    assert get_tf_version_tuple() >= (1, 7) and test_util.IsMklEnabled(), \
        'Inference requires either GPU support or MKL support!'
  assert args.load
  finalize_configs(is_training=False)

  if args.predict or args.visualize:
    cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

  # let the output has the same path logic as checkpoint
  if args.predict_unlabeled:
    output_dir = args.predict_unlabeled
    predict_unlabeled(MODEL, args.load, output_dir=output_dir)

  if args.visualize:
    do_visualize(MODEL, args.load, output_dir=output_dir)
  else:
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(args.load),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1])

    if args.output_pb:
      ModelExporter(predcfg).export_compact(args.output_pb, optimize=False)
    elif args.output_serving:
      ModelExporter(predcfg).export_serving(args.output_serving, optimize=False)

    if args.predict:
      predictor = OfflinePredictor(predcfg)
      for image_file in args.predict:
        do_predict(predictor, image_file)
    elif args.evaluate:
      assert args.evaluate.endswith('.json'), args.evaluate
      do_evaluate(predcfg, args.evaluate)
    elif args.eval_unlabeled:
      assert args.eval_unlabeled.endswith('.json'), args.eval_unlabeled
      do_evaluate_unlabeled(predcfg, args.eval_unlabeled)
    elif args.benchmark:
      df = get_eval_dataflow(cfg.DATA.VAL[0])
      df.reset_state()
      predictor = OfflinePredictor(predcfg)
      for _, img in enumerate(tqdm.tqdm(df, total=len(df), smoothing=0.5)):
        predict_image(img[0], predictor)


logger.warning('-' * 100)
  logger.warning('Write to {}'.format(output_dir))
  logger.warning('-' * 100)

  with tqdm.tqdm(total=nr_visualize) as pbar:
    for idx, dp in itertools.islice(enumerate(df), nr_visualize):
      img, img_id = dp  # dp['image'], dp['img_id']
      rpn_boxes, rpn_scores, all_scores, \
          final_boxes, final_scores, final_labels = pred(img)
      outs = {
          'proposals_boxes': rpn_boxes,  # (?,4)
          'proposals_scores': rpn_scores,  # (?,)
          'boxes': final_boxes,
          'scores': final_scores,
          'labels': final_labels
      }
      ratios = [10, 10]  # [top 20% as background, bottom 20% as background]
      bg_ind, fg_ind = custom.find_bg_and_fg_proposals(
          all_scores, ratios=ratios)

      bg_viz = draw_predictions(img, rpn_boxes[bg_ind], all_scores[bg_ind])

      fg_viz = draw_predictions(img, rpn_boxes[fg_ind], all_scores[fg_ind])

      results = [
          DetectionResult(*args)
          for args in zip(final_boxes, final_scores, final_labels, [None] *
                          len(final_labels))
      ]
      final_viz = draw_final_outputs(img, results)

      viz = tpviz.stack_patches([bg_viz, fg_viz, final_viz], 2, 2)

      if os.environ.get('DISPLAY', None):
        tpviz.interactive_imshow(viz)
      assert cv2.imwrite('{}/vis/{:03d}.png'.format(output_dir, idx), viz)
      pbar.update()
  logger.info('Write {} samples to {}'.format(nr_visualize, output_dir))

  ## Parallel inference the whole unlabled data
  pseudo_preds = collections.defaultdict(list)

"""Inference entrance."""
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-builtin
# pylint: disable=unused-variable
import argparse
import collections
import itertools
import numpy as np
import os
import shutil
import cv2
import tqdm
import deepdish as dd
import json

import tensorflow as tf
from tensorflow.python.framework import test_util

import tensorpack.utils.viz as tpviz
from tensorpack.predict import MultiTowerOfflinePredictor, OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit, get_tf_version_tuple
from tensorpack.tfutils.export import ModelExporter
from tensorpack.utils import fs, logger

from dataset import register_coco, register_voc
from config import config as cfg
from config import finalize_configs

from data import get_eval_unlabeled_dataflow

from FasterRCNN.dataset import DatasetRegistry
from FasterRCNN.data import get_eval_dataflow
from FasterRCNN.eval import DetectionResult, multithread_predict_dataflow, predict_image
from FasterRCNN.modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model
from FasterRCNN.viz import draw_final_outputs, draw_predictions
from FasterRCNN.utils import custom
from FasterRCNN.predict import do_evaluate, do_predict, do_visualize

def predict_unlabeled(model,
                      model_path,
                      nr_visualize=100,
                      output_dir='output_patch_samples'):
  """Predict the pseudo label information of unlabeled data."""

  assert cfg.EVAL.PSEUDO_INFERENCE, 'set cfg.EVAL.PSEUDO_INFERENCE=True'
  df, dataset_size = get_eval_unlabeled_dataflow(
      cfg.DATA.TRAIN, return_size=True)
  df.reset_state()
  predcfg = PredictConfig(
      model=model,
      session_init=SmartInit(model_path),
      input_names=['image'],  # ['image', 'gt_boxes', 'gt_labels'],
      output_names=[
          'generate_{}_proposals/boxes'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'generate_{}_proposals/scores'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'fastrcnn_all_scores',
          'output/boxes',
          'output/scores',  # score of the labels
          'output/labels',
      ])
  pred = OfflinePredictor(predcfg)

  if os.path.isdir(output_dir):
    if os.path.isfile(os.path.join(output_dir, 'pseudo_data.npy')):
      os.remove(os.path.join(output_dir, 'pseudo_data.npy'))
    if not os.path.isdir(os.path.join(output_dir, 'vis')):
      os.makedirs(os.path.join(output_dir, 'vis'))
    else:
      shutil.rmtree(os.path.join(output_dir, 'vis'))
      fs.mkdir_p(output_dir + '/vis')
  else:
    fs.mkdir_p(output_dir)
    fs.mkdir_p(output_dir + '/vis')

  num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
  graph_funcs = MultiTowerOfflinePredictor(predcfg, list(
      range(num_tower))).get_predictors()
  dataflows = [
      get_eval_unlabeled_dataflow(
          cfg.DATA.TRAIN, shard=k, num_shards=num_tower)
      for k in range(num_tower)
  ]

  all_results = multithread_predict_dataflow(dataflows, graph_funcs)

  for id, result in tqdm.tqdm(enumerate(all_results)):
    img_id = result['image_id']
    outs = {
        'proposals_boxes': result['proposal_box'].astype(np.float16),  # (?,4)
        'proposals_scores': result['proposal_score'].astype(np.float16),  # (?,)
        'boxes': result['bbox'].astype(np.float16),  # (?,4)
        'scores': result['score'].astype(np.float16),  # (?,)
        'labels':
            result['category_id'].astype(np.float16)  # (?,)
    }
    pseudo_preds[img_id] = outs
  logger.warn('Writing to {}'.format(
      os.path.join(output_dir, 'pseudo_data.npy')))
  try:
    dd.io.save(os.path.join(output_dir, 'pseudo_data.npy'), pseudo_preds)
  except RuntimeError:
    logger.error('Save failed. Check reasons manually...')

def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
  """Evaluate unlabled data."""

  for i, dataset in enumerate(cfg.DATA.VAL):
    output = output_file + '-' + dataset
    if not os.path.isfile(output) or not reuse:
      if i == 0:
        num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
        graph_funcs = MultiTowerOfflinePredictor(
            pred_config, list(range(num_tower))).get_predictors()
      logger.info('Evaluating {} ...'.format(dataset))
      dataflows = [
          get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
          for k in range(num_tower)
      ]
      all_results = multithread_predict_dataflow(dataflows, graph_funcs)
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
    else:
      all_results = json.load(open(output, 'r'))
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE, metric_only=True)

    with open(output + '_cocometric.json', 'w') as f:
      json.dump(eval_metrics, f)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--load', help='load a model for evaluation.', required=True)
  parser.add_argument(
      '--visualize', action='store_true', help='visualize intermediate results')
  parser.add_argument(
      '--predict_unlabeled', help='visualize intermediate results')
  parser.add_argument('--eval_unlabeled', help='visualize intermediate results')
  parser.add_argument(
      '--evaluate',
      help='Run evaluation. '
      'This argument is the path to the output json evaluation file')
  parser.add_argument(
      '--predict',
      help='Run prediction on a given image. '
      'This argument is the path to the input image file',
      nargs='+')
  parser.add_argument(
      '--benchmark',
      action='store_true',
      help='Benchmark the speed of the model + postprocessing')
  parser.add_argument(
      '--config',
      help='A list of KEY=VALUE to overwrite those defined in config.py',
      nargs='+')
  parser.add_argument('--output-pb', help='Save a model to .pb')
  parser.add_argument('--output-serving', help='Save a model to serving file')

  args = parser.parse_args()
  if args.config:
    cfg.update_args(args.config)
  try:
    register_voc(cfg.DATA.BASEDIR)  # add VOC datasets to the registry
  except NotImplementedError:
    logger.warning('VOC does not find!')
  register_coco(cfg.DATA.BASEDIR)  # add COCO datasets to the registry
  MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

  if not tf.test.is_gpu_available():
    assert get_tf_version_tuple() >= (1, 7) and test_util.IsMklEnabled(), \
        'Inference requires either GPU support or MKL support!'
  assert args.load
  finalize_configs(is_training=False)

  if args.predict or args.visualize:
    cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

  # let the output has the same path logic as checkpoint
  if args.predict_unlabeled:
    output_dir = args.predict_unlabeled
    predict_unlabeled(MODEL, args.load, output_dir=output_dir)

  if args.visualize:
    do_visualize(MODEL, args.load, output_dir=output_dir)
  else:
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(args.load),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1])

    if args.output_pb:
      ModelExporter(predcfg).export_compact(args.output_pb, optimize=False)
    elif args.output_serving:
      ModelExporter(predcfg).export_serving(args.output_serving, optimize=False)

    if args.predict:
      predictor = OfflinePredictor(predcfg)
      for image_file in args.predict:
        do_predict(predictor, image_file)
    elif args.evaluate:
      assert args.evaluate.endswith('.json'), args.evaluate
      do_evaluate(predcfg, args.evaluate)
    elif args.eval_unlabeled:
      assert args.eval_unlabeled.endswith('.json'), args.eval_unlabeled
      do_evaluate_unlabeled(predcfg, args.eval_unlabeled)
    elif args.benchmark:
      df = get_eval_dataflow(cfg.DATA.VAL[0])
      df.reset_state()
      predictor = OfflinePredictor(predcfg)
      for _, img in enumerate(tqdm.tqdm(df, total=len(df), smoothing=0.5)):
        predict_image(img[0], predictor)


num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
  graph_funcs = MultiTowerOfflinePredictor(predcfg, list(
      range(num_tower))).get_predictors()
  dataflows = [
      get_eval_unlabeled_dataflow(
          cfg.DATA.TRAIN, shard=k, num_shards=num_tower)
      for k in range(num_tower)
  ]

  all_results = multithread_predict_dataflow(dataflows, graph_funcs)

  for id, result in tqdm.tqdm(enumerate(all_results)):
    img_id = result['image_id']
    outs = {
        'proposals_boxes': result['proposal_box'].astype(np.float16),  # (?,4)
        'proposals_scores': result['proposal_score'].astype(np.float16),  # (?,)
        'boxes': result['bbox'].astype(np.float16),  # (?,4)
        'scores': result['score'].astype(np.float16),  # (?,)
        'labels':
            result['category_id'].astype(np.float16)  # (?,)
    }
    pseudo_preds[img_id] = outs
  logger.warn('Writing to {}'.format(
      os.path.join(output_dir, 'pseudo_data.npy')))
  try:
    dd.io.save(os.path.join(output_dir, 'pseudo_data.npy'), pseudo_preds)
  except RuntimeError:
    logger.error('Save failed. Check reasons manually...')

"""Inference entrance."""
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-builtin
# pylint: disable=unused-variable
import argparse
import collections
import itertools
import numpy as np
import os
import shutil
import cv2
import tqdm
import deepdish as dd
import json

import tensorflow as tf
from tensorflow.python.framework import test_util

import tensorpack.utils.viz as tpviz
from tensorpack.predict import MultiTowerOfflinePredictor, OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit, get_tf_version_tuple
from tensorpack.tfutils.export import ModelExporter
from tensorpack.utils import fs, logger

from dataset import register_coco, register_voc
from config import config as cfg
from config import finalize_configs

from data import get_eval_unlabeled_dataflow

from FasterRCNN.dataset import DatasetRegistry
from FasterRCNN.data import get_eval_dataflow
from FasterRCNN.eval import DetectionResult, multithread_predict_dataflow, predict_image
from FasterRCNN.modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model
from FasterRCNN.viz import draw_final_outputs, draw_predictions
from FasterRCNN.utils import custom
from FasterRCNN.predict import do_evaluate, do_predict, do_visualize

def predict_unlabeled(model,
                      model_path,
                      nr_visualize=100,
                      output_dir='output_patch_samples'):
  """Predict the pseudo label information of unlabeled data."""

  assert cfg.EVAL.PSEUDO_INFERENCE, 'set cfg.EVAL.PSEUDO_INFERENCE=True'
  df, dataset_size = get_eval_unlabeled_dataflow(
      cfg.DATA.TRAIN, return_size=True)
  df.reset_state()
  predcfg = PredictConfig(
      model=model,
      session_init=SmartInit(model_path),
      input_names=['image'],  # ['image', 'gt_boxes', 'gt_labels'],
      output_names=[
          'generate_{}_proposals/boxes'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'generate_{}_proposals/scores'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'fastrcnn_all_scores',
          'output/boxes',
          'output/scores',  # score of the labels
          'output/labels',
      ])
  pred = OfflinePredictor(predcfg)

  if os.path.isdir(output_dir):
    if os.path.isfile(os.path.join(output_dir, 'pseudo_data.npy')):
      os.remove(os.path.join(output_dir, 'pseudo_data.npy'))
    if not os.path.isdir(os.path.join(output_dir, 'vis')):
      os.makedirs(os.path.join(output_dir, 'vis'))
    else:
      shutil.rmtree(os.path.join(output_dir, 'vis'))
      fs.mkdir_p(output_dir + '/vis')
  else:
    fs.mkdir_p(output_dir)
    fs.mkdir_p(output_dir + '/vis')
  logger.warning('-' * 100)
  logger.warning('Write to {}'.format(output_dir))
  logger.warning('-' * 100)

  with tqdm.tqdm(total=nr_visualize) as pbar:
    for idx, dp in itertools.islice(enumerate(df), nr_visualize):
      img, img_id = dp  # dp['image'], dp['img_id']
      rpn_boxes, rpn_scores, all_scores, \
          final_boxes, final_scores, final_labels = pred(img)
      outs = {
          'proposals_boxes': rpn_boxes,  # (?,4)
          'proposals_scores': rpn_scores,  # (?,)
          'boxes': final_boxes,
          'scores': final_scores,
          'labels': final_labels
      }
      ratios = [10, 10]  # [top 20% as background, bottom 20% as background]
      bg_ind, fg_ind = custom.find_bg_and_fg_proposals(
          all_scores, ratios=ratios)

      bg_viz = draw_predictions(img, rpn_boxes[bg_ind], all_scores[bg_ind])

      fg_viz = draw_predictions(img, rpn_boxes[fg_ind], all_scores[fg_ind])

      results = [
          DetectionResult(*args)
          for args in zip(final_boxes, final_scores, final_labels, [None] *
                          len(final_labels))
      ]
      final_viz = draw_final_outputs(img, results)

      viz = tpviz.stack_patches([bg_viz, fg_viz, final_viz], 2, 2)

      if os.environ.get('DISPLAY', None):
        tpviz.interactive_imshow(viz)
      assert cv2.imwrite('{}/vis/{:03d}.png'.format(output_dir, idx), viz)
      pbar.update()
  logger.info('Write {} samples to {}'.format(nr_visualize, output_dir))

  ## Parallel inference the whole unlabled data
  pseudo_preds = collections.defaultdict(list)

def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
  """Evaluate unlabled data."""

  for i, dataset in enumerate(cfg.DATA.VAL):
    output = output_file + '-' + dataset
    if not os.path.isfile(output) or not reuse:
      if i == 0:
        num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
        graph_funcs = MultiTowerOfflinePredictor(
            pred_config, list(range(num_tower))).get_predictors()
      logger.info('Evaluating {} ...'.format(dataset))
      dataflows = [
          get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
          for k in range(num_tower)
      ]
      all_results = multithread_predict_dataflow(dataflows, graph_funcs)
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
    else:
      all_results = json.load(open(output, 'r'))
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE, metric_only=True)

    with open(output + '_cocometric.json', 'w') as f:
      json.dump(eval_metrics, f)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--load', help='load a model for evaluation.', required=True)
  parser.add_argument(
      '--visualize', action='store_true', help='visualize intermediate results')
  parser.add_argument(
      '--predict_unlabeled', help='visualize intermediate results')
  parser.add_argument('--eval_unlabeled', help='visualize intermediate results')
  parser.add_argument(
      '--evaluate',
      help='Run evaluation. '
      'This argument is the path to the output json evaluation file')
  parser.add_argument(
      '--predict',
      help='Run prediction on a given image. '
      'This argument is the path to the input image file',
      nargs='+')
  parser.add_argument(
      '--benchmark',
      action='store_true',
      help='Benchmark the speed of the model + postprocessing')
  parser.add_argument(
      '--config',
      help='A list of KEY=VALUE to overwrite those defined in config.py',
      nargs='+')
  parser.add_argument('--output-pb', help='Save a model to .pb')
  parser.add_argument('--output-serving', help='Save a model to serving file')

  args = parser.parse_args()
  if args.config:
    cfg.update_args(args.config)
  try:
    register_voc(cfg.DATA.BASEDIR)  # add VOC datasets to the registry
  except NotImplementedError:
    logger.warning('VOC does not find!')
  register_coco(cfg.DATA.BASEDIR)  # add COCO datasets to the registry
  MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

  if not tf.test.is_gpu_available():
    assert get_tf_version_tuple() >= (1, 7) and test_util.IsMklEnabled(), \
        'Inference requires either GPU support or MKL support!'
  assert args.load
  finalize_configs(is_training=False)

  if args.predict or args.visualize:
    cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

  # let the output has the same path logic as checkpoint
  if args.predict_unlabeled:
    output_dir = args.predict_unlabeled
    predict_unlabeled(MODEL, args.load, output_dir=output_dir)

  if args.visualize:
    do_visualize(MODEL, args.load, output_dir=output_dir)
  else:
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(args.load),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1])

    if args.output_pb:
      ModelExporter(predcfg).export_compact(args.output_pb, optimize=False)
    elif args.output_serving:
      ModelExporter(predcfg).export_serving(args.output_serving, optimize=False)

    if args.predict:
      predictor = OfflinePredictor(predcfg)
      for image_file in args.predict:
        do_predict(predictor, image_file)
    elif args.evaluate:
      assert args.evaluate.endswith('.json'), args.evaluate
      do_evaluate(predcfg, args.evaluate)
    elif args.eval_unlabeled:
      assert args.eval_unlabeled.endswith('.json'), args.eval_unlabeled
      do_evaluate_unlabeled(predcfg, args.eval_unlabeled)
    elif args.benchmark:
      df = get_eval_dataflow(cfg.DATA.VAL[0])
      df.reset_state()
      predictor = OfflinePredictor(predcfg)
      for _, img in enumerate(tqdm.tqdm(df, total=len(df), smoothing=0.5)):
        predict_image(img[0], predictor)


"""Inference entrance."""
# pylint: disable=redefined-outer-name
# pylint: disable=redefined-builtin
# pylint: disable=unused-variable
import argparse
import collections
import itertools
import numpy as np
import os
import shutil
import cv2
import tqdm
import deepdish as dd
import json

import tensorflow as tf
from tensorflow.python.framework import test_util

import tensorpack.utils.viz as tpviz
from tensorpack.predict import MultiTowerOfflinePredictor, OfflinePredictor, PredictConfig
from tensorpack.tfutils import SmartInit, get_tf_version_tuple
from tensorpack.tfutils.export import ModelExporter
from tensorpack.utils import fs, logger

from dataset import register_coco, register_voc
from config import config as cfg
from config import finalize_configs

from data import get_eval_unlabeled_dataflow

from FasterRCNN.dataset import DatasetRegistry
from FasterRCNN.data import get_eval_dataflow
from FasterRCNN.eval import DetectionResult, multithread_predict_dataflow, predict_image
from FasterRCNN.modeling.generalized_rcnn import ResNetFPNModel, ResNetC4Model
from FasterRCNN.viz import draw_final_outputs, draw_predictions
from FasterRCNN.utils import custom
from FasterRCNN.predict import do_evaluate, do_predict, do_visualize

def predict_unlabeled(model,
                      model_path,
                      nr_visualize=100,
                      output_dir='output_patch_samples'):
  """Predict the pseudo label information of unlabeled data."""

  assert cfg.EVAL.PSEUDO_INFERENCE, 'set cfg.EVAL.PSEUDO_INFERENCE=True'
  df, dataset_size = get_eval_unlabeled_dataflow(
      cfg.DATA.TRAIN, return_size=True)
  df.res

  predcfg = PredictConfig(
      model=model,
      session_init=SmartInit(model_path),
      input_names=['image'],  # ['image', 'gt_boxes', 'gt_labels'],
      output_names=[
          'generate_{}_proposals/boxes'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'generate_{}_proposals/scores'.format(
              'fpn' if cfg.MODE_FPN else 'rpn'),
          'fastrcnn_all_scores',
          'output/boxes',
          'output/scores',  # score of the labels
          'output/labels',
      ])
  pred = OfflinePredictor(predcfg)

  if os.path.isdir(output_dir):
    if os.path.isfile(os.path.join(output_dir, 'pseudo_data.npy')):
      os.remove(os.path.join(output_dir, 'pseudo_data.npy'))
    if not os.path.isdir(os.path.join(output_dir, 'vis')):
      os.makedirs(os.path.join(output_dir, 'vis'))
    else:
      shutil.rmtree(os.path.join(output_dir, 'vis'))
      fs.mkdir_p(output_dir + '/vis')
  else:
    fs.mkdir_p(output_dir)
    fs.mkdir_p(output_dir + '/vis')
  logger.warning('-' * 100)
  logger.warning('Write to {}'.format(output_dir))
  logger.warning('-' * 100)

  with tqdm.tqdm(total=nr_visualize) as pbar:
    for idx, dp in itertools.islice(enumerate(df), nr_visualize):
      img, img_id = dp  # dp['image'], dp['img_id']
      rpn_boxes, rpn_scores, all_scores, \
          final_boxes, final_scores, final_labels = pred(img)
      outs = {
          'proposals_boxes': rpn_boxes,  # (?,4)
          'proposals_scores': rpn_scores,  # (?,)
          'boxes': final_boxes,
          'scores': final_scores,
          'labels': final_labels
      }
      ratios = [10, 10]  # [top 20% as background, bottom 20% as background]
      bg_ind, fg_ind = custom.find_bg_and_fg_proposals(
          all_scores, ratios=ratios)

      bg_viz = draw_predictions(img, rpn_boxes[bg_ind], all_scores[bg_ind])

      fg_viz = draw_predictions(img, rpn_boxes[fg_ind], all_scores[fg_ind])

      results = [
          DetectionResult(*args)
          for args in zip(final_boxes, final_scores, final_labels, [None] *
                          len(final_labels))
      ]
      final_viz = draw_final_outputs(img, results)

      viz = tpviz.stack_patches([bg_viz, fg_viz, final_viz], 2, 2)

      if os.environ.get('DISPLAY', None):
        tpviz.interactive_imshow(viz)
      assert cv2.imwrite('{}/vis/{:03d}.png'.format(output_dir, idx), viz)
      pbar.update()
  logger.info('Write {} samples to {}'.format(nr_visualize, output_dir))

  ## Parallel inference the whole unlabled data
  pseudo_preds = collections.defaultdict(list)

  num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
  graph_funcs = MultiTowerOfflinePredictor(predcfg, list(
      range(num_tower))).get_predictors()
  dataflows = [
      get_eval_unlabeled_dataflow(
          cfg.DATA.TRAIN, shard=k, num_shards=num_tower)
      for k in range(num_tower)
  ]

  all_results = multithread_predict_dataflow(dataflows, graph_funcs)

  for id, result in tqdm.tqdm(enumerate(all_results)):
    img_id = result['image_id']
    outs = {
        'proposals_boxes': result['proposal_box'].astype(np.float16),  # (?,4)
        'proposals_scores': result['proposal_score'].astype(np.float16),  # (?,)
        'boxes': result['bbox'].astype(np.float16),  # (?,4)
        'scores': result['score'].astype(np.float16),  # (?,)
        'labels':
            result['category_id'].astype(np.float16)  # (?,)
    }
    pseudo_preds[img_id] = outs
  logger.warn('Writing to {}'.format(
      os.path.join(output_dir, 'pseudo_data.npy')))
  try:
    dd.io.save(os.path.join(output_dir, 'pseudo_data.npy'), pseudo_preds)
  except RuntimeError:
    logger.error('Save failed. Check reasons manually...')

def do_evaluate_unlabeled(pred_config, output_file, reuse=True):
  """Evaluate unlabled data."""

  for i, dataset in enumerate(cfg.DATA.VAL):
    output = output_file + '-' + dataset
    if not os.path.isfile(output) or not reuse:
      if i == 0:
        num_tower = max(cfg.TRAIN.NUM_GPUS, 1)
        graph_funcs = MultiTowerOfflinePredictor(
            pred_config, list(range(num_tower))).get_predictors()
      logger.info('Evaluating {} ...'.format(dataset))
      dataflows = [
          get_eval_dataflow(dataset, shard=k, num_shards=num_tower)
          for k in range(num_tower)
      ]
      all_results = multithread_predict_dataflow(dataflows, graph_funcs)
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE)
    else:
      all_results = json.load(open(output, 'r'))
      eval_metrics = DatasetRegistry.get(dataset).eval_inference_results2(
          all_results, output, threshold=cfg.TRAIN.CONFIDENCE, metric_only=True)

    with open(output + '_cocometric.json', 'w') as f:
      json.dump(eval_metrics, f)

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--load', help='load a model for evaluation.', required=True)
  parser.add_argument(
      '--visualize', action='store_true', help='visualize intermediate results')
  parser.add_argument(
      '--predict_unlabeled', help='visualize intermediate results')
  parser.add_argument('--eval_unlabeled', help='visualize intermediate results')
  parser.add_argument(
      '--evaluate',
      help='Run evaluation. '
      'This argument is the path to the output json evaluation file')
  parser.add_argument(
      '--predict',
      help='Run prediction on a given image. '
      'This argument is the path to the input image file',
      nargs='+')
  parser.add_argument(
      '--benchmark',
      action='store_true',
      help='Benchmark the speed of the model + postprocessing')
  parser.add_argument(
      '--config',
      help='A list of KEY=VALUE to overwrite those defined in config.py',
      nargs='+')
  parser.add_argument('--output-pb', help='Save a model to .pb')
  parser.add_argument('--output-serving', help='Save a model to serving file')

  args = parser.parse_args()
  if args.config:
    cfg.update_args(args.config)
  try:
    register_voc(cfg.DATA.BASEDIR)  # add VOC datasets to the registry
  except NotImplementedError:
    logger.warning('VOC does not find!')
  register_coco(cfg.DATA.BASEDIR)  # add COCO datasets to the registry
  MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

  if not tf.test.is_gpu_available():
    assert get_tf_version_tuple() >= (1, 7) and test_util.IsMklEnabled(), \
        'Inference requires either GPU support or MKL support!'
  assert args.load
  finalize_configs(is_training=False)

  if args.predict or args.visualize:
    cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

  # let the output has the same path logic as checkpoint
  if args.predict_unlabeled:
    output_dir = args.predict_unlabeled
    predict_unlabeled(MODEL, args.load, output_dir=output_dir)

  if args.visualize:
    do_visualize(MODEL, args.load, output_dir=output_dir)
  else:
    predcfg = PredictConfig(
        model=MODEL,
        session_init=SmartInit(args.load),
        input_names=MODEL.get_inference_tensor_names()[0],
        output_names=MODEL.get_inference_tensor_names()[1])

    if args.output_pb:
      ModelExporter(predcfg).export_compact(args.output_pb, optimize=False)
    elif args.output_serving:
      ModelExporter(predcfg).export_serving(args.output_serving, optimize=False)

    if args.predict:
      predictor = OfflinePredictor(predcfg)
      for image_file in args.predict:
        do_predict(predictor, image_file)
    elif args.evaluate:
      assert args.evaluate.endswith('.json'), args.evaluate
      do_evaluate(predcfg, args.evaluate)
    elif args.eval_unlabeled:
      assert args.eval_unlabeled.endswith('.json'), args.eval_unlabeled
      do_evaluate_unlabeled(predcfg, args.eval_unlabeled)
    elif args.benchmark:
      df = get_eval_dataflow(cfg.DATA.VAL[0])
      df.reset_state()
      predictor = OfflinePredictor(predcfg)
      for _, img in enumerate(tqdm.tqdm(df, total=len(df), smoothing=0.5)):
        predict_image(img[0], predictor)