for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v

import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:

        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)


def _replicateTestsOnDims(self, tests, source):

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):

        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")

        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break


# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_nu

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)

            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                if num < 0:
                    num = len(ip["shapes"])
                    break

# All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import copy
import os

from frameworks.framework_base import FrameworkBase
from utils.custom_logger import getLogger
from utils.utilities import getRunStatus

class Caffe2Framework(FrameworkBase):
    IDENTIFIER = "Caffe2Observer "
    NET = "NET"

    def __init__(self, tempdir, args):
        super(Caffe2Framework, self).__init__(args)
        self.tempdir = os.path.join(tempdir, self.getName())
        os.makedirs(self.tempdir, 0o777)
        # cannot have any variable pass among methods

    def getName(self):
        return "caffe2"

    def runBenchmark(self, info, benchmark, platform):
        output, output_files = super(Caffe2Framework, self).runBenchmark(
            info, benchmark, platform
        )
        return output, output_files

    def verifyBenchmarkFile(self, benchmark, filename, is_post):
        # model is now optional
        if "model" in benchmark:
            model = benchmark["model"]
            assert "files" in model, "Files field is missing in benchmark {}".format(
                filename
            )
            assert "name" in model, "Name field is missing in benchmark {}".format(
                filename
            )
            assert "format" in model, "Format field is missing in benchmark {}".format(
                filename
            )

            for f in model["files"]:
                field = model["files"][f]
                assert (
                    "filename" in field
                ), "Filename is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                assert (
                    "location" in field
                ), "Location is missing in file" + " {} of benchmark {}".format(
                    f, filename
                )
                if "md5" not in field:
                    assert not field["location"].startswith(
                        "//"
                    ), "MD5 is missing in file" + " {} of benchmark {}".format(
                        f, filename
                    )

        # tests is mandatory
        assert "tests" in benchmark, "Tests field is missing in benchmark {}".format(
            filename
        )
        tests = benchmark["tests"]

        if is_post:
            assert len(tests) == 1, (
                "After rewrite, only one test in " + "one benchmark."
            )
        else:
            assert len(tests) > 0, "Tests cannot be empty"

        is_generic_test = tests[0]["metric"] == "generic"

        for test in tests:
            assert (
                "metric" in test
            ), "Metric field is missing in " + "benchmark {}".format(filename)

            # no check is needed if the metric is generic
            if is_generic_test:
                assert test["metric"] == "generic", "All tests must be generic"
                continue

            if "iter" not in test:
                test["iter"] = -1
            if "warmup" not in test:
                test["warmup"] = -1

            assert (
                "identifier" in test
            ), "Identifier field is missing in " + "benchmark {}".format(filename)

            if "commands" in test or "command" in test or "arguments" in test:
                continue
            # for backward compatibility purpose
            assert (
                "inputs" in test
            ), "Inputs field is missing in " + "benchmark {}".format(filename)

            num = -1
            for ip_name in test["inputs"]:
                ip = test["inputs"][ip_name]
                assert "shapes" in ip, (
                    "Shapes field is missing in"
                    + " input {}".format(ip_name)
                    + " of benchmark {}".format(filename)
                )
                assert "type" in ip, "Type field is missing in input {}".format(
                    ip_name
                ) + " of benchmark {}".format(filename)
                assert isinstance(ip["shapes"], list), (
                    "Shape field should be a list. However, input "
                    + "{} of benchmark is not.".format(ip_name, filename)
                )

                dims = -1
                for item in ip["shapes"]:
                    assert isinstance(item, list), "Shapes must be a list of list."
                    if dims < 0:
                        dims = len(item)
                    else:
                        assert dims == len(item), (
                            "All shapes of one data must have " + "the same dimension"
                        )

                if num < 0:
                    num = len(ip["shapes"])
                else:
                    assert len(ip["shapes"]) == num, (
                        "The shapes of "
                        + "input {} ".format(ip_name)
                        + "are not of the same dimension in "
                        + "benchmark {}".format(filename)
                    )

    def rewriteBenchmarkTests(self, benchmark, filename):
        tests = benchmark.pop("tests")
        new_tests = self._replicateTestsOnDims(tests, filename)
        benchmark["tests"] = new_tests

    def _replicateTestsOnDims(self, tests, source):
        new_tests = []
        for test in tests:
            if "inputs" not in test:
                new_tests.append(copy.deepcopy(test))
                continue

            num = -1

            if num == 1:
                new_tests.append(copy.deepcopy(test))
            else:
                for i in range(num):
                    t = copy.deepcopy(test)
                    for ip_name in t["inputs"]:
                        t["inputs"][ip_name]["shapes"] = [
                            test["inputs"][ip_name]["shapes"][i]
                        ]
                    new_tests.append(t)
        return new_tests

    def _checkNumFiles(self, files, source, num, is_input):
        new_num = num
        ftype = "input" if is_input else "output"
        for name in files:
            fs = files[name]
            if isinstance(fs, list):
                if new_num < 0:
                    new_num = len(fs)
                else:
                    assert len(fs) == new_num, (
                        "The number of specified {} files ".format(ftype)
                        + "in blob {} do not ".format(name)
                        + "match in all input blobs in benchmark "
                        + "{}.".format(source)
                    )
            else:
                new_num = 1

        return new_num

    def composeRunCommand(
        self,
        commands,
        platform,
        programs,
        model,
        test,
        model_files,
        input_files,
        output_files,
        shared_libs,
        preprocess_files=None,
        main_command=False,
    ):
        cmds = super(Caffe2Framework, self).composeRunCommand(
            commands,
            platform,
            programs,
            model,
            test,
            model_files,
            input_files,
            output_files,
            shared_libs,
            preprocess_files,
            main_command,
        )
        if cmds:
            return cmds

        # old format, will deprecate
        cmd = [
            "--net",
            model_files["predict"],
            "--warmup",
            test["warmup"],
            "--iter",
            test["iter"],
        ]
        if "program" in programs:
            cmd = [programs["program"]] + cmd
        if "init" in model_files:
            cmd.append("--init_net")
            cmd.append(model_files["init"])
        if input_files:
            inputs = ",".join(list(input_files.keys()))
            cmd.extend(["--input_file", ",".join(list(input_files.values()))])
        else:
            inputs = ",".join(list(test["inputs"].keys()))
            input_dims = [
                ",".join([str(a) for a in test["inputs"][x]["shapes"][0]])
                for x in test["inputs"]
            ]
            input_dims = ";".join(input_dims)
            cmd.extend(["--input_dims", input_dims])
        cmd.extend(["--input", inputs])
        cmd.extend(["--input_type", list(test["inputs"].values())[0]["type"]])
        if "output_files" in test:
            outputs = ",".join(list(test["output_files"].keys()))
            cmd.extend(["--output", outputs])
            cmd.extend(["--text_output", "true"])
            cmd.extend(["--output_folder", platform.getOutputDir()])
        if "commands" in test:
            if "caffe2" in test["commands"]:
                for key in test["commands"]["caffe2"]:
                    val = test["commands"]["caffe2"][key]
                    cmd.extend(["--" + key, val])

        if shared_libs:
            cmd = [
                "export",
                "LD_LIBRARY_PATH=$\{LD_LIBRARY_PATH\}:"
                + os.path.dirname(shared_libs[0]),
                "&&",
            ] + cmd
        cmd = " ".join(str(s) for s in cmd)
        return [cmd]

    def runOnPlatform(self, total_num, cmd, platform, platform_args, converter):
        if converter is None:
            converter = {
                "name": "json_with_identifier_converter",
                "args": {"identifier": self.IDENTIFIER},
            }

        converter_obj = self.converters[converter["name"]]()
        args = converter.get("args")
        results = []
        num = 0
        while True:
            output, meta = platform.runBenchmark(cmd, platform_args=platform_args)
            one_result, valid_run_idxs = converter_obj.collect(output, args)
            valid_run_idxs = [num + idx for idx in valid_run_idxs]
            num += len(valid_run_idxs)
            results.extend(one_result)
            if getRunStatus() != 0:
                getLogger().info("Execution failed, terminating")
                break
            if num < total_num:
                num_items = len(valid_run_idxs)
                if num_items > 0:
                    getLogger().info(
                        "%d items collected, Still missing %d "
                        "runs. Collect again." % (num_items, total_num - num)
                    )

                    continue
                else:
                    getLogger().info("No new items collected, " "finish collecting...")
            elif total_num >= 0 and num > total_num:
                # if collect more than the needed number, get the
                # latest entries. This may happen when the data in
                # the previous runs are not cleared. e.g. on some
                # android 5 devices. Or, it may happen when multiple
                # runs are needed to collect the desired number of
                # iterations
                results = results[valid_run_idxs[num - total_num] :]
            break
        metric = converter_obj.convert(results)
        metric["meta"] = meta
        return metric


def process_response(self, _request, response):

"""
Middleware used for adjusting headers in a response before it is sent to the end user.
"""

from django.utils.deprecation import MiddlewareMixin
import six

class HeaderControlMiddleware(MiddlewareMixin):
    """
    Middleware that can modify/remove headers in a response.

    This can be used, for example, to remove headers i.e. drop any Vary headers to improve cache performance.
    """

"""
        Processes the given response, potentially remove or modifying headers.
        """

        for header in getattr(response, 'remove_headers', []):
            del response[header]

        for header, value in six.iteritems(getattr(response, 'force_headers', {})):
            response[header] = value

        return response


"""
        Processes the given response, potentially remove or modifying headers.
        """

        for header in getattr(response, 'remove_headers', []):
            del response[header]

        for header, value in six.iteritems(getattr(response, 'force_headers', {})):
            response[header] = value

        return response

"""
Middleware used for adjusting headers in a response before it is sent to the end user.
"""

from django.utils.deprecation import MiddlewareMixin
import six

class HeaderControlMiddleware(MiddlewareMixin):
    """
    Middleware that can modify/remove headers in a response.

    This can be used, for example, to remove headers i.e. drop any Vary headers to improve cache performance.
    """

    def process_response(self, _request, response):


"""
        Processes the given response, potentially remove or modifying headers.
        """

        for header in getattr(response, 'remove_headers', []):
            del response[header]

        for header, value in six.iteritems(getattr(response, 'force_headers', {})):
            response[header] = value

        return response


for header, value in six.iteritems(getattr(response, 'force_headers', {})):
            response[header] = value

"""
Middleware used for adjusting headers in a response before it is sent to the end user.
"""

from django.utils.deprecation import MiddlewareMixin
import six

class HeaderControlMiddleware(MiddlewareMixin):
    """
    Middleware that can modify/remove headers in a response.

    This can be used, for example, to remove headers i.e. drop any Vary headers to improve cache performance.
    """

    def process_response(self, _request, response):
        """
        Processes the given response, potentially remove or modifying headers.
        """

        for header in getattr(response, 'remove_headers', []):
            del response[header]


"""
Middleware used for adjusting headers in a response before it is sent to the end user.
"""

from django.utils.deprecation import MiddlewareMixin
import six

class HeaderControlMiddleware(MiddlewareMixin):
    """
    Middleware that can modify/remove headers in a response.

    This can be used, for example, to remove headers i.e. drop any Vary headers to improve cache performance.
    """

    def process_response(self, _request, response):
        """
        Processes the given response, potentially remove or modifying headers.
        """

        for header in getattr(response, 'remove_headers', []):
            del response[header]

        for header, value in six.iteritems(getattr(response, 'force_headers', {})):
            response[header] = value