// Copyright (C) 2016  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.
#ifndef DLIB_DNn_VALIDATION_H_
#define DLIB_DNn_VALIDATION_H_

#include "../svm/cross_validate_object_detection_trainer_abstract.h"
#include "../svm/cross_validate_object_detection_trainer.h"
#include "layers.h"

namespace dlib
{

    template <
        typename SUBNET,
        typename image_array_type
        >
    const matrix<double,1,3> test_object_detection_function (
        loss_mmod<SUBNET>& detector,
        const image_array_type& images,
        const std::vector<std::vector<mmod_rect>>& truth_dets,
        const test_box_overlap& overlap_tester = test_box_overlap(),
        const double adjust_threshold = 0
    )
    {
        // make sure requires clause is not broken
        DLIB_CASSERT( is_learning_problem(images,truth_dets) == true , 
                    "\t matrix test_object_detection_function()"
                    << "\n\t invalid inputs were given to this function"
                    << "\n\t is_learning_problem(images,truth_dets): " << is_learning_problem(images,truth_dets)
                    << "\n\t images.size(): " << images.size() 
                    );



        double correct_hits = 0;
        double total_true_targets = 0;

        std::vector<std::pair<double,bool> > all_dets;
        unsigned long missing_detections = 0;

        resizable_tensor temp;

        for (unsigned long i = 0; i < images.size(); ++i)
        {
            std::vector<mmod_rect> hits; 
            detector.to_tensor(&images[i], &images[i]+1, temp);
            detector.subnet().forward(temp);
            detector.loss_details().to_label(temp, detector.subnet(), &hits, adjust_threshold);


            std::vector<full_object_detection> truth_boxes;
            std::vector<rectangle> ignore;
            std::vector<std::pair<double,rectangle>> boxes;
            // copy hits and truth_dets into the above three objects
            for (auto&& b : truth_dets[i])
            {
                if (b.ignore)
                    ignore.push_back(b);
                else
                    truth_boxes.push_back(full_object_detection(b.rect));
            }
            for (auto&& b : hits)
                boxes.push_back(std::make_pair(b.detection_confidence, b.rect));

            correct_hits += impl::number_of_truth_hits(truth_boxes, ignore, boxes, overlap_tester, all_dets, missing_detections);
            total_true_targets += truth_boxes.size();
        }

        std::sort(all_dets.rbegin(), all_dets.rend());

        double precision, recall;

        double total_hits = all_dets.size();

        if (total_hits == 0)
            precision = 1;
        else
            precision = correct_hits / total_hits;

        if (total_true_targets == 0)
            recall = 1;
        else
            recall = correct_hits / total_true_targets;

        matrix<double, 1, 3> res;
        res = precision, recall, average_precision(all_dets, missing_detections);
        return res;
    }

// ----------------------------------------------------------------------------------------

}

#endif // DLIB_DNn_VALIDATION_H_

