/*---------------------------------------------------------------------------*\
  =========                 |
  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   \\    /   O peration     |
    \\  /    A nd           | www.openfoam.com
     \\/     M anipulation  |
-------------------------------------------------------------------------------
    Copyright (C) 2011-2017 OpenFOAM Foundation
    Copyright (C) 2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
    This file is part of OpenFOAM.

    OpenFOAM is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    for more details.

    You should have received a copy of the GNU General Public License
    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.

\*---------------------------------------------------------------------------*/

#include "GGAMGSolver.H"
#include "GGAMGInterface.H"
#include "GPCG.H"
#include "GPBiCGStab.H"

// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //

namespace Foam
{
    defineTypeNameAndDebug(GGAMGSolver, 0);

    gpulduMatrix::solver::addsymMatrixConstructorToTable<GGAMGSolver>
        addGGAMGSolverMatrixConstructorToTable_;

    gpulduMatrix::solver::addasymMatrixConstructorToTable<GGAMGSolver>
        addGGAMGAsymSolverMatrixConstructorToTable_;

    PageLockedBuffer<scalar> GGAMGSolver::coarseBuffer;
}


// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //

Foam::GGAMGSolver::GGAMGSolver
(
    const word& fieldName,
    const gpulduMatrix& matrix,
    const FieldField<gpuField, scalar>& interfaceBouCoeffs,
    const FieldField<gpuField, scalar>& interfaceIntCoeffs,
    const lduInterfacegpuFieldPtrsList& interfaces,
    const dictionary& solverControls
)
:
    gpulduMatrix::solver
    (
        fieldName,
        matrix,
        interfaceBouCoeffs,
        interfaceIntCoeffs,
        interfaces,
        solverControls
    ),

    // Default values for all controls
    // which may be overridden by those in controlDict
    cacheAgglomeration_(true),
    nPreSweeps_(0),
    preSweepsLevelMultiplier_(1),
    maxPreSweeps_(4),
    nPostSweeps_(2),
    postSweepsLevelMultiplier_(1),
    maxPostSweeps_(4),
    nFinestSweeps_(2),
    interpolateCorrection_(false),
    scaleCorrection_(matrix.symmetric()),
    directSolveCoarsest_(false),
    agglomeration_(GGAMGAgglomeration::New(matrix_, controlDict_)),

    matrixLevels_(agglomeration_.size()),
    primitiveInterfaceLevels_(agglomeration_.size()),
    interfaceLevels_(agglomeration_.size()),
    interfaceLevelsBouCoeffs_(agglomeration_.size()),
    interfaceLevelsIntCoeffs_(agglomeration_.size())
{
    readControls();

    forAll(agglomeration_, fineLevelIndex)
    {
        // So:
        // - pass in incorrect mesh (= fine mesh instead of coarse)
        // - pass in dummy interfaces
        agglomerateMatrix
        (
            fineLevelIndex,
            agglomeration_.meshLevel(fineLevelIndex + 1),
            agglomeration_.interfaceLevel(fineLevelIndex + 1)
        );
    }

    // Tell coarsest matrix its status
    if(agglomeration_.size())
    {
        matrixLevels_[agglomeration_.size()-1].coarsestLevel() = true;
    }


    if ((log_ >= 2) || (debug & 2))
    {
        for
        (
            label fineLevelIndex = 0;
            fineLevelIndex <= matrixLevels_.size();
            fineLevelIndex++
        )
        {
            if (fineLevelIndex == 0 || matrixLevels_.set(fineLevelIndex-1))
            {
                const gpulduMatrix& matrix = matrixLevel(fineLevelIndex);
                const lduInterfacegpuFieldPtrsList& interfaces =
                    interfaceLevel(fineLevelIndex);

                Pout<< "level:" << fineLevelIndex << nl
                    << "    nCells:" << matrix.gpuDiag().size() << nl
                    << "    nFaces:" << matrix.gpuLower().size() << nl
                    << "    nInterfaces:" << interfaces.size()
                    << endl;

                forAll(interfaces, i)
                {
                    if (interfaces.set(i))
                    {
                        Pout<< "        " << i
                            << "\ttype:" << interfaces[i].type()
                            << endl;
                    }
                }
            }
            else
            {
                Pout<< "level:" << fineLevelIndex << " : no matrix" << endl;
            }
        }
        Pout<< endl;
    }


    if (matrixLevels_.size())
    {
        const label coarsestLevel = matrixLevels_.size() - 1;

        if (matrixLevels_.set(coarsestLevel))
        {
            const gpulduMatrix& coarsestMatrix = matrixLevels_[coarsestLevel];
            label coarseSize = coarsestMatrix.lduAddr().size();
            coarsestBufferPtr_ = &coarseBuffer.buffer(coarseSize);

            if (directSolveCoarsest_)
            {
                coarsestLUMatrixPtr_.reset
                (
                    new gpuLUscalarMatrix
                    (
                        coarsestMatrix,
                        interfaceLevelsBouCoeffs_[coarsestLevel],
                        interfaceLevels_[coarsestLevel]
                    )
                );
            }
            else
            {
                entry* coarseEntry = controlDict_.findEntry
                (
                    "coarsestLevelCorr",
                    keyType::LITERAL_RECURSIVE
                );
                if (coarseEntry && coarseEntry->isDict())
                {
                    coarsestSolverPtr_ = gpulduMatrix::solver::New
                    (
                        "coarsestLevelCorr",
                        coarsestMatrix,
                        interfaceLevelsBouCoeffs_[coarsestLevel],
                        interfaceLevelsIntCoeffs_[coarsestLevel],
                        interfaceLevels_[coarsestLevel],
                        coarseEntry->dict()
                    );
                }
                else if (coarsestMatrix.asymmetric())
                {
                    coarsestSolverPtr_.reset
                    (
                        new GPBiCGStab
                        (
                            "coarsestLevelCorr",
                            coarsestMatrix,
                            interfaceLevelsBouCoeffs_[coarsestLevel],
                            interfaceLevelsIntCoeffs_[coarsestLevel],
                            interfaceLevels_[coarsestLevel],
                            GPBiCGStabSolverDict(tolerance_, relTol_)
                        )
                    );
                }
                else
                {
                    coarsestSolverPtr_.reset
                    (
                        new GPCG
                        (
                            "coarsestLevelCorr",
                            coarsestMatrix,
                            interfaceLevelsBouCoeffs_[coarsestLevel],
                            interfaceLevelsIntCoeffs_[coarsestLevel],
                            interfaceLevels_[coarsestLevel],
                            GPCGsolverDict(tolerance_, relTol_)
                        )
                    );
                }
            }
        }
    }
    else
    {
        FatalErrorInFunction
            << "No coarse levels created, either matrix too small for GAMG"
               " or nCellsInCoarsestLevel too large.\n"
               "    Either choose another solver of reduce "
               "nCellsInCoarsestLevel."
            << exit(FatalError);
    }
}


// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //

Foam::GGAMGSolver::~GGAMGSolver()
{
    if (!cacheAgglomeration_)
    {
        delete &agglomeration_;
    }
}


// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //

void Foam::GGAMGSolver::readControls()
{
    gpulduMatrix::solver::readControls();

    controlDict_.readIfPresent("cacheAgglomeration", cacheAgglomeration_);
    controlDict_.readIfPresent("nPreSweeps", nPreSweeps_);
    controlDict_.readIfPresent
    (
        "preSweepsLevelMultiplier",
        preSweepsLevelMultiplier_
    );
    controlDict_.readIfPresent("maxPreSweeps", maxPreSweeps_);
    controlDict_.readIfPresent("nPostSweeps", nPostSweeps_);
    controlDict_.readIfPresent
    (
        "postSweepsLevelMultiplier",
        postSweepsLevelMultiplier_
    );
    controlDict_.readIfPresent("maxPostSweeps", maxPostSweeps_);
    controlDict_.readIfPresent("nFinestSweeps", nFinestSweeps_);
    controlDict_.readIfPresent("interpolateCorrection", interpolateCorrection_);
    controlDict_.readIfPresent("scaleCorrection", scaleCorrection_);
    controlDict_.readIfPresent("directSolveCoarsest", directSolveCoarsest_);

    if ((log_ >= 2) || debug)
    {
        Info<< "GGAMGSolver settings :"
            << " cacheAgglomeration:" << cacheAgglomeration_
            << " nPreSweeps:" << nPreSweeps_
            << " preSweepsLevelMultiplier:" << preSweepsLevelMultiplier_
            << " maxPreSweeps:" << maxPreSweeps_
            << " nPostSweeps:" << nPostSweeps_
            << " postSweepsLevelMultiplier:" << postSweepsLevelMultiplier_
            << " maxPostSweeps:" << maxPostSweeps_
            << " nFinestSweeps:" << nFinestSweeps_
            << " interpolateCorrection:" << interpolateCorrection_
            << " scaleCorrection:" << scaleCorrection_
            //<< " directSolveCoarsest:" << directSolveCoarsest_
            << endl;
    }
}


const Foam::gpulduMatrix& Foam::GGAMGSolver::matrixLevel(const label i) const
{
    if (i == 0)
    {
        return matrix_;
    }
    else
    {
        return matrixLevels_[i - 1];
    }
}


const Foam::lduInterfacegpuFieldPtrsList& Foam::GGAMGSolver::interfaceLevel
(
    const label i
) const
{
    if (i == 0)
    {
        return interfaces_;
    }
    else
    {
        return interfaceLevels_[i - 1];
    }
}


const Foam::FieldField<Foam::gpuField, Foam::scalar>&
Foam::GGAMGSolver::interfaceBouCoeffsLevel
(
    const label i
) const
{
    if (i == 0)
    {
        return interfaceBouCoeffs_;
    }
    else
    {
        return interfaceLevelsBouCoeffs_[i - 1];
    }
}


const Foam::FieldField<Foam::gpuField, Foam::scalar>&
Foam::GGAMGSolver::interfaceIntCoeffsLevel
(
    const label i
) const
{
    if (i == 0)
    {
        return interfaceIntCoeffs_;
    }
    else
    {
        return interfaceLevelsIntCoeffs_[i - 1];
    }
}


// ************************************************************************* //
