try {
        auto status = _enqueueDocumentsFn(firstDocToApply, documents.cend(), info);
        if (!status.isOK()) {
            return status;
        }
    } catch (const DBException& e) {
        return e.toStatus().withContext("Error inserting documents into oplog buffer collection");
    }

/**
 *
 */

#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplication

#include "mongo/db/repl/oplog_fetcher.h"

#include "mongo/base/counter.h"
#include "mongo/bson/mutable/document.h"
#include "mongo/db/commands/server_status_metric.h"
#include "mongo/db/jsobj.h"
#include "mongo/db/matcher/matcher.h"
#include "mongo/db/pipeline/document_source_find_and_modify_image_lookup.h"
#include "mongo/db/pipeline/document_source_match.h"
#include "mongo/db/repl/repl_server_parameters_gen.h"
#include "mongo/db/repl/replication_auth.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/stats/timer_stats.h"
#include "mongo/logv2/log.h"
#include "mongo/rpc/metadata/oplog_query_metadata.h"
#include "mongo/s/resharding/resume_token_gen.h"
#include "mongo/util/assert_util.h"
#include "mongo/util/fail_point.h"
#include "mongo/util/time_support.h"

namespace mongo {
namespace repl {

MONGO_FAIL_POINT_DEFINE(stopReplProducer);
MONGO_FAIL_POINT_DEFINE(stopReplProducerOnDocument);
MONGO_FAIL_POINT_DEFINE(setSmallOplogGetMoreMaxTimeMS);
MONGO_FAIL_POINT_DEFINE(hangAfterOplogFetcherCallbackScheduled);
MONGO_FAIL_POINT_DEFINE(hangBeforeStartingOplogFetcher);
MONGO_FAIL_POINT_DEFINE(hangBeforeOplogFetcherRetries);
MONGO_FAIL_POINT_DEFINE(hangBeforeProcessingSuccessfulBatch);
MONGO_FAIL_POINT_DEFINE(hangOplogFetcherBeforeAdvancingLastFetched);

namespace {
class OplogBatchStats {
public:
    void recordMillis(int millis, bool isEmptyBatch);
    BSONObj getReport() const;
    operator BSONObj() const {
        return getReport();
    }

private:
    TimerStats _getMores;
    Counter64 _numEmptyBatches;
};

void OplogBatchStats::recordMillis(int millis, bool isEmptyBatch) {
    _getMores.recordMillis(millis);
    if (isEmptyBatch) {
        _numEmptyBatches.increment();
    }
}

BSONObj OplogBatchStats::getReport() const {
    BSONObjBuilder b(_getMores.getReport());
    b.append("numEmptyBatches", _numEmptyBatches.get());
    return b.obj();
}

// The number and time spent reading batches off the network
OplogBatchStats oplogBatchStats;
ServerStatusMetricField<OplogBatchStats> displayBatchesRecieved("repl.network.getmores",
                                                                &oplogBatchStats);
// The oplog entries read via the oplog reader
Counter64 opsReadStats;
ServerStatusMetricField<Counter64> displayOpsRead("repl.network.ops", &opsReadStats);
// The bytes read via the oplog reader
Counter64 networkByteStats;
ServerStatusMetricField<Counter64> displayBytesRead("repl.network.bytes", &networkByteStats);

Counter64 readersCreatedStats;
ServerStatusMetricField<Counter64> displayReadersCreated("repl.network.readersCreated",
                                                         &readersCreatedStats);

const Milliseconds maximumAwaitDataTimeoutMS(30 * 1000);

/**
 * Calculates await data timeout based on the current replica set configuration.
 */
Milliseconds calculateAwaitDataTimeout(const ReplSetConfig& config) {
    if (MONGO_unlikely(setSmallOplogGetMoreMaxTimeMS.shouldFail())) {
        return Milliseconds(50);
    }

    // Under protocol version 1, make the awaitData timeout (maxTimeMS) dependent on the election
    // timeout. This enables the sync source to communicate liveness of the primary to secondaries.
    // We never wait longer than 30 seconds.
    return std::min((config.getElectionTimeoutPeriod() / 2), maximumAwaitDataTimeoutMS);
}
}  // namespace

StatusWith<OplogFetcher::DocumentsInfo> OplogFetcher::validateDocuments(
    const OplogFetcher::Documents& documents,
    bool first,
    Timestamp lastTS,
    StartingPoint startingPoint) {
    if (first && documents.empty()) {
        return Status(ErrorCodes::OplogStartMissing,
                      str::stream() << "The first batch of oplog entries is empty, but expected at "
                                       "least 1 document matching ts: "
                                    << lastTS.toString());
    }
    DocumentsInfo info;
    // The count of the bytes of the documents read off the network.
    info.networkDocumentBytes = 0;
    info.networkDocumentCount = 0;
    for (auto&& doc : documents) {
        info.networkDocumentBytes += doc.objsize();
        ++info.networkDocumentCount;

        // If this is the first response (to the $gte query) then we already applied the first doc.
        if (first && info.networkDocumentCount == 1U) {
            continue;
        }

        auto docOpTime = OpTime::parseFromOplogEntry(doc);
        if (!docOpTime.isOK()) {
            return docOpTime.getStatus();
        }
        info.lastDocument = docOpTime.getValue();

        // Check to see if the oplog entry goes back in time for this document.
        const auto docTS = info.lastDocument.getTimestamp();
        if (lastTS >= docTS) {
            return Status(ErrorCodes::OplogOutOfOrder,
                          str::stream() << "Out of order entries in oplog. lastTS: "
                                        << lastTS.toString() << " outOfOrderTS:" << docTS.toString()
                                        << " in batch with " << info.networkDocumentCount
                                        << "docs; first-batch:" << first << ", doc:" << doc);
        }
        lastTS = docTS;
    }

    // These numbers are for the documents we will apply.
    info.toApplyDocumentCount = documents.size();
    info.toApplyDocumentBytes = info.networkDocumentBytes;
    if (first && startingPoint == StartingPoint::kSkipFirstDoc) {
        // The count is one less since the first document found was already applied ($gte $ts query)
        // and we will not apply it again.
        --info.toApplyDocumentCount;
        auto alreadyAppliedDocument = documents.cbegin();
        info.toApplyDocumentBytes -= alreadyAppliedDocument->objsize();
    }

    return info;
}

OplogFetcher::OplogFetcher(executor::TaskExecutor* executor,
                           std::unique_ptr<OplogFetcherRestartDecision> oplogFetcherRestartDecision,
                           DataReplicatorExternalState* dataReplicatorExternalState,
                           EnqueueDocumentsFn enqueueDocumentsFn,
                           OnShutdownCallbackFn onShutdownCallbackFn,
                           Config config)
    : AbstractAsyncComponent(executor, config.name),
      _receivedRBID(config.requiredRBID),
      _oplogFetcherRestartDecision(std::move(oplogFetcherRestartDecision)),
      _onShutdownCallbackFn(onShutdownCallbackFn),
      _lastFetched(config.initialLastFetched),
      _createClientFn(
          [] { return std::make_unique<DBClientConnection>(true /* autoReconnect */); }),
      _dataReplicatorExternalState(dataReplicatorExternalState),
      _enqueueDocumentsFn(enqueueDocumentsFn),
      _awaitDataTimeout(calculateAwaitDataTimeout(config.replSetConfig)),
      _config(std::move(config)) {
    invariant(_config.replSetConfig.isInitialized());
    invariant(!_lastFetched.isNull());
    invariant(onShutdownCallbackFn);
    invariant(enqueueDocumentsFn);
}

OplogFetcher::~OplogFetcher() {
    shutdown();
    join();
}

void OplogFetcher::setConnection(std::unique_ptr<DBClientConnection>&& _connectedClient) {
    // Can only call this once, before startup.
    invariant(!_conn);
    _conn = std::move(_connectedClient);
}

Status OplogFetcher::_doStartup_inlock() noexcept {
    return _scheduleWorkAndSaveHandle_inlock(
        [this](const executor::TaskExecutor::CallbackArgs& args) {
            // Tests use this failpoint to prevent the oplog fetcher from starting.  If those
            // tests fail and the oplog fetcher is canceled, we want to continue so we see
            // a test failure quickly instead of a test timeout eventually.
            while (hangBeforeStartingOplogFetcher.shouldFail() && !args.myHandle.isCanceled()) {
                sleepmillis(100);
            }
            _runQuery(args);
        },
        &_runQueryHandle,
        "_runQuery");
}

void OplogFetcher::_doShutdown_inlock() noexcept {
    _cancelHandle_inlock(_runQueryHandle);

    if (_conn) {
        _conn->shutdownAndDisallowReconnect();
    }
    _shutdownCondVar.notify_all();
}

Mutex* OplogFetcher::_getMutex() noexcept {
    return &_mutex;
}

std::string OplogFetcher::toString() {
    stdx::lock_guard lock(_mutex);
    str::stream output;
    output << "OplogFetcher -";
    output << " last optime fetched: " << _lastFetched.toString();
    output << " source: " << _config.source.toString();
    output << " namespace: " << _nss.toString();
    output << " active: " << _isActive_inlock();
    output << " shutting down?:" << _isShuttingDown_inlock();
    output << " first batch: " << _firstBatch;
    output << " initial find timeout: " << _getInitialFindMaxTime();
    output << " retried find timeout: " << _getRetriedFindMaxTime();
    output << " awaitData timeout: " << _awaitDataTimeout;
    return output;
}

OplogFetcher::StartingPoint OplogFetcher::getStartingPoint_forTest() const {
    return _config.startingPoint;
}

OpTime OplogFetcher::getLastOpTimeFetched_forTest() const {
    return _getLastOpTimeFetched();
}

BSONObj OplogFetcher::getFindQueryFilter_forTest() const {
    return _makeFindQueryFilter();
}

Query OplogFetcher::getFindQuerySettings_forTest(long long findTimeout) const {
    return _makeFindQuerySettings(findTimeout);
}

Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const {
    return _awaitDataTimeout;
}

void OplogFetcher::setCreateClientFn_forTest(const CreateClientFn& createClientFn) {
    stdx::lock_guard lock(_mutex);
    _createClientFn = createClientFn;
}

DBClientConnection* OplogFetcher::getDBClientConnection_forTest() const {
    stdx::lock_guard lock(_mutex);
    return _conn.get();
}

Milliseconds OplogFetcher::getInitialFindMaxTime_forTest() const {
    return _getInitialFindMaxTime();
}

Milliseconds OplogFetcher::getRetriedFindMaxTime_forTest() const {
    return _getRetriedFindMaxTime();
}

void OplogFetcher::_setSocketTimeout(long long timeout) {
    stdx::lock_guard<Latch> lock(_mutex);
    invariant(_conn);
    // setSoTimeout takes a double representing the number of seconds for send and receive
    // timeouts. Thus, we must express the timeout in milliseconds and divide by 1000.0 to get
    // the number of seconds with a fractional part.
    _conn->setSoTimeout(timeout / 1000.0 + oplogNetworkTimeoutBufferSeconds.load());
}

OpTime OplogFetcher::_getLastOpTimeFetched() const {
    stdx::lock_guard<Latch> lock(_mutex);
    return _lastFetched;
}

Milliseconds OplogFetcher::_getInitialFindMaxTime() const {
    return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000);
}

Milliseconds OplogFetcher::_getRetriedFindMaxTime() const {
    return Milliseconds(oplogRetriedFindMaxSeconds.load() * 1000);
}

void OplogFetcher::_finishCallback(Status status) {
    invariant(isActive());
    // If the oplog fetcher is shutting down, consolidate return code to CallbackCanceled.
    if (_isShuttingDown() && status != ErrorCodes::CallbackCanceled) {
        status = Status(ErrorCodes::CallbackCanceled,
                        str::stream() << "Got error: \"" << status.toString()
                                      << "\" while oplog fetcher is shutting down");
    }
    _onShutdownCallbackFn(status, _receivedRBID);

    decltype(_onShutdownCallbackFn) onShutdownCallbackFn;
    decltype(_oplogFetcherRestartDecision) oplogFetcherRestartDecision;
    stdx::lock_guard<Latch> lock(_mutex);
    _transitionToComplete_inlock();

    // Release any resources that might be held by the '_onShutdownCallbackFn' function object.
    // The function object will be destroyed outside the lock since the temporary variable
    // 'onShutdownCallbackFn' is declared before 'lock'.
    invariant(_onShutdownCallbackFn);
    std::swap(_onShutdownCallbackFn, onShutdownCallbackFn);

    // Release any resources held by the OplogFetcherRestartDecision.
    invariant(_oplogFetcherRestartDecision);
    std::swap(_oplogFetcherRestartDecision, oplogFetcherRestartDecision);
}

void OplogFetcher::_runQuery(const executor::TaskExecutor::CallbackArgs& callbackData) noexcept {
    Status responseStatus =
        _checkForShutdownAndConvertStatus(callbackData, "error running oplog fetcher");
    if (!responseStatus.isOK()) {
        _finishCallback(responseStatus);
        return;
    }

    bool hadExistingConnection = true;
    {
        stdx::lock_guard<Latch> lock(_mutex);
        if (!_conn) {
            _conn = _createClientFn();
            hadExistingConnection = false;
        }
    }

    hangAfterOplogFetcherCallbackScheduled.pauseWhileSet();

    if (!hadExistingConnection) {
        auto connectStatus = _connect();

        // Error out if we failed to connect after exhausting the allowed retry attempts.
        if (!connectStatus.isOK()) {
            _finishCallback(connectStatus);
            return;
        }
    }

    _setMetadataWriterAndReader();
    auto cursorStatus = _createNewCursor(true /* initialFind */);
    if (!cursorStatus.isOK()) {
        invariant(_config.forTenantMigration);
        // If we are a TenantOplogFetcher, we never retry as we will always restart the entire
        // TenantMigrationRecipient state machine on failure. So instead, we just fail and exit.
        _finishCallback(cursorStatus);
        return;
    }
    while (true) {
        Status status{Status::OK()};
        {
            // Both of these checks need to happen while holding the mutex since they could race
            // with shutdown.
            stdx::lock_guard<Latch> lock(_mutex);
            if (_isShuttingDown_inlock()) {
                status = {ErrorCodes::CallbackCanceled, "oplog fetcher shutting down"};
            } else if (_runQueryHandle.isCanceled()) {
                invariant(_getExecutor()->isShuttingDown());
                status = {ErrorCodes::CallbackCanceled, "oplog fetcher task executor shutdown"};
            }
        }
        if (!status.isOK()) {
            _finishCallback(status);
            return;
        }

        auto batchResult = _getNextBatch();
        if (!batchResult.isOK()) {
            auto brStatus = batchResult.getStatus();

            // Recreate a cursor if we have enough retries left.
            // If we are a TenantOplogFetcher, we never retry as we will always restart the
            // TenantMigrationRecipient state machine on failure. So instead, we just fail and exit.
            if (_oplogFetcherRestartDecision->shouldContinue(this, brStatus) &&
                !_config.forTenantMigration) {
                hangBeforeOplogFetcherRetries.pauseWhileSet();
                _cursor.reset();
                continue;
            } else {
                _finishCallback(brStatus);
                return;
            }
        }

        // This will advance our view of _lastFetched.
        status = _onSuccessfulBatch(batchResult.getValue());
        if (!status.isOK()) {
            // The stopReplProducer fail point expects this to return successfully. If another fail
            // point wants this to return unsuccessfully, it should use a different error code.
            if (status == ErrorCodes::FailPointEnabled) {
                _finishCallback(Status::OK());
                return;
            }

            _finishCallback(status);
            return;
        }

        if (_cursor->isDead()) {
            if (!_cursor->tailable()) {
                try {
                    auto opCtx = cc().makeOperationContext();
                    stdx::unique_lock<Latch> lk(_mutex);
                    // Wait a little before re-running the aggregation command on the donor's
                    // oplog. We are not actually intending to wait for shutdown here, we use
                    // this as a way to wait while still being able to be interrupted outside of
                    // primary-only service shutdown.
                    opCtx->waitForConditionOrInterruptFor(_shutdownCondVar,
                                                          lk,
                                                          _awaitDataTimeout,
                                                          [&] { return _isShuttingDown_inlock(); });
                    _cursor.reset();
                    continue;
                } catch (const DBException& e) {
                    _finishCallback(e.toStatus().withContext(
                        "Interrupted while waiting to create a new aggregation cursor"));
                    return;
                }
            } else {
                // This means the sync source closes the tailable cursor with a returned cursorId of
                // 0. Any users of the oplog fetcher should create a new oplog fetcher if they see a
                // successful status and would like to continue fetching more oplog entries.
                _finishCallback(Status::OK());
                return;
            }
        }
    }
}

Status OplogFetcher::_connect() {
    Status connectStatus = Status::OK();
    do {
        if (_isShuttingDown()) {
            return Status(ErrorCodes::CallbackCanceled, "oplog fetcher shutting down");
        }
        connectStatus = [&] {
            try {
                if (!connectStatus.isOK()) {
                    // If this is a retry, let the DBClientConnection handle the reconnect itself
                    // for proper backoff behavior.
                    LOGV2(23437,
                          "OplogFetcher reconnecting due to error: {error}",
                          "OplogFetcher reconnecting due to error",
                          "error"_attr = connectStatus);
                    _conn->checkConnection();
                } else {
                    uassertStatusOK(_conn->connect(_config.source, "OplogFetcher", boost::none));
                }
                uassertStatusOK(replAuthenticate(_conn.get())
                                    .withContext(str::stream()
                                                 << "OplogFetcher failed to authenticate to "
                                                 << _config.source));
                // Reset any state needed to track restarts on successful connection.
                _oplogFetcherRestartDecision->fetchSuccessful(this);
                return Status::OK();
            } catch (const DBException& e) {
                hangBeforeOplogFetcherRetries.pauseWhileSet();
                return e.toStatus();
            }
        }();
    } while (!connectStatus.isOK() &&
             _oplogFetcherRestartDecision->shouldContinue(this, connectStatus));

    return connectStatus;
}

void OplogFetcher::_setMetadataWriterAndReader() {
    invariant(_conn);

    _vectorClockMetadataHook =
        std::make_unique<rpc::VectorClockMetadataHook>(getGlobalServiceContext());

    _conn->setRequestMetadataWriter([this](OperationContext* opCtx, BSONObjBuilder* metadataBob) {
        *metadataBob << rpc::kReplSetMetadataFieldName << 1;
        *metadataBob << rpc::kOplogQueryMetadataFieldName << 1;
        metadataBob->appendElements(ReadPreferenceSetting::secondaryPreferredMetadata());

        // Run VectorClockMetadataHook on request metadata so this matches the behavior of
        // the connections in the replication coordinator thread pool.
        return _vectorClockMetadataHook->writeRequestMetadata(opCtx, metadataBob);
    });

    _conn->setReplyMetadataReader(
        [this](OperationContext* opCtx, const BSONObj& metadataObj, StringData source) {
            _metadataObj = metadataObj.getOwned();

            // Run VectorClockMetadataHook on reply metadata so this matches the behavior of the
            // connections in the replication coordinator thread pool.
            return _vectorClockMetadataHook->readReplyMetadata(opCtx, source, _metadataObj);
        });
}

AggregateCommandRequest OplogFetcher::_makeAggregateCommandRequest(long long maxTimeMs,
                                                                   Timestamp startTs) const {
    auto opCtx = cc().makeOperationContext();
    StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces;
    boost::intrusive_ptr<ExpressionContext> expCtx =
        make_intrusive<ExpressionContext>(opCtx.get(),
                                          boost::none, /* explain */
                                          false,       /* fromMongos */
                                          false,       /* needsMerge */
                                          true,        /* allowDiskUse */
                                          true,        /* bypassDocumentValidation */
                                          false,       /* isMapReduceCommand */
                                          _nss,
                                          boost::none, /* runtimeConstants */
                                          nullptr,     /* collator */
                                          MongoProcessInterface::create(opCtx.get()),
                                          std::move(resolvedNamespaces),
                                          boost::none); /* collUUID */
    Pipeline::SourceContainer stages;
    // Match oplog entries greater than or equal to the last fetched timestamp.
    BSONObjBuilder builder(BSON("$or" << BSON_ARRAY(_config.queryFilter << BSON("ts" << startTs))));
    builder.append("ts", BSON("$gte" << startTs));
    stages.emplace_back(DocumentSourceMatch::createFromBson(
        Document{{"$match", Document{builder.obj()}}}.toBson().firstElement(), expCtx));
    stages.emplace_back(DocumentSourceFindAndModifyImageLookup::create(expCtx));
    // Do another filter on the timestamp as the 'FindAndModifyImageLookup' stage can forge
    // synthetic no-op entries if the oplog corresponding to the 'startTs' is a retryable
    BSONObjBuilder secondMatchBuilder(BSON("ts" << BSON("$gte" << startTs)));
    stages.emplace_back(DocumentSourceMatch::createFromBson(
        Document{{"$match", Document{secondMatchBuilder.obj()}}}.toBson().firstElement(), expCtx));
    const auto serializedPipeline = Pipeline::create(std::move(stages), expCtx)->serializeToBson();

    AggregateCommandRequest aggRequest(_nss, std::move(serializedPipeline));
    aggRequest.setReadConcern(_config.queryReadConcern.toBSONInner());
    aggRequest.setMaxTimeMS(maxTimeMs);
    if (_config.requestResumeToken) {
        aggRequest.setHint(BSON("$natural" << 1));
        aggRequest.setRequestReshardingResumeToken(true);
    }
    if (_config.batchSize) {
        SimpleCursorOptions cursor;
        cursor.setBatchSize(_config.batchSize);
        aggRequest.setCursor(cursor);
    }
    aggRequest.setWriteConcern(WriteConcernOptions());
    return aggRequest;
}

BSONObj OplogFetcher::_makeFindQueryFilter() const {
    BSONObjBuilder queryBob;

    auto lastOpTimeFetched = _getLastOpTimeFetched();
    BSONObjBuilder filterBob;
    filterBob.append("ts", BSON("$gte" << lastOpTimeFetched.getTimestamp()));
    // Handle caller-provided filter.
    if (!_config.queryFilter.isEmpty()) {
        filterBob.append(
            "$or",
            BSON_ARRAY(_config.queryFilter << BSON("ts" << lastOpTimeFetched.getTimestamp())));
    }
    return filterBob.obj();
}

Query OplogFetcher::_makeFindQuerySettings(long long findTimeout) const {
    Query query = Query().maxTimeMS(findTimeout);
    if (_config.requestResumeToken) {
        query.hint(BSON("$natural" << 1)).requestResumeToken(true);
    }

    auto lastCommittedWithCurrentTerm =
        _dataReplicatorExternalState->getCurrentTermAndLastCommittedOpTime();
    auto term = lastCommittedWithCurrentTerm.value;
    if (term != OpTime::kUninitializedTerm) {
        query.term(term);
    }

    if (_config.queryReadConcern.isEmpty()) {
        // This ensures that the sync source waits for all earlier oplog writes to be visible.
        // Since Timestamp(0, 0) isn't allowed, Timestamp(0, 1) is the minimal we can use.
        query.readConcern(BSON("level"
                               << "local"
                               << "afterClusterTime" << Timestamp(0, 1)));
    } else {
        // Caller-provided read concern.
        query.appendElements(_config.queryReadConcern.toBSON());
    }

    return query;
}

Status OplogFetcher::_createNewCursor(bool initialFind) {
    invariant(_conn);

    // Set the socket timeout to the 'find' timeout plus a network buffer.
    auto maxTimeMs = durationCount<Milliseconds>(initialFind ? _getInitialFindMaxTime()
                                                             : _getRetriedFindMaxTime());
    _setSocketTimeout(maxTimeMs);

    if (_config.forTenantMigration) {
        // We set 'secondaryOk'=false here to avoid duplicating OP_MSG fields since we already
        // set the request metadata readPreference to `secondaryPreferred`.
        auto ret = DBClientCursor::fromAggregationRequest(
            _conn.get(),
            _makeAggregateCommandRequest(maxTimeMs, _getLastOpTimeFetched().getTimestamp()),
            false /* secondaryOk */,
            oplogFetcherUsesExhaust);
        if (!ret.isOK()) {
            LOGV2_DEBUG(5761701,
                        2,
                        "Failed to create aggregation cursor in TenantOplogFetcher",
                        "status"_attr = ret.getStatus());
            return ret.getStatus();
        }
        _cursor = std::move(ret.getValue());
    } else {
        _cursor = std::make_unique<DBClientCursor>(
            _conn.get(),
            _nss,
            _makeFindQueryFilter(),
            _makeFindQuerySettings(maxTimeMs),
            0 /* nToReturn */,
            0 /* nToSkip */,
            nullptr /* fieldsToReturn */,
            QueryOption_CursorTailable | QueryOption_AwaitData |
                (oplogFetcherUsesExhaust ? QueryOption_Exhaust : 0),
            _config.batchSize);
    }

    _firstBatch = true;

    readersCreatedStats.increment();
    return Status::OK();
}

StatusWith<OplogFetcher::Documents> OplogFetcher::_getNextBatch() {
    Documents batch;
    try {
        Timer timer;
        if (!_cursor) {
            // An error occurred and we should recreate the cursor.
            // The OplogFetcher uses an aggregation command in tenant migrations, which does not
            // support tailable cursors. When recreating the cursor, use the longer initial max time
            // to avoid timing out.
            const bool initialFind = _config.forTenantMigration;
            auto status = _createNewCursor(initialFind /* initialFind */);
            if (!status.isOK()) {
                return status;
            }
        }
        // If it is the first batch, we should initialize the cursor, which will run the find query.
        // Otherwise we should call more() to get the next batch.
        if (_firstBatch) {
            // Network errors manifest as exceptions that are handled in the catch block. If init
            // returns false it means that the sync source responded with nothing, which could
            // indicate a problem with the sync source.
            // We can't call `init()` when using an aggregate command because
            // `DBClientCursor::fromAggregationRequest` will already have processed the `cursorId`.
            if (!_config.forTenantMigration && !_cursor->init()) {
                _cursor.reset();
                return {ErrorCodes::InvalidSyncSource,
                        str::stream() << "Oplog fetcher could not create cursor on source: "
                                      << _config.source};
            }

            // Aggregate commands do not support tailable cursors outside of change streams.
            if (!_config.forTenantMigration) {
                // This will also set maxTimeMS on the generated getMore command.
                _cursor->setAwaitDataTimeoutMS(_awaitDataTimeout);
            }

            // The 'find' command has already been executed, so reset the socket timeout to reflect
            // the awaitData timeout with a network buffer.
            _setSocketTimeout(durationCount<Milliseconds>(_awaitDataTimeout));

            // TODO SERVER-46240: Handle batchSize 1 in DBClientCursor.
            // Due to a bug in DBClientCursor, it actually uses batchSize 2 if the given batchSize
            // is 1 for the find command. So if the given batchSize is 1, we need to set it
            if (_config.batchSize == 1) {
                _cursor->setBatchSize(_config.batchSize);
            }
        } else {
            auto lastCommittedWithCurrentTerm =
                _dataReplicatorExternalState->getCurrentTermAndLastCommittedOpTime();
            if (lastCommittedWithCurrentTerm.value != OpTime::kUninitializedTerm) {
                _cursor->setCurrentTermAndLastCommittedOpTime(lastCommittedWithCurrentTerm.value,
                                                              lastCommittedWithCurrentTerm.opTime);
            }
            _cursor->more();
        }
        while (_cursor->moreInCurrentBatch()) {
            batch.emplace_back(_cursor->nextSafe());
        }

        // This value is only used on a successful batch for metrics.repl.network.getmores. This
        // metric intentionally tracks the time taken by the initial find as well.
        _lastBatchElapsedMS = timer.millis();
    } catch (const DBException& ex) {
        if (_cursor->connectionHasPendingReplies()) {
            // Close the connection because the connection cannot be used anymore as more data is on
            // the way from the server for the exhaust stream. Thus, we have to reconnect. The
            // DBClientConnection does autoreconnect on the next network operation.
            _conn->shutdown();
        }
        return ex.toStatus("Error while getting the next batch in the oplog fetcher");
    }

    return batch;
}

Status OplogFetcher::_onSuccessfulBatch(const Documents& documents) {
    hangBeforeProcessingSuccessfulBatch.pauseWhileSet();

    if (_isShuttingDown()) {
        return Status(ErrorCodes::CallbackCanceled, "oplog fetcher shutting down");
    }

    _oplogFetcherRestartDecision->fetchSuccessful(this);

    // Stop fetching and return on fail point.
    // This fail point makes the oplog fetcher ignore the downloaded batch of operations and not
    // error out. The FailPointEnabled error will be caught by the caller.
    if (MONGO_unlikely(stopReplProducer.shouldFail())) {
        return Status(ErrorCodes::FailPointEnabled, "stopReplProducer fail point is enabled");
    }

    // Stop fetching and return when we reach a particular document. This failpoint should be used
    // with the setParameter bgSyncOplogFetcherBatchSize=1, so that documents are fetched one at a
    {
        Status status = Status::OK();
        stopReplProducerOnDocument.executeIf(
            [&](auto&&) {
                static constexpr char message[] =
                    "stopReplProducerOnDocument fail point is enabled";
                LOGV2(21269, message);
                status = {ErrorCodes::FailPointEnabled, message};
            },
            [&](const BSONObj& data) {
                auto opCtx = cc().makeOperationContext();
                boost::intrusive_ptr<ExpressionContext> expCtx(
                    new ExpressionContext(opCtx.get(), nullptr, _nss));
                Matcher m(data["document"].Obj(), expCtx);
                // TODO SERVER-46240: Handle batchSize 1 in DBClientCursor.
                // Due to a bug in DBClientCursor, it actually uses batchSize 2 if the given
                // batchSize is 1 for the find command. So we need to check up to two documents.
                return !documents.empty() &&
                    (m.matches(documents.front()["o"].Obj()) ||
                     m.matches(documents.back()["o"].Obj()));
            });
        if (!status.isOK()) {
            return status;
        }
    }

    auto firstDocToApply = documents.cbegin();

    if (!documents.empty()) {
        LOGV2_DEBUG(21270,
                    2,
                    "oplog fetcher read {batchSize} operations from remote oplog starting at "
                    "{firstTimestamp} and ending at {lastTimestamp}",
                    "Oplog fetcher read batch from remote oplog",
                    "batchSize"_attr = documents.size(),
                    "firstTimestamp"_attr = documents.front()["ts"],
                    "lastTimestamp"_attr = documents.back()["ts"]);
    } else {
        LOGV2_DEBUG(21271, 2, "Oplog fetcher read 0 operations from remote oplog");
    }

    auto oqMetadataResult = rpc::OplogQueryMetadata::readFromMetadata(_metadataObj);
    if (!oqMetadataResult.isOK()) {
        LOGV2_ERROR(21278,
                    "invalid oplog query metadata from sync source {syncSource}: "
                    "{error}: {metadata}",
                    "Invalid oplog query metadata from sync source",
                    "syncSource"_attr = _config.source,
                    "error"_attr = oqMetadataResult.getStatus(),
                    "metadata"_attr = _metadataObj);
        return oqMetadataResult.getStatus();
    }
    auto oqMetadata = oqMetadataResult.getValue();

    if (_firstBatch) {
        auto status =
            _checkRemoteOplogStart(documents, oqMetadata.getLastOpApplied(), oqMetadata.getRBID());
        if (!status.isOK()) {
            // Stop oplog fetcher and execute rollback if necessary.
            return status;
        }

        LOGV2_DEBUG(21272,
                    1,
                    "oplog fetcher successfully fetched from {syncSource}",
                    "Oplog fetcher successfully fetched from sync source",
                    "syncSource"_attr = _config.source);

        // We do not always enqueue the first document. We elect to skip it for the following
        //    1. This is the first batch and no rollback is needed. Callers specify
        //       StartingPoint::kSkipFirstDoc when they want this behavior.
        //    2. We have already enqueued that document in a previous attempt. We can get into
        //       this situation if we had a batch with StartingPoint::kEnqueueFirstDoc that failed
        //       right after that first document was enqueued. In such a scenario, we would not
        //       have advanced the lastFetched opTime, so we skip past that document to avoid
        //       duplicating it.
        //    3. We have a query filter, and the first document doesn't match that filter.  This
        //       happens on the first batch when we always accept a document with the previous
        //       fetched timestamp.

        if (_config.startingPoint == StartingPoint::kSkipFirstDoc) {
            firstDocToApply++;
        } else if (!_config.queryFilter.isEmpty()) {
            auto opCtx = cc().makeOperationContext();
            auto expCtx =
                make_intrusive<ExpressionContext>(opCtx.get(), nullptr /* collator */, _nss);
            Matcher m(_config.queryFilter, expCtx);
            if (!m.matches(*firstDocToApply))
                firstDocToApply++;
        }
    }

    // This lastFetched value is the last OpTime from the previous batch.
    auto previousOpTimeFetched = _getLastOpTimeFetched();

    auto validateResult = OplogFetcher::validateDocuments(
        documents, _firstBatch, previousOpTimeFetched.getTimestamp(), _config.startingPoint);
    if (!validateResult.isOK()) {
        return validateResult.getStatus();
    }
    auto info = validateResult.getValue();
    // If the batch is empty, set 'lastDocOpTime' to the lastFetched from the previous batch.
    auto lastDocOpTime = info.lastDocument.isNull() ? previousOpTimeFetched : info.lastDocument;

    // Process replset metadata.  It is important that this happen after we've validated the
    // first batch, so we don't progress our knowledge of the commit point from a
    // response that triggers a rollback.
    auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(_metadataObj);
    if (!metadataResult.isOK()) {
        LOGV2_ERROR(21279,
                    "invalid replication metadata from sync source {syncSource}: "
                    "{error}: {metadata}",
                    "Invalid replication metadata from sync source",
                    "syncSource"_attr = _config.source,
                    "error"_attr = metadataResult.getStatus(),
                    "metadata"_attr = _metadataObj);
        return metadataResult.getStatus();
    }
    auto replSetMetadata = metadataResult.getValue();

    // Determine if we should stop syncing from our current sync source.
    auto changeSyncSourceAction = _dataReplicatorExternalState->shouldStopFetching(
        _config.source, replSetMetadata, oqMetadata, previousOpTimeFetched, lastDocOpTime);
    str::stream errMsg;
    errMsg << "sync source " << _config.source.toString();
    errMsg << " (config version: " << replSetMetadata.getConfigVersion();
    errMsg << "; last applied optime: " << oqMetadata.getLastOpApplied().toString();
    errMsg << "; sync source index: " << oqMetadata.getSyncSourceIndex();
    errMsg << "; has primary index: " << oqMetadata.hasPrimaryIndex();
    errMsg << ") is no longer valid";
    errMsg << " previous batch last fetched optime: " << previousOpTimeFetched.toString();
    errMsg << " current batch last fetched optime: " << lastDocOpTime.toString();

    if (changeSyncSourceAction == ChangeSyncSourceAction::kStopSyncingAndDropLastBatch) {
        return Status(ErrorCodes::InvalidSyncSource, errMsg);
    }

    _dataReplicatorExternalState->processMetadata(replSetMetadata, oqMetadata);

    // Increment stats. We read all of the docs in the query.
    opsReadStats.increment(info.networkDocumentCount);
    networkByteStats.increment(info.networkDocumentBytes);

    oplogBatchStats.recordMillis(_lastBatchElapsedMS, documents.empty());

    if (_cursor->getPostBatchResumeToken()) {
        auto pbrt = ResumeTokenOplogTimestamp::parse(
            IDLParserErrorContext("OplogFetcher PostBatchResumeToken"),
            *_cursor->getPostBatchResumeToken());
        info.resumeToken = pbrt.getTs();
    }

    if (changeSyncSourceAction == ChangeSyncSourceAction::kStopSyncingAndEnqueueLastBatch) {
        return Status(ErrorCodes::InvalidSyncSource, errMsg);
    }

    if (MONGO_unlikely(hangOplogFetcherBeforeAdvancingLastFetched.shouldFail())) {
        hangOplogFetcherBeforeAdvancingLastFetched.pauseWhileSet();
    }

    // Start skipping the first doc after at least one doc has been enqueued in the lifetime
    _config.startingPoint = StartingPoint::kSkipFirstDoc;

    // We have now processed the batch. We should only move forward our view of _lastFetched if the
    // batch was not empty.
    if (lastDocOpTime != previousOpTimeFetched) {
        LOGV2_DEBUG(21273,
                    3,
                    "Oplog fetcher setting last fetched optime ahead after batch: {lastDocOpTime}",
                    "Oplog fetcher setting last fetched optime ahead after batch",
                    "lastDocOpTime"_attr = lastDocOpTime);

        stdx::lock_guard<Latch> lock(_mutex);
        _lastFetched = lastDocOpTime;
    }

    _firstBatch = false;
    return Status::OK();
}

Status OplogFetcher::_checkRemoteOplogStart(const OplogFetcher::Documents& documents,
                                            OpTime remoteLastOpApplied,
                                            int remoteRBID) {
    using namespace fmt::literals;

    // Once we establish our cursor, if we use rollback-via-refetch, we need to ensure that our
    // upstream node hasn't rolled back since that could cause it to not have our required minValid
    // point. The cursor will be killed if the upstream node rolls back so we don't need to keep
    // checking once the cursor is established. If we do not use rollback-via-refetch, this check is
    // not necessary, and _config.requiredRBID will be set to kUninitializedRollbackId in that case.
    if (_config.requiredRBID != ReplicationProcess::kUninitializedRollbackId &&
        remoteRBID != _config.requiredRBID) {
        return Status(ErrorCodes::InvalidSyncSource,
                      "Upstream node rolled back after choosing it as a sync source. Choosing "
                      "new sync source.");
    }
    // Set _receivedRBID to remoteRBID so that it can be returned when the oplog fetcher shuts down.
    _receivedRBID = remoteRBID;

    // Sometimes our remoteLastOpApplied may be stale; if we received a document with an
    // opTime later than remoteLastApplied, we can assume the remote is at least up to that
    if (!documents.empty()) {
        const auto docOpTime = OpTime::parseFromOplogEntry(documents.back());
        if (docOpTime.isOK()) {
            remoteLastOpApplied = std::max(remoteLastOpApplied, docOpTime.getValue());
        }
    }

    auto lastFetched = _getLastOpTimeFetched();

    // The sync source could be behind us if it rolled back after we selected it. We could have
    // failed to detect the rollback if it occurred between sync source selection (when we check the
    // candidate is ahead of us) and sync source resolution (when we got '_receivedRBID'). If the
    // sync source is now behind us, choose a new sync source to prevent going into rollback.
    if (remoteLastOpApplied < lastFetched) {
        return Status(ErrorCodes::InvalidSyncSource,
                      "Sync source's last applied OpTime {} is older than our last fetched OpTime "
                      "{}. Choosing new sync source."_format(remoteLastOpApplied.toString(),
                                                             lastFetched.toString()));
    }

    // If '_requireFresherSyncSource' is true, we must check that the sync source's
    // lastApplied is ahead of us to prevent forming a cycle. Although we check for
    // this condition in sync source selection, if an undetected rollback occurred between sync
    // source selection and sync source resolution, this condition may no longer hold.
    // '_requireFresherSyncSource' is false for initial sync, since no other node can sync off an
    // initial syncing node, so we do not need to check for cycles. In addition, it would be
    // problematic to check this condition for initial sync, since the 'lastFetched' OpTime will
    // almost always equal the 'remoteLastApplied', since we fetch the sync source's last applied
    // OpTime to determine where to start our OplogFetcher.
    if (_config.requireFresherSyncSource == RequireFresherSyncSource::kRequireFresherSyncSource &&
        remoteLastOpApplied <= lastFetched) {
        return Status(ErrorCodes::InvalidSyncSource,
                      "Sync source must be ahead of me. My last fetched oplog optime: {}, latest "
                      "oplog optime of sync source: {}"_format(lastFetched.toString(),
                                                               remoteLastOpApplied.toString()));
    }

    // At this point we know that our sync source has our minValid and is not behind us, so if our
    // history diverges from our sync source's we should prefer its history and roll back ours.

    // Since we checked for rollback and our sync source is ahead of us, an empty batch means that
    // we have a higher timestamp on our last fetched OpTime than our sync source's last applied
    // OpTime, but a lower term. When this occurs, we must roll back our inconsistent oplog entry.
    if (documents.empty()) {
        return Status(ErrorCodes::OplogStartMissing, "Received an empty batch from sync source.");
    }

    const auto& o = documents.front();
    auto opTimeResult = OpTime::parseFromOplogEntry(o);

    if (!opTimeResult.isOK()) {
        return Status(ErrorCodes::InvalidBSON,
                      "our last optime fetched: {}. failed to parse optime from first oplog in "
                      "batch on source: {}: {}"_format(lastFetched.toString(),
                                                       o.toString(),
                                                       opTimeResult.getStatus().toString()));
    }
    auto opTime = opTimeResult.getValue();
    if (opTime != lastFetched) {
        Status status = _checkTooStaleToSyncFromSource(lastFetched, opTime);

        // We should never return an OK status here.
        invariant(!status.isOK());
        return status;
    }
    return Status::OK();
}

Status OplogFetcher::_checkTooStaleToSyncFromSource(const OpTime lastFetched,
                                                    const OpTime firstOpTimeInDocument) {
    // Check to see if the sync source's first oplog entry is later than 'lastFetched'. If it is, we
    // are too stale to sync from this node. If it isn't, we should go into rollback instead.
    BSONObj remoteFirstOplogEntry;
    try {
        // Query for the first oplog entry in the sync source's oplog.
        FindCommandRequest findRequest{_nss};
        findRequest.setSort(BSON("$natural" << 1));
        // Since this function is called after the first batch, the exhaust stream has not been
        // started yet. As a result, using the same connection is safe.
        remoteFirstOplogEntry = _conn->findOne(std::move(findRequest));
    } catch (DBException& e) {
        // If an error occurs with the query, throw an error.
        return Status(ErrorCodes::TooStaleToSyncFromSource, e.reason());
    }

    using namespace fmt::literals;

    StatusWith<OpTime> remoteFirstOpTimeResult = OpTime::parseFromOplogEntry(remoteFirstOplogEntry);
    if (!remoteFirstOpTimeResult.isOK()) {
        return Status(
            ErrorCodes::InvalidBSON,
            "failed to parse optime from first entry in source's oplog: {}: {}"_format(
                remoteFirstOplogEntry.toString(), remoteFirstOpTimeResult.getStatus().toString()));
    }

    auto remoteFirstOpTime = remoteFirstOpTimeResult.getValue();
    if (remoteFirstOpTime.isNull()) {
        return Status(ErrorCodes::InvalidBSON,
                      "optime of first entry in source's oplog cannot be null: {}"_format(
                          remoteFirstOplogEntry.toString()));
    }

    // remoteFirstOpTime may come from a very old config, so we cannot compare their terms.
    if (lastFetched.getTimestamp() < remoteFirstOpTime.getTimestamp()) {
        // We are too stale to sync from our current sync source.
        return Status(ErrorCodes::TooStaleToSyncFromSource,
                      "we are too stale to sync from the sync source's oplog. our last fetched "
                      "timestamp is earlier than the sync source's first timestamp. our last "
                      "optime fetched: {}. sync source's first optime: {}"_format(
                          lastFetched.toString(), remoteFirstOpTime.toString()));
    }

    // If we are not too stale to sync from the source, we should go into rollback.
    std::string message =
        "the sync source's oplog and our oplog have diverged, going into rollback. our last optime "
        "fetched: {}. source's GTE: {}"_format(lastFetched.toString(),
                                               firstOpTimeInDocument.toString());
    return Status(ErrorCodes::OplogStartMissing, message);
}

bool OplogFetcher::OplogFetcherRestartDecisionDefault::shouldContinue(OplogFetcher* fetcher,
                                                                      Status status) {
    // If we try to sync from a node that is shutting down, do not attempt to reconnect.
    // We should choose a new sync source.
    if (status.code() == ErrorCodes::ShutdownInProgress) {
        LOGV2(4696202,
              "Not recreating cursor for oplog fetcher because sync source is shutting down",
              "error"_attr = redact(status));
        return false;
    }
    if (_numRestarts == _maxRestarts) {
        LOGV2(21274,
              "Error returned from oplog query (no more query restarts left): {error}",
              "Error returned from oplog query (no more query restarts left)",
              "error"_attr = redact(status));
        return false;
    }
    LOGV2(21275,
          "Recreating cursor for oplog fetcher due to error: {error}. Last fetched optime: "
          "{lastOpTimeFetched}. Attempts remaining: {attemptsRemaining}",
          "Recreating cursor for oplog fetcher due to error",
          "lastOpTimeFetched"_attr = fetcher->_getLastOpTimeFetched(),
          "attemptsRemaining"_attr = (_maxRestarts - _numRestarts),
          "error"_attr = redact(status));
    _numRestarts++;
    return true;
}

void OplogFetcher::OplogFetcherRestartDecisionDefault::fetchSuccessful(OplogFetcher* fetcher) {
    _numRestarts = 0;
}

OplogFetcher::OplogFetcherRestartDecision::~OplogFetcherRestartDecision(){};

}  // namespace repl
}  // namespace mongo


try
    {
        RUN_TEST(1, test_1_usage, num_test, num_fail);
        RUN_TEST(2, test_2_timing, num_test, num_fail);
        RUN_TEST(3, test_3_measure, num_test, num_fail);
    } catch(std::exception& e)
    {
        std::cerr << e.what() << std::endl;
    }

// required approvals from the U.S. Dept. of Energy).  All rights reserved.

#define TIMEMORY_INPUT_ARCHIVE ::tim::cereal::XMLInputArchive
#define TIMEMORY_OUTPUT_ARCHIVE ::tim::cereal::XMLOutputArchive

#include "timemory/timemory.hpp"
#include "timemory/utility/signals.hpp"
#include "timemory/utility/testing.hpp"

#include <cassert>
#include <chrono>
#include <cmath>
#include <fstream>
#include <future>
#include <iterator>
#include <random>
#include <thread>
#include <unordered_map>
#include <vector>

using namespace tim::stl;
using namespace tim::component;

using papi_tuple_t = papi_tuple<PAPI_TOT_CYC, PAPI_TOT_INS, PAPI_LST_INS>;

using auto_tuple_t =
    tim::auto_tuple_t<wall_clock, system_clock, thread_cpu_clock, thread_cpu_util,
                      process_cpu_clock, process_cpu_util, papi_tuple_t, tau_marker>;

using measurement_t =
    tim::component_tuple_t<peak_rss, page_rss, virtual_memory, num_major_page_faults,
                           num_minor_page_faults, priority_context_switch,
                           voluntary_context_switch, tau_marker>;

// fibonacci calculation
int64_t
fibonacci(int32_t n)
{
    return (n < 2) ? n : fibonacci(n - 1) + fibonacci(n - 2);
}

// time fibonacci with return type and arguments
int64_t
time_fibonacci(int32_t n)
{
    TIMEMORY_MARKER(auto_tuple_t, "");
    return fibonacci(n);
}

void
print_info(const std::string&);
void
print_string(const std::string& str);
void
test_1_usage();
void
test_2_timing();
void
test_3_measure();
void
print_mpi_storage();

int
main(int argc, char** argv)
{
    tim::settings::banner()      = true;
    tim::settings::json_output() = true;
    tim::enable_signal_detection();
    tim::dmp::initialize(argc, argv);
    tim::timemory_init(argc, argv);

    tim::component_tuple_t<papi_tuple_t> m("PAPI measurements");
    m.start();

    CONFIGURE_TEST_SELECTOR(3);

    int num_fail = 0;
    int num_test = 0;

    std::cout << "# tests: " << tests.size() << std::endl;

    m.stop();

    std::cout << "\n" << m << std::endl;

    TEST_SUMMARY(argv[0], num_test, num_fail);

    print_mpi_storage();

    tim::timemory_finalize();
    tim::dmp::finalize();

    exit(num_fail);
}

void
print_info(const std::string& func)
{
    if(tim::dmp::rank() == 0)
    {
        std::cout << "\n[" << tim::dmp::rank() << "]\e[1;33m TESTING \e[0m["
                  << "\e[1;36m" << func << "\e[0m"
                  << "]...\n"
                  << std::endl;
    }
}

void
print_string(const std::string& str)
{
    std::stringstream _ss;
    _ss << "[" << tim::dmp::rank() << "] " << str << std::endl;
    std::cout << _ss.str();
}

template <typename Tp>
size_t
random_entry(const std::vector<Tp>& v)
{
    std::mt19937 rng;
    rng.seed(std::random_device()());
    std::uniform_int_distribution<std::mt19937::result_type> dist(0, v.size() - 1);
    return v.at(dist(rng));
}

template <typename Tp>
void
serialize(const std::string& fname, const std::string& title, const Tp& obj)
{
    static constexpr auto spacing =
        tim::cereal::JSONOutputArchive::Options::IndentChar::space;
    std::stringstream ss;
    {
        // ensure json write final block during destruction before the file is closed
        //                                  args: precision, spacing, indent size
        tim::cereal::JSONOutputArchive::Options opts(12, spacing, 4);
        tim::cereal::JSONOutputArchive          oa(ss, opts);
        oa(tim::cereal::make_nvp(title, obj));
    }
    std::ofstream ofs(fname.c_str());
    ofs << ss.str() << std::endl;
}

void
test_1_usage()
{
    print_info(__FUNCTION__);
    auto test_1_marker = TIMEMORY_HANDLE(auto_tuple_t, "");

    measurement_t _use_beg("test_1_usage_begin");
    measurement_t _use_delta("test_1_usage_delta");
    measurement_t _use_end("test_1_usage_end");

    auto n = 5000000;
    _use_beg.record();
    _use_delta.start();
    std::vector<int64_t> v(n, 30);
    long                 nfib = random_entry(v);
    time_fibonacci(nfib);
    _use_delta.stop();
    _use_end.record();

    std::cout << "usage (begin): " << _use_beg << std::endl;
    std::cout << "usage (delta): " << _use_delta << std::endl;
    std::cout << "usage (end):   " << _use_end << std::endl;

    std::vector<std::pair<std::string, measurement_t>> measurements = {
        { "begin", _use_beg },
        { "delta", _use_delta },
        { "end", _use_end }
    };
    test_1_marker.stop();
}

void
test_2_timing()
{
    print_info(__FUNCTION__);

    auto_tuple_t runtime(TIMEMORY_JOIN("_", __func__, "runtime"), tim::scope::tree{});
    std::atomic<int64_t> ret;
    {
        TIMEMORY_MARKER(auto_tuple_t, "");

        auto run_fib = [&](long n) {
            TIMEMORY_BLANK_MARKER(auto_tuple_t, "run_fib/", n);
            ret += time_fibonacci(n);
        };

        std::vector<std::thread> threads;
        runtime.start();
        {
            for(int i = 0; i < 7; ++i)
                threads.push_back(std::thread(run_fib, 35));
            threads.push_back(std::thread(run_fib, 43));

            run_fib(40);

            for(auto& itr : threads)
                itr.join();
        }
        runtime.stop();
    }

    print_string(TIMEMORY_JOIN("/", __FUNCTION__, __LINE__));
    std::cout << "total runtime: " << runtime << std::endl;
    std::cout << "std::get: " << std::get<0>(runtime) << std::endl;
    std::cout << "fibonacci total: " << ret.load() << "\n" << std::endl;
    std::cout << "runtime process cpu time: " << runtime.get<process_cpu_clock>() << "\n";
    print_string(TIMEMORY_JOIN("/", __FUNCTION__, __LINE__));
}

void
test_3_measure()
{
    print_info(__FUNCTION__);

    tim::component_tuple<page_rss, peak_rss> prss(TIMEMORY_LABEL(""));
    {
        TIMEMORY_VARIADIC_BASIC_AUTO_TUPLE("[init]", page_rss, peak_rss);
        // just record the peak rss
        prss.measure();
        std::cout << "  Current rss: " << prss << std::endl;
    }

    {
        TIMEMORY_VARIADIC_AUTO_TUPLE("[delta]", page_rss, peak_rss);
        // do something, where you want delta peak rss
        auto                 n = 10000000;
        std::vector<int64_t> v(n, 10);
        long                 nfib = random_entry(v);
        fibonacci(nfib);
        prss.stop();
        std::cout << "Change in rss: " << prss << std::endl;
    }

    prss.measure();
    std::cout << "  Current rss: " << prss << std::endl;
}

void
print_mpi_storage()
{
    auto ret = tim::storage<tim::component::wall_clock>::instance()->mpi_get();
    if(tim::dmp::rank() != 0)
        return;

    uint64_t _w = 0;
    for(const auto& jitr : ret)
        for(const auto& iitr : jitr)
        {
            _w = std::max<uint64_t>(_w, iitr.prefix().length());
        }

    for(uint64_t j = 0; j < ret.size(); ++j)
    {
        std::cout << "[RANK: " << j << "]\n";
        std::stringstream ss;
        for(const auto& itr : ret[j])
        {
            ss << "\t" << std::setw(_w) << std::left << itr.prefix() << " : "
               << itr.data() << "\n";
        }
        std::cout << ss.str();
    }
}


try
		{
			throw Exception("");
		}
		catch (Exception &exception)
		{
			Application->ShowException(&exception);
		}

/**
 *
 */

#include <vcl.h>
#pragma hdrstop
USEFORM("mainForm1.cpp", Form5);
USEFORM("dsnManager.cpp", Form1);
USEFORM("addUser.cpp", Form2);
USEFORM("delimeter.cpp", Form3);
USEFORM("resource.cpp", Form6);
USEFORM("createIndex.cpp", Form7);
USEFORM("iloaderStatus.cpp", Form8);
USEFORM("addDatafile2.cpp", Form10);
USEFORM("createTbs.cpp", Form9);
USEFORM("winISQL.cpp", Form4);
USEFORM("winISQL_dsnManager.cpp", Form11);
WINAPI WinMain(HINSTANCE, HINSTANCE, LPSTR, int)
{
	try
	{
		Application->Initialize();
		Application->CreateForm(__classid(TForm5), &Form5);
		Application->CreateForm(__classid(TForm1), &Form1);
		Application->CreateForm(__classid(TForm2), &Form2);
		Application->CreateForm(__classid(TForm3), &Form3);
		Application->CreateForm(__classid(TForm6), &Form6);
		Application->CreateForm(__classid(TForm7), &Form7);
		Application->CreateForm(__classid(TForm8), &Form8);
		Application->CreateForm(__classid(TForm10), &Form10);
		Application->CreateForm(__classid(TForm9), &Form9);
		Application->CreateForm(__classid(TForm4), &Form4);
		Application->CreateForm(__classid(TForm11), &Form11);
		Application->Run();
	}
	catch (Exception &exception)
	{
		Application->ShowException(&exception);
	}
	catch (...)
	{


try
        {
          service->log(reference, level, message, exception, file, function, line);
        }
        catch (...)
        {
          // TODO: consider printing to System Error
        }

/*=============================================================================

  Library: CTK

=============================================================================*/

#include "ctkMTLogTracker_p.h"

#include <QDateTime>

ctkMTLogTracker::ctkMTLogTracker(ctkPluginContext* context, QIODevice* out)
  : ctkServiceTracker<ctkLogService*>(context), out(out), logLevel(std::numeric_limits<int>::max())
{

}

int ctkMTLogTracker::getLogLevel() const
{
  QMutexLocker lock(&mutex);
  return logLevel;
}

ctkLogService* ctkMTLogTracker::addingService(const ctkServiceReference& reference)
{
  ctkLogService* service = getService(reference);

  QMutexLocker lock(&mutex);
  if (logLevel == std::numeric_limits<int>::max() ||
      service->getLogLevel() > logLevel)
  {
    logLevel = service->getLogLevel();
  }
  return service;
}

void ctkMTLogTracker::removedService(const ctkServiceReference& reference, ctkLogService* service)
{
  int newLogLevel = std::numeric_limits<int>::max();

  QList<ctkLogService*> services = getServices();
  foreach (ctkLogService* s, services)
  {
    if (s == service) continue;
    if (newLogLevel == std::numeric_limits<int>::max() ||
        s->getLogLevel() > newLogLevel)
    {
      newLogLevel = s->getLogLevel();
    }
  }

  {
    QMutexLocker lock(&mutex);
    logLevel = newLogLevel;
  }

  ctkServiceTracker<ctkLogService*>::removedService(reference, service);
}

void ctkMTLogTracker::log(int level, const QString& message, const std::exception* exception,
                          const char* file, const char* function, int line)
{
  logToAll(ctkServiceReference(), level, message, exception, file, function, line);
}

void ctkMTLogTracker::log(const ctkServiceReference& reference, int level, const QString& message,
                          const std::exception* exception, const char* file, const char* function, int line)
{
  logToAll(reference, level, message, exception, file, function, line);
}

void ctkMTLogTracker::logToAll(const ctkServiceReference& reference, int level, const QString& message,
                               const std::exception* exception, const char* file, const char* function, int line)
{
  QList<ctkServiceReference> references = getServiceReferences();

  if (!references.isEmpty())
  {
    foreach (ctkServiceReference ref, references)
    {
      ctkLogService* service = getService(ref);
      if (service != 0)
      {

      }
    }
    return;
  }

  noLogService(reference, level, message, exception, file, function, line);
}

void ctkMTLogTracker::noLogService(const ctkServiceReference& reference, int level, const QString& message,
                                   const std::exception* exc, const char* file, const char* function, int line)
{
  QMutexLocker lock(&mutex);
  QString timestamp = QDateTime::currentDateTime().toString();

  out << timestamp;

  if (level == ctkLogService::LOG_DEBUG)
  {
    out << " Debug";
  }
  else if (level == LOG_INFO)
  {
    out << " Info";
  }
  else if (level == LOG_WARNING)
  {
    out << " Warning";
  }
  else if (level == LOG_ERROR)
  {
    out << " Error";
  }
  else
  {
    out << "[Unknown Log Level]";
  }

  out << ": " << message << '\n';

  if (reference)
  {
    out << reference << '\n';
  }

  if (exc)
  {
    out << exc->what() << '\n';
  }

  if (file || function)
  {
    out << "in " << function << ", " << file << ":" << line << '\n';
  }

  out.flush();
}


try {
            TEST(cl);
            if(towupper_l(L'i',cl) == 0x130) {
                test_one<CharType>(gen(name),"i","i","İ");
            }
            else {
                std::cout <<"  Turkish locale is not supported well" << std::endl;
            }
        }
        catch(...) {
            if(cl) freelocale(cl);
            throw;
        }

#ifdef BOOST_LOCALE_NO_POSIX_BACKEND
#include <iostream>
int main()
{
        std::cout << "POSIX Backend is not build... Skipping" << std::endl;
}
#else
#include <boost/locale/conversion.hpp>
#include <boost/locale/localization_backend.hpp>
#include <boost/locale/generator.hpp>
#include <boost/locale/info.hpp>
#include <iomanip>
#include "test_locale.hpp"
#include "test_locale_tools.hpp"
#include "test_posix_tools.hpp"
#include <iostream>

#include <wctype.h>

template<typename CharType>
void test_one(std::locale const &l,std::string src,std::string tgtl,std::string tgtu)
{
    TEST(boost::locale::to_upper(to_correct_string<CharType>(src,l),l) == to_correct_string<CharType>(tgtu,l));
    TEST(boost::locale::to_lower(to_correct_string<CharType>(src,l),l) == to_correct_string<CharType>(tgtl,l));
    TEST(boost::locale::fold_case(to_correct_string<CharType>(src,l),l) == to_correct_string<CharType>(tgtl,l));
}

template<typename CharType>
void test_char()
{
    boost::locale::generator gen;

    std::cout << "- Testing at least C" << std::endl;

    std::locale l = gen("en_US.UTF-8");

    test_one<CharType>(l,"Hello World i","hello world i","HELLO WORLD I");

    std::string name = "en_US.UTF-8";
    if(have_locale(name)) {
        std::cout << "- Testing " << name << std::endl;
        std::locale l=gen(name);
        test_one<CharType>(l,"Façade","façade","FAÇADE");
    }
    else {
        std::cout << "- en_US.UTF-8 is not supported, skipping" << std::endl;
    }

    name = "en_US.ISO8859-1";
    if(have_locale(name)) {
        std::cout << "Testing " << name << std::endl;
        std::locale l=gen(name);
        test_one<CharType>(l,"Hello World","hello world","HELLO WORLD");
        #if defined(__APPLE__) || defined(__FreeBSD__)
        if(sizeof(CharType)!=1)
        #endif
            test_one<CharType>(l,"Façade","façade","FAÇADE");
    }
    else {
        std::cout << "- en_US.ISO8859-1 is not supported, skipping" << std::endl;
    }

    name = "tr_TR.UTF-8";
    if(have_locale(name)) {
        std::cout << "Testing " << name << std::endl;
        locale_t cl = newlocale(LC_ALL_MASK,name.c_str(),0);

        if(cl) freelocale(cl);

    }
    else
    {
        std::cout << "- tr_TR.UTF-8 is not supported, skipping" << std::endl;
    }
}

int main()
{
    try {
        boost::locale::localization_backend_manager mgr = boost::locale::localization_backend_manager::global();
        mgr.select("posix");
        boost::locale::localization_backend_manager::global(mgr);

        std::cout << "Testing char" << std::endl;
        test_char<char>();
        std::cout << "Testing wchar_t" << std::endl;
        test_char<wchar_t>();
    }
    catch(std::exception const &e) {
        std::cerr << "Failed " << e.what() << std::endl;
        return EXIT_FAILURE;
    }
    FINALIZE();

}

#endif // POSIX

// boostinspect:noascii