Inside Bitcoin

501}
502
503void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
504{
505    //nBestSeenHeight is not updated yet for the new block
506    int blocksAgo = nBestSeenHeight - entryHeight;
507    if (nBestSeenHeight == 0)  // the BlockPolicyEstimator hasn't seen any blocks yet
508        blocksAgo = 0;
509    if (blocksAgo < 0) {
510        LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
511        return;  //This can't happen because we call this with our best seen height, no entries can have higher
512    }
513
514    if (blocksAgo >= (int)unconfTxs.size()) {
515        if (oldUnconfTxs[bucketindex] > 0) {
516            oldUnconfTxs[bucketindex]--;
517        } else {
518            LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
519                     bucketindex);
520        }
521    }
522    else {
523        unsigned int blockIndex = entryHeight % unconfTxs.size();
524        if (unconfTxs[blockIndex][bucketindex] > 0) {
525            unconfTxs[blockIndex][bucketindex]--;
526        } else {
527            LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
528                     blockIndex, bucketindex);
529        }
530    }
531    if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
532        assert(scale != 0);
533        unsigned int periodsAgo = blocksAgo / scale;
534        for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
535            failAvg[i][bucketindex]++;
536        }
537    }
538}
539
540bool CBlockPolicyEstimator::removeTx(Txid hash)
541{
542    LOCK(m_cs_fee_estimator);
543    return _removeTx(hash, /*inBlock=*/false);
544}
545
546bool CBlockPolicyEstimator::_removeTx(const Txid& hash, bool inBlock)
547{
548    AssertLockHeld(m_cs_fee_estimator);
549    std::map<Txid, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
550    if (pos != mapMemPoolTxs.end()) {
551        feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
552        shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
553        longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
554        mapMemPoolTxs.erase(hash);
555        return true;
556    } else {
557        return false;
558    }
559}
560
561CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates)
562    : m_estimation_filepath{estimation_filepath}
563{
564    static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
565    size_t bucketIndex = 0;
566
567    for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
568        buckets.push_back(bucketBoundary);
569        bucketMap[bucketBoundary] = bucketIndex;
570    }
571    buckets.push_back(INF_FEERATE);
572    bucketMap[INF_FEERATE] = bucketIndex;
573    assert(bucketMap.size() == buckets.size());
574
575    feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
576    shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
577    longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
578
579    AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")};
580
581    if (est_file.IsNull()) {
582        LogInfo("%s is not found. Continue anyway.", fs::PathToString(m_estimation_filepath));
583        return;
584    }
585
586    std::chrono::hours file_age = GetFeeEstimatorFileAge();
587    if (file_age > MAX_FILE_AGE && !read_stale_estimates) {
588        LogWarning("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE));
589        return;
590    }
591
592    if (!Read(est_file)) {
593        LogWarning("Failed to read fee estimates from %s. Continue anyway.", fs::PathToString(m_estimation_filepath));
594    }
595}
596
597CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
598
599void CBlockPolicyEstimator::processTransaction(const NewMempoolTransactionInfo& tx)
600{
601    LOCK(m_cs_fee_estimator);
602    const unsigned int txHeight = tx.info.txHeight;
603    const auto& hash = tx.info.m_tx->GetHash();
604    if (mapMemPoolTxs.contains(hash)) {
605        LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n",
606                 hash.ToString());
607        return;
608    }
609
610    if (txHeight != nBestSeenHeight) {
611        // Ignore side chains and re-orgs; assuming they are random they don't
612        // affect the estimate.  We'll potentially double count transactions in 1-block reorgs.
613        // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
614        // It will be synced next time a block is processed.
615        return;
616    }
617    // This transaction should only count for fee estimation if:
618    // - it's not being re-added during a reorg which bypasses typical mempool fee limits
619    // - the node is not behind
620    // - the transaction is not dependent on any other transactions in the mempool
621    // - it's not part of a package.
622    const bool validForFeeEstimation = !tx.m_mempool_limit_bypassed && !tx.m_submitted_in_package && tx.m_chainstate_is_current && tx.m_has_no_mempool_parents;
623
624    // Only want to be updating estimates when our blockchain is synced,
625    // otherwise we'll miscalculate how many blocks its taking to get included.
626    if (!validForFeeEstimation) {
627        untrackedTxs++;
628        return;
629    }
630    trackedTxs++;
631
632    // Feerates are stored and reported as BTC-per-kb:
633    const CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size);
634
635    mapMemPoolTxs[hash].blockHeight = txHeight;
636    unsigned int bucketIndex = feeStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
637    mapMemPoolTxs[hash].bucketIndex = bucketIndex;
638    unsigned int bucketIndex2 = shortStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
639    assert(bucketIndex == bucketIndex2);
640    unsigned int bucketIndex3 = longStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
641    assert(bucketIndex == bucketIndex3);
642}
643
644bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const RemovedMempoolTransactionInfo& tx)
645{
646    AssertLockHeld(m_cs_fee_estimator);
647    if (!_removeTx(tx.info.m_tx->GetHash(), true)) {
648        // This transaction wasn't being tracked for fee estimation
649        return false;
650    }
651
652    // How many blocks did it take for miners to include this transaction?
653    // blocksToConfirm is 1-based, so a transaction included in the earliest
654    // possible block has confirmation count of 1
655    int blocksToConfirm = nBlockHeight - tx.info.txHeight;
656    if (blocksToConfirm <= 0) {
657        // This can't happen because we don't process transactions from a block with a height
658        // lower than our greatest seen height
659        LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
660        return false;
661    }
662
663    // Feerates are stored and reported as BTC-per-kb:
664    CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size);
665
666    feeStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
667    shortStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
668    longStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
669    return true;
670}
671
672void CBlockPolicyEstimator::processBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block,
673                                         unsigned int nBlockHeight)
674{
675    LOCK(m_cs_fee_estimator);
676    if (nBlockHeight <= nBestSeenHeight) {
677        // Ignore side chains and re-orgs; assuming they are random
678        // they don't affect the estimate.
679        // And if an attacker can re-org the chain at will, then
680        // you've got much bigger problems than "attacker can influence
681        // transaction fees."
682        return;
683    }
684
685    // Must update nBestSeenHeight in sync with ClearCurrent so that
686    // calls to removeTx (via processBlockTx) correctly calculate age
687    // of unconfirmed txs to remove from tracking.
688    nBestSeenHeight = nBlockHeight;
689
690    // Update unconfirmed circular buffer
691    feeStats->ClearCurrent(nBlockHeight);
692    shortStats->ClearCurrent(nBlockHeight);
693    longStats->ClearCurrent(nBlockHeight);
694
695    // Decay all exponential averages
696    feeStats->UpdateMovingAverages();
697    shortStats->UpdateMovingAverages();
698    longStats->UpdateMovingAverages();
699
700    unsigned int countedTxs = 0;
701    // Update averages with data points from current block
702    for (const auto& tx : txs_removed_for_block) {
703        if (processBlockTx(nBlockHeight, tx))
704            countedTxs++;
705    }
706
707    if (firstRecordedHeight == 0 && countedTxs > 0) {
708        firstRecordedHeight = nBestSeenHeight;
709        LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
710    }
711
712
713    LogDebug(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
714             countedTxs, txs_removed_for_block.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
715             MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
716
717    trackedTxs = 0;
718    untrackedTxs = 0;
719}
720
721CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const
722{
723    // It's not possible to get reasonable estimates for confTarget of 1
724    if (confTarget <= 1)
725        return CFeeRate(0);
726
727    return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE);
728}
729
730CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
731{
732    TxConfirmStats* stats = nullptr;
733    double sufficientTxs = SUFFICIENT_FEETXS;
734    switch (horizon) {
735    case FeeEstimateHorizon::SHORT_HALFLIFE: {
736        stats = shortStats.get();
737        sufficientTxs = SUFFICIENT_TXS_SHORT;
738        break;
739    }
740    case FeeEstimateHorizon::MED_HALFLIFE: {
741        stats = feeStats.get();
742        break;
743    }
744    case FeeEstimateHorizon::LONG_HALFLIFE: {
745        stats = longStats.get();
746        break;
747    }
748    } // no default case, so the compiler can warn about missing cases
749    assert(stats);
750