此系列是为了记录自己学习VTM10.0的过程，目前正在看编码端。主要的参考文档有JVET-S2001-vH和JVET-S2002-v1。由于本人水平有限，出现的错误恳请大家指正，欢迎与大家一起交流进步。

上一篇博文(VTM10.0代码学习11)的末尾留着一个涉及CU划分的分支没讲，本篇博文就来说说。这个分支里会调用函数xCheckModeSplit()，而这个函数里面又会调用xCompressCU()。所以从CTU到任何一个CU的编码过程可以描述为如下，先从compressCtu()开始，之后就是xCompressCU()和xCheckModeSplit()的交替。那接下来就先从xCompressCU()中涉及划分的分支说起

1. 涉及划分的分支

int signalModeConsVal = tempCS->signalModeCons( getPartSplit( currTestMode ), partitioner, modeTypeParent );
int numRoundRdo = signalModeConsVal == LDT_MODE_TYPE_SIGNAL ? 2 : 1;
bool skipInterPass = false;
for( int i = 0; i < numRoundRdo; i++ )
{
}

这三个变量皆与modeType的设置有关系。当numRoundRdo为2时，modeType需要先后设置为MODE_TYPE_INTER和MODE_TYPE_INTRA进行测试。

for循环里的内容就参考1.1小节

1.1 for循环

if( signalModeConsVal == LDT_MODE_TYPE_SIGNAL )
{
    tempCS->modeType = partitioner.modeType = (i == 0) ? MODE_TYPE_INTER : MODE_TYPE_INTRA;
}
else if( signalModeConsVal == LDT_MODE_TYPE_INFER )
{
    tempCS->modeType = partitioner.modeType = MODE_TYPE_INTRA;
}
else if( signalModeConsVal == LDT_MODE_TYPE_INHERIT )
{
    tempCS->modeType = partitioner.modeType = modeTypeParent;
}

设置modeType

if( modeTypeParent == MODE_TYPE_ALL && tempCS->modeType == MODE_TYPE_INTER )
{
    m_pcIntraSearch->setSaveCuCostInSCIPU( true );
    m_pcIntraSearch->setNumCuInSCIPU( 0 );
}
else if( modeTypeParent == MODE_TYPE_ALL && tempCS->modeType != MODE_TYPE_INTER )
{
    m_pcIntraSearch->setSaveCuCostInSCIPU( false );
    if( tempCS->modeType == MODE_TYPE_ALL )
    {
        m_pcIntraSearch->setNumCuInSCIPU( 0 );
    }
}

与SCIPU有关

xCheckModeSplit( tempCS, bestCS, partitioner, currTestMode, modeTypeParent, skipInterPass );
//recover cons modes
tempCS->modeType = partitioner.modeType = modeTypeParent;
tempCS->treeType = partitioner.treeType = treeTypeParent;
partitioner.chType = chTypeParent;

xCheckModeSplit()：进行划分模式的测试，具体参考第2大节

之后恢复到未划分之前的modeType、treeType和chType

if( modeTypeParent == MODE_TYPE_ALL )
{
    m_pcIntraSearch->setSaveCuCostInSCIPU( false );
    if( numRoundRdo == 2 && tempCS->modeType == MODE_TYPE_INTRA )
    {
        m_pcIntraSearch->initCuAreaCostInSCIPU();
    }
}

与SCIPU有关

if( skipInterPass )
{
    break;
}

如果skipInterPass为True，跳过之后modeType为MODE_TYPE_INTRA的测试

2. xCheckModeSplit()

const int qp                = encTestMode.qp;
const Slice &slice          = *tempCS->slice;
const int oldPrevQp         = tempCS->prevQP[partitioner.chType];
const auto oldMotionLut     = tempCS->motionLut;
const auto oldPLT           = tempCS->prevPLT;

const PartSplit split = getPartSplit( encTestMode );
const ModeType modeTypeChild = partitioner.modeType;

tempCS->initStructData( qp );

oldPrevQp、oldMotionLut和oldPLT都是进行划分模式测试之后需要恢复的变量

split：当前测试的划分模式

modeTypeChild：划分后的modeType

initStructData()：初始化父CU的tempCS

m_CABACEstimator->getCtx() = m_CurrCtx->start;

const TempCtx ctxStartSP( m_CtxCache, SubCtx( Ctx::SplitFlag,   m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartQt( m_CtxCache, SubCtx( Ctx::SplitQtFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartHv( m_CtxCache, SubCtx( Ctx::SplitHvFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStart12( m_CtxCache, SubCtx( Ctx::Split12Flag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartMC( m_CtxCache, SubCtx( Ctx::ModeConsFlag, m_CABACEstimator->getCtx() ) );
m_CABACEstimator->resetBits();

m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitFlag,   ctxStartSP );
m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitQtFlag, ctxStartQt );
m_CABACEstimator->getCtx() = SubCtx( Ctx::SplitHvFlag, ctxStartHv );
m_CABACEstimator->getCtx() = SubCtx( Ctx::Split12Flag, ctxStart12 );
m_CABACEstimator->getCtx() = SubCtx( Ctx::ModeConsFlag, ctxStartMC );

CABAC相关的设置

m_CABACEstimator->split_cu_mode( split, *tempCS, partitioner );//编码划分信息
m_CABACEstimator->mode_constraint( split, *tempCS, partitioner, modeTypeChild );//编码modeType

const double factor = ( tempCS->currQP[partitioner.chType] > 30 ? 1.1 : 1.075 );
tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
const double cost   = m_pcRdCost->calcRdCost( uint64_t( m_CABACEstimator->getEstFracBits() + ( ( bestCS->fracBits ) / factor ) ), Distortion( bestCS->dist / factor ) ) + bestCS->costDbOffset / factor;

if (cost > bestCS->cost + bestCS->costDbOffset)
{
    xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
    return;
}

split_cu_mode()：编码划分信息

mode_constraint()：编码modeType

if分支：如果编码划分信息和modeType的比特数消耗相对过大，就结束此函数

xCheckBestMode()：判断是否使用当前测试的模式的结果，如果使用则交换tempCS和bestCS并记录最佳的Ctx

const bool chromaNotSplit = modeTypeParent == MODE_TYPE_ALL && modeTypeChild == MODE_TYPE_INTRA ? true : false;
if( partitioner.treeType != TREE_D )
{
    tempCS->treeType = TREE_L;
}
else
{
    if( chromaNotSplit )
    {
        tempCS->treeType = partitioner.treeType = TREE_L;
    }
    else
    {
        tempCS->treeType = partitioner.treeType = TREE_D;
    }
}

判断是否停止色度划分

partitioner.splitCurrArea( split, *tempCS );//进行CU划分

m_CurrCtx++;

tempCS->getRecoBuf().fill( 0 );
tempCS->getPredBuf().fill(0);

splitCurrArea()：进行CU划分

AffineMVInfo tmpMVInfo;
bool isAffMVInfoSaved;
m_pcInterSearch->savePrevAffMVInfo(0, tmpMVInfo, isAffMVInfoSaved);
BlkUniMvInfo tmpUniMvInfo;
bool         isUniMvInfoSaved = false;
if (!tempCS->slice->isIntra())
{
    m_pcInterSearch->savePrevUniMvInfo(tempCS->area.Y(), tmpUniMvInfo, isUniMvInfoSaved);
}

与MV信息存储有关

do
{
} while( partitioner.nextPart( *tempCS ) );

partitioner.exitCurrSplit();

m_CurrCtx--;

do-while：对划分后的每个子CU调用xCompressCU()，具体参考2.1小节

exitCurrSplit()：退出当前划分

if( chromaNotSplit )
{
    uint32_t numCuPuTu[6];
    tempCS->picture->cs->getNumCuPuTuOffset( numCuPuTu );
    tempCS->picture->cs->useSubStructure( *tempCS, partitioner.chType, CS::getArea( *tempCS, partitioner.currArea(), partitioner.chType ), false, true, false, false, false );

    if (isChromaEnabled(tempCS->pcv->chrFormat))
    {
        partitioner.chType = CHANNEL_TYPE_CHROMA;
        tempCS->treeType = partitioner.treeType = TREE_C;

        m_CurrCtx++;

        const unsigned wIdx = gp_sizeIdxInfo->idxFrom( partitioner.currArea().lwidth() );
        const unsigned hIdx = gp_sizeIdxInfo->idxFrom( partitioner.currArea().lheight() );
        CodingStructure *tempCSChroma = m_pTempCS2[wIdx][hIdx];
        CodingStructure *bestCSChroma = m_pBestCS2[wIdx][hIdx];
        tempCS->initSubStructure( *tempCSChroma, partitioner.chType, partitioner.currArea(), false );
        tempCS->initSubStructure( *bestCSChroma, partitioner.chType, partitioner.currArea(), false );
        tempCS->treeType = TREE_D;
        xCompressCU( tempCSChroma, bestCSChroma, partitioner );

        //attach chromaCS to luma CS and update cost
        bool keepResi = KEEP_PRED_AND_RESI_SIGNALS;
        tempCS->useSubStructure( *bestCSChroma, partitioner.chType, CS::getArea( *bestCSChroma, partitioner.currArea(), partitioner.chType ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, true, true );

        //release tmp resource
        tempCSChroma->releaseIntermediateData();
        bestCSChroma->releaseIntermediateData();
        //tempCS->picture->cs->releaseIntermediateData();
        m_CurrCtx--;
    }
    tempCS->picture->cs->clearCuPuTuIdxMap( partitioner.currArea(), numCuPuTu[0], numCuPuTu[1], numCuPuTu[2], numCuPuTu + 3 );

    //recover luma tree status
    partitioner.chType = CHANNEL_TYPE_LUMA;
    partitioner.treeType = TREE_D;
    partitioner.modeType = MODE_TYPE_ALL;
}

如果色度停止划分，需要对色度CU单独测试

const PartSplit implicitSplit = partitioner.getImplicitSplit( *tempCS );

{
    bool enforceQT = implicitSplit == CU_QUAD_SPLIT;

    // LARGE CTU bug
    if( m_pcEncCfg->getUseFastLCTU() )
    {
        unsigned minDepth = 0;
        unsigned maxDepth = floorLog2(tempCS->sps->getCTUSize()) - floorLog2(tempCS->sps->getMinQTSize(slice.getSliceType(), partitioner.chType));

        if( auto ad = dynamic_cast<AdaptiveDepthPartitioner*>( &partitioner ) )
        {
            ad->setMaxMinDepth( minDepth, maxDepth, *tempCS );
        }

        if( minDepth > partitioner.currQtDepth )
        {
            // enforce QT
            enforceQT = true;
        }
    }

    //计算编码划分信息和modeType花费的比特数
    if( !enforceQT )
    {
        m_CABACEstimator->resetBits();

        m_CABACEstimator->split_cu_mode( split, *tempCS, partitioner );
        partitioner.modeType = modeTypeParent;
        m_CABACEstimator->mode_constraint( split, *tempCS, partitioner, modeTypeChild );
        tempCS->fracBits += m_CABACEstimator->getEstFracBits(); // split bits
    }
}

如果不是强制四叉树划分，则耗费的比特数需要加上编码划分信息和modeType所花费

//计算当前测试的划分方式的cost
tempCS->cost = m_pcRdCost->calcRdCost( tempCS->fracBits, tempCS->dist );

if( tempCS->cus.size() > 0 && modeTypeParent == MODE_TYPE_ALL && modeTypeChild == MODE_TYPE_INTER )
{
    int areaSizeNoResiCu = 0;
    for( int k = 0; k < tempCS->cus.size(); k++ )
    {
        areaSizeNoResiCu += (tempCS->cus[k]->rootCbf == false) ? tempCS->cus[k]->lumaSize().area() : 0;
    }
    if( areaSizeNoResiCu >= (tempCS->area.lumaSize().area() >> 1) )
    {
        skipInterPass = true;
    }
}

xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );

if分支：判断是否跳过之后的modeType为MODE_TYPE_INTRA的测试

if (isAffMVInfoSaved)
    m_pcInterSearch->addAffMVInfo(tmpMVInfo);
if (!tempCS->slice->isIntra() && isUniMvInfoSaved)
{
    m_pcInterSearch->addUniMvInfo(tmpUniMvInfo);
}

与MV信息存储有关

tempCS->motionLut = oldMotionLut;
tempCS->prevPLT   = oldPLT;
tempCS->releaseIntermediateData();
tempCS->prevQP[partitioner.chType] = oldPrevQp;

恢复父CU的tempCS的三个变量，释放中间数据

2.1 do-while

const auto &subCUArea  = partitioner.currArea();

const unsigned wIdx    = gp_sizeIdxInfo->idxFrom( subCUArea.lwidth () );
const unsigned hIdx    = gp_sizeIdxInfo->idxFrom( subCUArea.lheight() );

//取出对应块大小的tempSubCS和bestSubCS
CodingStructure *tempSubCS = m_pTempCS[wIdx][hIdx];
CodingStructure *bestSubCS = m_pBestCS[wIdx][hIdx];

//将父CU的CS的信息传入子CU的CS中
tempCS->initSubStructure( *tempSubCS, partitioner.chType, subCUArea, false );
tempCS->initSubStructure( *bestSubCS, partitioner.chType, subCUArea, false );
tempSubCS->bestParent = bestSubCS->bestParent = bestCS;
double newMaxCostAllowed = isLuma(partitioner.chType) ? std::min(encTestMode.maxCostAllowed, bestCS->cost - m_pcRdCost->calcRdCost(tempCS->fracBits, tempCS->dist)) : MAX_DOUBLE;
newMaxCostAllowed = std::max(0.0, newMaxCostAllowed);
xCompressCU(tempSubCS, bestSubCS, partitioner, newMaxCostAllowed);//对子CU进行编码
tempSubCS->bestParent = bestSubCS->bestParent = nullptr;

subCUArea：子CU的位置和大小信息

initSubStructure()：将父CU的CS的信息传入子CU的CS中

newMaxCostAllowed：最大允许的RDcost

if( bestSubCS->cost == MAX_DOUBLE )
{
    tempCS->cost = MAX_DOUBLE;
    tempCS->costDbOffset = 0;
    tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
    m_CurrCtx--;
    partitioner.exitCurrSplit();
    xCheckBestMode( tempCS, bestCS, partitioner, encTestMode );
    if( partitioner.chType == CHANNEL_TYPE_LUMA )
    {
        tempCS->motionLut = oldMotionLut;
    }
    return;
}

如果子CU未得到合适的测试结果，则结束此函数

bool keepResi = KEEP_PRED_AND_RESI_SIGNALS;
//将子CU的CS的信息传入父CU的CS中
tempCS->useSubStructure( *bestSubCS, partitioner.chType, CS::getArea( *tempCS, subCUArea, partitioner.chType ), KEEP_PRED_AND_RESI_SIGNALS, true, keepResi, keepResi, true );

if( partitioner.currQgEnable() )
{
    tempCS->prevQP[partitioner.chType] = bestSubCS->prevQP[partitioner.chType];
}

tempSubCS->releaseIntermediateData();
bestSubCS->releaseIntermediateData();

useSubStructure()：将子CU的CS的信息传入父CU的CS中

releaseIntermediateData()：释放中间数据

if( !tempCS->slice->isIntra() && partitioner.isConsIntra() )
{
    tempCS->cost = m_pcRdCost->calcRdCost( tempCS->fracBits, tempCS->dist );
    if( tempCS->cost > bestCS->cost )
    {
        tempCS->cost = MAX_DOUBLE;
        tempCS->costDbOffset = 0;
        tempCS->useDbCost = m_pcEncCfg->getUseEncDbOpt();
        m_CurrCtx--;
        partitioner.exitCurrSplit();
        if( partitioner.chType == CHANNEL_TYPE_LUMA )
        {
            tempCS->motionLut = oldMotionLut;
        }
        return;
    }
}

当子CU只可进行帧内预测且所在slice不是I slice时，如果已经测试的子CU的RDcost累加超过父CU的最佳RDcost，则结束此函数

标签：12,modeType,xCheckModeSplit,chType,partitioner,VTM10.0,MODE,tempCS,TYPE
来源： https://blog.csdn.net/hjhyxq2014/article/details/113817482