H.266/VVC变换代码学习：xT函数
作者：互联网
H.266/VVC的变换代码中，xT函数是进行主变换的函数，主要是先进行水平和垂直变换核选择然后分别进行水平和垂直变换。
基本流程如下：
- 通过调用getTrTypes()函数获取垂直和水平变换核类型（默认为DCT-2）；
- 根据垂直和水平变换核类型以及宽度和高度确定SkipWidth和SkipHeight（主要是用来进行高频调零操作：对于使用DCT2的大尺寸(64)变换块进行高频调零，只保留低频系数(32)；对于使用MTS的大尺寸(32)变换块进行高频调零，即只保留低频系数(16)）
- 通过是否使用LFNST二次变换调整SkipWidth和SkipHeight（对于形状为4xN、Nx4(N > 8)的块， LFNST只应用于左上角的4x4区域；对于形状为NxN(N>8)的块，LFNST只应用于左上角的8x8块）？？？
- 根据垂直或者水平变换核类型以及尺寸分别选择对应的变换函数(DCT-2/DST7/DCT8)并分别进行变换（垂直和水平变换是分开的，即对于二维变换调用两次fastFwdTrans）
代码如下：
void TrQuant::xT( const TransformUnit &tu, const ComponentID &compID, const CPelBuf &resi, CoeffBuf &dstCoeff, const int width, const int height )
{
const unsigned maxLog2TrDynamicRange = tu.cs->sps->getMaxLog2TrDynamicRange( toChannelType( compID ) );//15
const unsigned bitDepth = tu.cs->sps->getBitDepth( toChannelType( compID ) );
const int TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift[TRANSFORM_FORWARD];
const uint32_t transformWidthIndex = g_aucLog2[width ] - 1; // nLog2WidthMinus1, since transform start from 2-point
const uint32_t transformHeightIndex = g_aucLog2[height] - 1; // nLog2HeightMinus1, since transform start from 2-point
int trTypeHor = DCT2;
int trTypeVer = DCT2;
//获取水平和垂直的变换类型（根据是否是隐性MTS和ISP模式或者SBT或者显性MTS来获取）
getTrTypes ( tu, compID, trTypeHor, trTypeVer );
#if !JVET_O0094_LFNST_ZERO_PRIM_COEFFS
const int skipWidth = ( trTypeHor != DCT2 && width == 32 ) ? 16 : width > JVET_C0024_ZERO_OUT_TH ? width - JVET_C0024_ZERO_OUT_TH : 0;
const int skipHeight = ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0;
#else
//对于使用DCT2的大尺寸(64x64)变换块进行高频调零
//对于使用MTS的大尺寸(32x32)变换块进行高频调零，即只保留16x16低频系数
int skipWidth = ( trTypeHor != DCT2 && width == 32 ) ? 16 : width > JVET_C0024_ZERO_OUT_TH ? width - JVET_C0024_ZERO_OUT_TH : 0;
int skipHeight = ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0;
if( tu.cs->sps->getUseLFNST() && tu.cu->lfnstIdx )
{
//如果使用LFNST则只对部分块进行变换
//对于形状为4xN、Nx4和N > 8的块，建议的限制意味着LFNST现在只应用一次，而只应用于左上角的4x4区域，即将第一4x4子组外的所有主系数都归零
if( (width == 4 && height > 4) || (width > 4 && height == 4) )
{
skipWidth = width - 4;
skipHeight = height - 4;
}
else if( (width >= 8 && height >= 8) )
{
skipWidth = width - 8;
skipHeight = height - 8;
}
}
#endif
#if RExt__DECODER_DEBUG_TOOL_STATISTICS
if ( trTypeHor != DCT2 )
{
CodingStatistics::IncrementStatisticTool( CodingStatisticsClassType{ STATS__TOOL_EMT, uint32_t( width ), uint32_t( height ), compID } );
}
#endif
//__declspec(align(nBytes))
ALIGN_DATA( MEMORY_ALIGN_DEF_SIZE, TCoeff block[MAX_TB_SIZEY * MAX_TB_SIZEY] );
const Pel *resiBuf = resi.buf; //像素矩阵
const int resiStride = resi.stride; //每行的使用的像素数目
for( int y = 0; y < height; y++ )
{
for( int x = 0; x < width; x++ )
{
block[( y * width ) + x] = resiBuf[( y * resiStride ) + x];//将像素矩阵赋值给block矩阵
}
}
if( width > 1 && height > 1 ) // 2-D transform
{
//TRANSFORM_MATRIX_SHIFT=6, COM16_C806_TRANS_PREC=0
const int shift_1st = ((g_aucLog2[width ]) + bitDepth + TRANSFORM_MATRIX_SHIFT) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
const int shift_2nd = (g_aucLog2[height]) + TRANSFORM_MATRIX_SHIFT + COM16_C806_TRANS_PREC;
CHECK( shift_1st < 0, "Negative shift" );
CHECK( shift_2nd < 0, "Negative shift" );
TCoeff *tmp = ( TCoeff * ) alloca( width * height * sizeof( TCoeff ) );
//fastFwdTrans函数是一个DCT2、DST7、DCT8的3x6的函数数组，第一行代表DCT2(2/4/8/16/32/64)第二行代表DCT8(null/4/8/16/32/null)第三行代表DST7(null/4/8/16/32/null)
fastFwdTrans[trTypeHor][transformWidthIndex ](block, tmp, shift_1st, height, 0, skipWidth);//horizontal transform
fastFwdTrans[trTypeVer][transformHeightIndex](tmp, dstCoeff.buf, shift_2nd, width, skipWidth, skipHeight);//vertical transform
}
else if( height == 1 ) //1-D horizontal transform
{
const int shift = ((g_aucLog2[width ]) + bitDepth + TRANSFORM_MATRIX_SHIFT) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECKD( ( transformWidthIndex < 0 ), "There is a problem with the width." );
fastFwdTrans[trTypeHor][transformWidthIndex]( block, dstCoeff.buf, shift, 1, 0, skipWidth );
}
else //if (iWidth == 1) //1-D vertical transform
{
int shift = ( ( g_aucLog2[height] ) + bitDepth + TRANSFORM_MATRIX_SHIFT ) - maxLog2TrDynamicRange + COM16_C806_TRANS_PREC;
CHECK( shift < 0, "Negative shift" );
CHECKD( ( transformHeightIndex < 0 ), "There is a problem with the height." );
fastFwdTrans[trTypeVer][transformHeightIndex]( block, dstCoeff.buf, shift, 1, 0, skipHeight );
}
}
其中，fastFwdTrans是一种二维函数数组，它的定义如下：
FwdTrans *fastFwdTrans[NUM_TRANS_TYPE][g_numTransformMatrixSizes] =
{
{ fastForwardDCT2_B2, fastForwardDCT2_B4, fastForwardDCT2_B8, fastForwardDCT2_B16, fastForwardDCT2_B32, fastForwardDCT2_B64 },
{ nullptr, fastForwardDCT8_B4, fastForwardDCT8_B8, fastForwardDCT8_B16, fastForwardDCT8_B32, nullptr },
{ nullptr, fastForwardDST7_B4, fastForwardDST7_B8, fastForwardDST7_B16, fastForwardDST7_B32, nullptr },
};
第一个索引是变换类型（DCT-2、DCT-8、DST-7），第二个索引是相应的变换块尺寸。
函数getTrTypes()是用于获取水平和垂直的变换核类型，主要是根据是否是显性MTS、隐性MTS、ISP模式、MIP模式、SBT来选择变换核，其中MTS、SBT变换核选择原理参考：https://blog.csdn.net/BigDream123/article/details/102599000
代码如下：
void TrQuant::getTrTypes(const TransformUnit tu, const ComponentID compID, int &trTypeHor, int &trTypeVer)
{
const bool isExplicitMTS = (CU::isIntra(*tu.cu) ? tu.cs->sps->getUseIntraMTS() : tu.cs->sps->getUseInterMTS() && CU::isInter(*tu.cu)) && isLuma(compID);
#if JVET_O0529_IMPLICIT_MTS_HARMONIZE
const bool isImplicitMTS = CU::isIntra(*tu.cu) && tu.cs->sps->getUseImplicitMTS() && isLuma(compID) && tu.cu->lfnstIdx == 0 && tu.cu->mipFlag == 0;
#else
const bool isImplicitMTS = CU::isIntra(*tu.cu) && tu.cs->sps->getUseImplicitMTS() && isLuma(compID);
#endif
const bool isISP = CU::isIntra(*tu.cu) && tu.cu->ispMode && isLuma(compID);
const bool isSBT = CU::isInter(*tu.cu) && tu.cu->sbtInfo && isLuma(compID);//对于SBT仅亮度块的转换核与位置有关，色度块都用DCT-2
trTypeHor = DCT2;
trTypeVer = DCT2;
#if JVET_O0538_SPS_CONTROL_ISP_SBT
if (!tu.cs->sps->getUseMTS())
return;
#endif
if (isImplicitMTS || isISP)
{
int width = tu.blocks[compID].width;
int height = tu.blocks[compID].height;
bool widthDstOk = width >= 4 && width <= 16;
bool heightDstOk = height >= 4 && height <= 16;
if (widthDstOk)
trTypeHor = DST7;
if (heightDstOk)
trTypeVer = DST7;
return;
}
if (isSBT)
{
uint8_t sbtIdx = tu.cu->getSbtIdx();
uint8_t sbtPos = tu.cu->getSbtPos();
if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD )
{
assert( tu.lwidth() <= MTS_INTER_MAX_CU_SIZE ); //垂直分割时如果宽度大于32则出错
if( tu.lheight() > MTS_INTER_MAX_CU_SIZE )
{
trTypeHor = trTypeVer = DCT2;//若残差的一侧大于32则两个维度变换均设为DCT-2
}
else
{
if( sbtPos == SBT_POS0 ) { trTypeHor = DCT8; trTypeVer = DST7; }
else { trTypeHor = DST7; trTypeVer = DST7; }
}
}
else
{
assert( tu.lheight() <= MTS_INTER_MAX_CU_SIZE ); //水平分割时如果高度大于32则出错
if( tu.lwidth() > MTS_INTER_MAX_CU_SIZE )
{
trTypeHor = trTypeVer = DCT2;////若残差的一侧大于32则两个维度变换均设为DCT-2
}
else
{
if( sbtPos == SBT_POS0 ) { trTypeHor = DST7; trTypeVer = DCT8; }
else { trTypeHor = DST7; trTypeVer = DST7; }
}
}
return;
}
if (isExplicitMTS)
{
if (tu.mtsIdx > MTS_SKIP)
{
int indHor = (tu.mtsIdx - MTS_DST7_DST7) & 1;
int indVer = (tu.mtsIdx - MTS_DST7_DST7) >> 1;
trTypeHor = indHor ? DCT8 : DST7;
trTypeVer = indVer ? DCT8 : DST7;
}
}
}
标签：H.266,const,int,tu,height,width,&&,VVC,xT 来源： https://blog.csdn.net/BigDream123/article/details/102748739