openh264 帧间预测编码过程源码分析
openh264
OpenH264 是一个开源的 H.264 编码和解码器,由思科系统开发并维护。它专为实时应用程序如 WebRTC 设计,提供了从基础到高级特性的广泛支持。OpenH264 的编码器支持从 Constrained Baseline Profile 到 5.2 级别,允许任意分辨率的编码,不限于 16x16 的倍数,并且具有自适应量化或恒定量化的速率控制、多切片选项、多线程自动用于多切片等特性。此外,它还支持高达 4 层的时序可伸缩性、单一输入的 4 种空间分辨率的 Spatial Simulcast、长期参考帧(LTR)、内存管理控制操作(MMCO)等功能。
帧间预测编码
视频帧间预测编码(Inter-frame prediction coding)是视频压缩技术中的一种关键方法,主要用于减少视频序列中时间维度上的冗余。这种编码方式依赖于视频帧之间的空间相关性,通过预测和补偿来减少数据量,从而实现高效的视频压缩。
帧间预测编码是视频编码中非常有效的技术,广泛应用于各种视频编码标准,如H.264/AVC、H.265/HEVC、VP9和AV1等。通过减少时间冗余,帧间预测编码显著提高了视频数据的压缩率,同时保持了视频质量。
帧间预测的核心技术主要包括以下几个方面:
运动估计(Motion Estimation, ME):
- 运动估计是指在参考帧中搜索与当前块最匹配的区域,以确定最佳匹配块的位置。
运动补偿(Motion Compensation, MC):
- 运动补偿使用运动估计得到的运动向量来预测当前块,通过补偿先前图像的抽样点来生成当前图像块的预测值。
宏块(Macroblock, MB)和子宏块(Sub-Macroblock)的树状结构分块:
- H.264 支持多种宏块分割方式,如16x16、16x8、8x16和8x8,以及更细致的子宏块分割,如8x8、8x4、4x8和4x4。
多参考帧预测:
- 特别是在B帧中,可以使用两个方向的参考帧(List0和List1)进行双向预测。
亚像素精度的运动估计:
- 除了整像素精度外,H.264还支持1/2像素和1/4像素的亚像素插值,以提高预测精度。
快速搜索算法:
- 为了减少计算复杂度,使用快速搜索算法来确定最佳匹配块。
运动向量的编码:
- 运动向量的编码通常涉及编码运动向量与预测值的差值(MVD),而不是直接编码运动向量本身。
预测模式的选择:
- H.264定义了多种预测模式,包括直接模式、双向模式、List0和List1模式等。
参考帧管理:
- 管理参考帧以确保解码器能够正确地重建预测图像。
变换/量化编码和熵编码:
- 这些步骤与帧内编码相同或相似,用于进一步压缩预测残差。
帧间预测通过这些技术有效地减少了视频序列中的时间冗余,提高了压缩效率。
openh264 帧间预测编码过程
- 帧间预测过程函数关系图:
- 关键模块说明:
- 通过API 函数
EncodeFrame
完成具体的编码过程,可执行程序封装该函数进行编码;- 通过二维数组g_pWelsSliceCoding[2][2]来完成具体的I 帧、P 帧,以及变化片数和非变化片数编码;
- 在非变化片数帧间编码过程中,在
WelsMdInterMbLoop
函数中循环处理每个宏块;- 在变化片数帧间编码过程中,在
WelsMdInterMbLoopOverDynamicSlice
函数中循环处理每个宏块;pfInterMd
函数指针指向具体的增强层WelsMdInterMbEnhancelayer
或基本层WelsMdInterMb
帧间预测过程;- 在基本层预测过程中,
WelsMdInterSecondaryModesEnc
函数完成二级帧间预测过程编码;- 在增强层预测过程中,
WelsMdSpatialelInterMbIlfmdNoilp
函数完成具体的预测过程编码;
- 帧间预测过程核心函数介绍:
WelsMdInterMbLoopOverDynamicSlice
函数
- 用于动态切片的宏块编码过程,循环处理每个 MB;
// Only for inter dynamic slicing
int32_t WelsMdInterMbLoopOverDynamicSlice (sWelsEncCtx* pEncCtx, SSlice* pSlice, void* pWelsMd,const int32_t kiSliceFirstMbXY) {SWelsMD* pMd = (SWelsMD*)pWelsMd;SBitStringAux* pBs = pSlice->pSliceBsa;SDqLayer* pCurLayer = pEncCtx->pCurDqLayer;SSliceCtx* pSliceCtx = &pCurLayer->sSliceEncCtx;SMbCache* pMbCache = &pSlice->sMbCacheInfo;SMB* pMbList = pCurLayer->sMbDataP;SMB* pCurMb = NULL;int32_t iNumMbCoded = 0;const int32_t kiTotalNumMb = pCurLayer->iMbWidth * pCurLayer->iMbHeight;int32_t iNextMbIdx = kiSliceFirstMbXY;int32_t iCurMbIdx = -1;const int32_t kiMvdInterTableStride = pEncCtx->iMvdCostTableStride;uint16_t* pMvdCostTable = &pEncCtx->pMvdCostTable[pEncCtx->iMvdCostTableSize];const int32_t kiSliceIdx = pSlice->iSliceIdx;const int32_t kiPartitionId = (kiSliceIdx % pEncCtx->iActiveThreadsNum);const uint8_t kuiChromaQpIndexOffset = pCurLayer->sLayerInfo.pPpsP->uiChromaQpIndexOffset;int32_t iEncReturn = ENC_RETURN_SUCCESS;SDynamicSlicingStack sDss;if (pEncCtx->pSvcParam->iEntropyCodingModeFlag) {WelsInitSliceCabac (pEncCtx, pSlice);sDss.iStartPos = sDss.iCurrentPos = 0;sDss.pRestoreBuffer = pEncCtx->pDynamicBsBuffer[kiPartitionId];} else {sDss.iStartPos = BsGetBitsPos (pBs);}pSlice->iMbSkipRun = 0;for (;;) {//DYNAMIC_SLICING_ONE_THREAD - MultiD//stack pBs pointerpEncCtx->pFuncList->pfStashMBStatus (&sDss, pSlice, pSlice->iMbSkipRun);//point to current pMbiCurMbIdx = iNextMbIdx;pCurMb = &pMbList[ iCurMbIdx ];//step(1): set QP for the current MBpEncCtx->pFuncList->pfRc.pfWelsRcMbInit (pEncCtx, pCurMb, pSlice);// if already reaches the largest number of slices, set QPs to the upper boundif (pSlice->bDynamicSlicingSliceSizeCtrlFlag) {//a clearer logic may be://if there is no need from size control from the pSlice size, the QP will be decided by RC; else it will be set to the max QP// however, there are some parameter updating in the rc_mb_init() function, so it cannot be skipped?pCurMb->uiLumaQp = pEncCtx->pWelsSvcRc[pEncCtx->uiDependencyId].iMaxQp;pCurMb->uiChromaQp = g_kuiChromaQpTable[CLIP3_QP_0_51 (pCurMb->uiLumaQp + kuiChromaQpIndexOffset)];}//step (2). save some vale for future use, initial pWelsMdWelsMdIntraInit (pEncCtx, pCurMb, pMbCache, kiSliceFirstMbXY);WelsMdInterInit (pEncCtx, pSlice, pCurMb, kiSliceFirstMbXY);TRY_REENCODING:WelsInitInterMDStruc (pCurMb, pMvdCostTable, kiMvdInterTableStride, pMd);pEncCtx->pFuncList->pfInterMd (pEncCtx, pMd, pSlice, pCurMb, pMbCache);//mb_qp//step (4): save from the MD process from future useWelsMdInterSaveSadAndRefMbType ((pCurLayer->pDecPic->uiRefMbType), pMbCache, pCurMb, pMd);pEncCtx->pFuncList->pfMdBackgroundInfoUpdate (pCurLayer, pCurMb, pMbCache->bCollocatedPredFlag,pEncCtx->pRefPic->iPictureType);//step (5): update cacheUpdateNonZeroCountCache (pCurMb, pMbCache);//step (6): begin to write bit stream; if the pSlice size is controlled, the writing may be skippediEncReturn = pEncCtx->pFuncList->pfWelsSpatialWriteMbSyn (pEncCtx, pSlice, pCurMb);if (iEncReturn == ENC_RETURN_VLCOVERFLOWFOUND && (pCurMb->uiLumaQp < 50)) {pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);UpdateQpForOverflow (pCurMb, kuiChromaQpIndexOffset);goto TRY_REENCODING;}if (ENC_RETURN_SUCCESS != iEncReturn)return iEncReturn;//DYNAMIC_SLICING_ONE_THREAD - MultiDsDss.iCurrentPos = pEncCtx->pFuncList->pfGetBsPosition (pSlice);if (DynSlcJudgeSliceBoundaryStepBack (pEncCtx, pSlice, pSliceCtx, pCurMb, &sDss)) {pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);pCurLayer->LastCodedMbIdxOfPartition[kiPartitionId] = iCurMbIdx -1; // update LastCodedMbIdxOfPartition, need to -1 due to stepping back++ pCurLayer->NumSliceCodedOfPartition[kiPartitionId];break;}//step (7): reconstruct current MBpCurMb->uiSliceIdc = kiSliceIdx;OutputPMbWithoutConstructCsRsNoCopy (pEncCtx, pCurLayer, pSlice, pCurMb);#if defined(MB_TYPES_CHECK)WelsCountMbType (pEncCtx->sPerInfo.iMbCount, P_SLICE, pCurMb);
#endif//MB_TYPES_CHECK//step (8): update status and other parameterspEncCtx->pFuncList->pfRc.pfWelsRcMbInfoUpdate (pEncCtx, pCurMb, pMd->iCostLuma, pSlice);/*judge if all pMb in cur pSlice has been encoded*/++ iNumMbCoded;iNextMbIdx = WelsGetNextMbOfSlice (pCurLayer, iCurMbIdx);//whether all of MB in current pSlice encoded or notif (iNextMbIdx == -1 || iNextMbIdx >= kiTotalNumMb || iNumMbCoded >= kiTotalNumMb) {pCurLayer->LastCodedMbIdxOfPartition[kiPartitionId] = iCurMbIdx;++ pCurLayer->NumSliceCodedOfPartition[kiPartitionId];break;}}if (pSlice->iMbSkipRun) {BsWriteUE (pBs, pSlice->iMbSkipRun);}return iEncReturn;
}}//namespace WelsEnc
WelsMdInterMbLoop
函数
- 用于固定切片的宏块编码过程,循环处理每个 MB;
// for inter non-dynamic pSlice
int32_t WelsMdInterMbLoop (sWelsEncCtx* pEncCtx, SSlice* pSlice, void* pWelsMd, const int32_t kiSliceFirstMbXY) {SWelsMD* pMd = (SWelsMD*)pWelsMd;SBitStringAux* pBs = pSlice->pSliceBsa;SDqLayer* pCurLayer = pEncCtx->pCurDqLayer;SMbCache* pMbCache = &pSlice->sMbCacheInfo;SMB* pMbList = pCurLayer->sMbDataP;SMB* pCurMb = NULL;int32_t iNumMbCoded = 0;int32_t iNextMbIdx = kiSliceFirstMbXY;int32_t iCurMbIdx = -1;const int32_t kiTotalNumMb = pCurLayer->iMbWidth * pCurLayer->iMbHeight;const int32_t kiMvdInterTableStride = pEncCtx->iMvdCostTableStride;uint16_t* pMvdCostTable = &pEncCtx->pMvdCostTable[pEncCtx->iMvdCostTableSize];const int32_t kiSliceIdx = pSlice->iSliceIdx;const uint8_t kuiChromaQpIndexOffset = pCurLayer->sLayerInfo.pPpsP->uiChromaQpIndexOffset;int32_t iEncReturn = ENC_RETURN_SUCCESS;SDynamicSlicingStack sDss;if (pEncCtx->pSvcParam->iEntropyCodingModeFlag) {WelsInitSliceCabac (pEncCtx, pSlice);sDss.pRestoreBuffer = NULL;sDss.iStartPos = sDss.iCurrentPos = 0;}pSlice->iMbSkipRun = 0;for (;;) {if (!pEncCtx->pSvcParam->iEntropyCodingModeFlag)pEncCtx->pFuncList->pfStashMBStatus (&sDss, pSlice, pSlice->iMbSkipRun);//point to current pMbiCurMbIdx = iNextMbIdx;pCurMb = &pMbList[ iCurMbIdx ];//step(1): set QP for the current MBpEncCtx->pFuncList->pfRc.pfWelsRcMbInit (pEncCtx, pCurMb, pSlice);//step (2). save some vale for future use, initial pWelsMdWelsMdIntraInit (pEncCtx, pCurMb, pMbCache, kiSliceFirstMbXY);WelsMdInterInit (pEncCtx, pSlice, pCurMb, kiSliceFirstMbXY);TRY_REENCODING:WelsInitInterMDStruc (pCurMb, pMvdCostTable, kiMvdInterTableStride, pMd);pEncCtx->pFuncList->pfInterMd (pEncCtx, pMd, pSlice, pCurMb, pMbCache);//mb_qp//step (4): save from the MD process from future useWelsMdInterSaveSadAndRefMbType ((pCurLayer->pDecPic->uiRefMbType), pMbCache, pCurMb, pMd);pEncCtx->pFuncList->pfMdBackgroundInfoUpdate (pCurLayer, pCurMb, pMbCache->bCollocatedPredFlag,pEncCtx->pRefPic->iPictureType);//step (5): update cacheUpdateNonZeroCountCache (pCurMb, pMbCache);//step (6): begin to write bit stream; if the pSlice size is controlled, the writing may be skippediEncReturn = pEncCtx->pFuncList->pfWelsSpatialWriteMbSyn (pEncCtx, pSlice, pCurMb);if (!pEncCtx->pSvcParam->iEntropyCodingModeFlag) {if (iEncReturn == ENC_RETURN_VLCOVERFLOWFOUND && (pCurMb->uiLumaQp < 50)) {pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);UpdateQpForOverflow (pCurMb, kuiChromaQpIndexOffset);goto TRY_REENCODING;}}if (ENC_RETURN_SUCCESS != iEncReturn)return iEncReturn;//step (7): reconstruct current MBpCurMb->uiSliceIdc = kiSliceIdx;OutputPMbWithoutConstructCsRsNoCopy (pEncCtx, pCurLayer, pSlice, pCurMb);#if defined(MB_TYPES_CHECK)WelsCountMbType (pEncCtx->sPerInfo.iMbCount, P_SLICE, pCurMb);
#endif//MB_TYPES_CHECK//step (8): update status and other parameterspEncCtx->pFuncList->pfRc.pfWelsRcMbInfoUpdate (pEncCtx, pCurMb, pMd->iCostLuma, pSlice);/*judge if all pMb in cur pSlice has been encoded*/++ iNumMbCoded;iNextMbIdx = WelsGetNextMbOfSlice (pCurLayer, iCurMbIdx);//whether all of MB in current pSlice encoded or notif (iNextMbIdx == -1 || iNextMbIdx >= kiTotalNumMb || iNumMbCoded >= kiTotalNumMb) {break;}}if (pSlice->iMbSkipRun) {BsWriteUE (pBs, pSlice->iMbSkipRun);}return iEncReturn;
}
WelsMdInterMb
函数
- 基本层的预测编码的核心实现函数,主要用模式决策等过程;
void WelsMdInterMb (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice, SMB* pCurMb, SMbCache* pUnused) {SDqLayer* pCurDqLayer = pEncCtx->pCurDqLayer;SMbCache* pMbCache = &pSlice->sMbCacheInfo;const uint32_t kuiNeighborAvail = pCurMb->uiNeighborAvail;const int32_t kiMbWidth = pCurDqLayer->iMbWidth;const SMB* top_mb = pCurMb - kiMbWidth;const bool bMbLeftAvailPskip = ((kuiNeighborAvail & LEFT_MB_POS) ? IS_SKIP ((pCurMb - 1)->uiMbType) : false);const bool bMbTopAvailPskip = ((kuiNeighborAvail & TOP_MB_POS) ? IS_SKIP (top_mb->uiMbType) : false);const bool bMbTopLeftAvailPskip = ((kuiNeighborAvail & TOPLEFT_MB_POS) ? IS_SKIP ((top_mb - 1)->uiMbType) : false);const bool bMbTopRightAvailPskip = ((kuiNeighborAvail & TOPRIGHT_MB_POS) ? IS_SKIP ((top_mb + 1)->uiMbType) : false);bool bTrySkip = bMbLeftAvailPskip || bMbTopAvailPskip || bMbTopLeftAvailPskip || bMbTopRightAvailPskip;bool bKeepSkip = bMbLeftAvailPskip && bMbTopAvailPskip && bMbTopRightAvailPskip;bool bSkip = false;//try BGD skipif (pEncCtx->pFuncList->pfInterMdBackgroundDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, &bKeepSkip)) {return;}//try static or scrolled Pskipif (pEncCtx->pFuncList->pfSCDPSkipDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache)) {return;}//step 1: try SKIPbSkip = WelsMdInterJudgePskip (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bTrySkip);if (bSkip) {if (bKeepSkip) {WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);return;}} else {PredictSad (pMbCache->sMvComponents.iRefIndexCache, pMbCache->iSadCost, 0, &pWelsMd->iSadPredMb);//step 2: P_16x16pWelsMd->iCostLuma = WelsMdP16x16 (pEncCtx->pFuncList, pCurDqLayer, pWelsMd, pSlice, pCurMb);pCurMb->uiMbType = MB_TYPE_16x16;}WelsMdInterSecondaryModesEnc (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bSkip);
}
WelsMdInterSecondaryModesEnc
函数
- 基本层帧间二级模式的编码实现函数,主要用与判断出了 skip 和 p16x16 块类型之外的模式决策;
void WelsMdInterSecondaryModesEnc (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice, SMB* pCurMb,SMbCache* pMbCache, const bool bSkip) {//step 2: Intraconst bool kbTrySkip = pEncCtx->pFuncList->pfFirstIntraMode (pEncCtx, pWelsMd, pCurMb, pMbCache);if (kbTrySkip)return;if (bSkip) {WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);} else {//Step 3: SubP16 MDpEncCtx->pFuncList->pfSetScrollingMv (pEncCtx->pVaa, pWelsMd); //SCCpEncCtx->pFuncList->pfInterFineMd (pEncCtx, pWelsMd, pSlice, pCurMb, pWelsMd->iCostLuma);//refinement for inter typeWelsMdInterMbRefinement (pEncCtx, pWelsMd, pCurMb, pMbCache);//step 7: invoke encodingWelsMdInterEncode (pEncCtx, pSlice, pCurMb, pMbCache);//step 8: double check PskipWelsMdInterDoubleCheckPskip (pCurMb, pMbCache);}
}
WelsMdSpatialelInterMbIlfmdNoilp
函数
- 增强层的预测编码的核心实现函数,主要用模式决策等过程;
//
// MD for enhancement layers
//
void WelsMdSpatialelInterMbIlfmdNoilp (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice,SMB* pCurMb, const Mb_Type kuiRefMbType) {SDqLayer* pCurDqLayer = pEncCtx->pCurDqLayer;SMbCache* pMbCache = &pSlice->sMbCacheInfo;const uint32_t kuiNeighborAvail = pCurMb->uiNeighborAvail;const int32_t kiMbWidth = pCurDqLayer->iMbWidth;const SMB* kpTopMb = pCurMb - kiMbWidth;const bool kbMbLeftAvailPskip = ((kuiNeighborAvail & LEFT_MB_POS) ? IS_SKIP ((pCurMb - 1)->uiMbType) : false);const bool kbMbTopAvailPskip = ((kuiNeighborAvail & TOP_MB_POS) ? IS_SKIP (kpTopMb->uiMbType) : false);const bool kbMbTopLeftAvailPskip = ((kuiNeighborAvail & TOPLEFT_MB_POS) ? IS_SKIP ((kpTopMb - 1)->uiMbType) : false);const bool kbMbTopRightAvailPskip = ((kuiNeighborAvail & TOPRIGHT_MB_POS) ? IS_SKIP ((kpTopMb + 1)->uiMbType) : false);bool bTrySkip = kbMbLeftAvailPskip | kbMbTopAvailPskip | kbMbTopLeftAvailPskip | kbMbTopRightAvailPskip;bool bKeepSkip = kbMbLeftAvailPskip & kbMbTopAvailPskip & kbMbTopRightAvailPskip;bool bSkip = false;if (pEncCtx->pFuncList->pfInterMdBackgroundDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, &bKeepSkip)) {return;}//step 1: try SKIPbSkip = WelsMdInterJudgePskip (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bTrySkip);if (bSkip && bKeepSkip) {WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);return;}if (! IS_SVC_INTRA (kuiRefMbType)) {if (!bSkip) {PredictSad (pMbCache->sMvComponents.iRefIndexCache, pMbCache->iSadCost, 0, &pWelsMd->iSadPredMb);//step 2: P_16x16pWelsMd->iCostLuma = WelsMdP16x16 (pEncCtx->pFuncList, pCurDqLayer, pWelsMd, pSlice, pCurMb);pCurMb->uiMbType = MB_TYPE_16x16;}WelsMdInterSecondaryModesEnc (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bSkip);} else { //BLMODE == SVC_INTRA//initial prediction memory for I_16x16const int32_t kiCostI16x16 = WelsMdI16x16 (pEncCtx->pFuncList, pEncCtx->pCurDqLayer, pMbCache, pWelsMd->iLambda);if (bSkip && (pWelsMd->iCostLuma <= kiCostI16x16)) {WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);} else {pWelsMd->iCostLuma = kiCostI16x16;pCurMb->uiMbType = MB_TYPE_INTRA16x16;WelsMdIntraSecondaryModesEnc (pEncCtx, pWelsMd, pCurMb, pMbCache);}}
}