View Issue Details
| ID | Project | Category | View Status | Date Submitted | Last Update |
|---|---|---|---|---|---|
| 0000293 | JVT JM H.264/AVC reference software | encoder | public | 2012-02-23 09:49 | 2012-02-23 22:44 |
| Reporter | Pierre Andrivon | Assigned To | |||
| Priority | normal | Severity | minor | Reproducibility | always |
| Status | new | Resolution | open | ||
| Platform | Dell Precision T5500 | OS | Windows 7 x64 | OS Version | Enterprise |
| Product Version | JM 18.3 | ||||
| Summary | 0000293: Fixing 14 bits content encoding RDOQ overflows when 32 bits compiled | ||||
| Description | When compiled with 32 bits compilers (int = 32 bits), considering some (14 bits to be encoded) contents with high luma DC, overflows may appear in RDOQ trellis functions (int32 multiplication overflows int32 lvalue or rvalue temporary) leading to bad encoding and error propagation. The problem did not appear for x64 compilation. A fix is proposed using ad hoc casts (attached). It stands for a good trade-off that is not too much invasive in the source code even if it might impact slightly speed performances. Alternative could be to change variables types but this may cascade changes. | ||||
| Tags | No tags attached. | ||||
|
|
20120223-32bits-14bitsContent-bugfix.patch (34,028 bytes)
Index: lencod/src/rdoq_cabac.c
===================================================================
--- lencod/src/rdoq_cabac.c (revision 2693)
+++ lencod/src/rdoq_cabac.c (revision 2694)
@@ -628,7 +628,8 @@
int end_coeff_ctr = ( ( type == LUMA_4x4 ) ? 16 : 15 );
int q_bits = Q_BITS + qp_per;
int q_offset = ( 1 << (q_bits - 1) );
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
double err, estErr;
for (coeff_ctr = 0; coeff_ctr < end_coeff_ctr; coeff_ctr++)
@@ -650,11 +651,11 @@
{
estErr = ((double) estErr4x4[qp_rem][j][i]) / currSlice->norm_factor_4x4;
- scaled_coeff = iabs(*m7) * q_params_4x4[j][i].ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params_4x4[j][i].ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt = ((scaled_coeff - (level << q_bits)) < q_offset )? 1 : 0;
+ lowerInt = ((scaled_coeff - ((int64)level << q_bits)) < q_offset )? 1 : 0;
#if RDOQ_SQ
dataLevel->level[0] = max(0, level - 1);
@@ -694,7 +695,7 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
}
@@ -721,7 +722,7 @@
int q_offset = ( 1 << (q_bits - 1) );
double err, estErr;
int level, lowerInt, k;
- int scaled_coeff;
+ int64 scaled_coeff;
for (coeff_ctr = 0; coeff_ctr < end_coeff_ctr; coeff_ctr++)
{
@@ -742,11 +743,11 @@
{
estErr = (double) estErr8x8[qp_rem][j][i] / currSlice->norm_factor_8x8;
- scaled_coeff = iabs(*m7) * q_params[j][i].ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params[j][i].ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt = ((scaled_coeff - (level << q_bits)) < q_offset )? 1 : 0;
+ lowerInt = ((scaled_coeff - ((int64)level << q_bits)) < q_offset )? 1 : 0;
#if RDOQ_SQ
dataLevel->level[0] = max(0, level - 1);
@@ -786,7 +787,7 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
}
@@ -811,7 +812,8 @@
int i, j, coeff_ctr, end_coeff_ctr = 16;
int q_bits = Q_BITS + qp_per + 1;
int q_offset = ( 1 << (q_bits - 1) );
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
int *m7;
double err, estErr = (double) estErr4x4[qp_rem][0][0] / currSlice->norm_factor_4x4; // note that we could also use int64
@@ -831,11 +833,11 @@
}
else
{
- scaled_coeff = iabs(*m7) * q_params->ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params->ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt=( (scaled_coeff - (level<<q_bits)) < q_offset )? 1 : 0;
+ lowerInt=( (scaled_coeff - ((int64)level<<q_bits)) < q_offset )? 1 : 0;
#if RDOQ_SQ
dataLevel->level[0] = max(0, level - 1);
@@ -875,7 +877,7 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
}
Index: lencod/src/quant4x4_around.c
===================================================================
--- lencod/src/quant4x4_around.c (revision 2693)
+++ lencod/src/quant4x4_around.c (revision 2694)
@@ -62,7 +62,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -84,15 +84,15 @@
if (*m7 != 0)
{
q_params = &q_params_4x4[j][i];
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
- level = (scaled_coeff + q_params->OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params->OffsetComp) >> q_bits);
if (level != 0)
{
if (is_cavlc)
level = imin(level, CAVLC_LEVEL_LIMIT);
- *padjust4x4 = rshift_rnd_sf((AdaptRndWeight * (scaled_coeff - (level << q_bits))), q_bits + 1);
+ *padjust4x4 = rshift_rnd_sf((AdaptRndWeight * (int)(scaled_coeff - (int64)(level << q_bits))), q_bits + 1);
*coeff_cost += (level > 1) ? MAX_VALUE : c_cost[run];
@@ -148,7 +148,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -168,15 +168,15 @@
if (*m7 != 0)
{
q_params = &q_params_4x4[j][i];
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
- level = (scaled_coeff + q_params->OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params->OffsetComp) >> q_bits);
if (level != 0)
{
if (is_cavlc)
level = imin(level, CAVLC_LEVEL_LIMIT);
- *padjust4x4 = rshift_rnd_sf((AdaptRndWeight * (scaled_coeff - (level << q_bits))), (q_bits + 1));
+ *padjust4x4 = rshift_rnd_sf((AdaptRndWeight * (int)(scaled_coeff - (int64)(level << q_bits))), (q_bits + 1));
*coeff_cost += (level > 1) ? MAX_VALUE : c_cost[run];
@@ -225,7 +225,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -245,8 +245,8 @@
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
Index: lencod/src/rdoq.c
===================================================================
--- lencod/src/rdoq.c (revision 2693)
+++ lencod/src/rdoq.c (revision 2694)
@@ -54,7 +54,8 @@
int q_bits = Q_BITS + qp_per + 1;
int q_offset = ( 1 << (q_bits - 1) );
double err;
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
double estErr = (double) estErr4x4[qp_rem][0][0] / currSlice->norm_factor_4x4; // note that we could also use int64
for (coeff_ctr = 0; coeff_ctr < end_coeff_ctr; coeff_ctr++)
@@ -75,11 +76,11 @@
}
else
{
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt = ((scaled_coeff - (level << q_bits)) < q_offset )? 1 : 0;
+ lowerInt = ((scaled_coeff - ((int64)level << q_bits)) < q_offset )? 1 : 0;
dataLevel->level[0] = 0;
if (level == 0 && lowerInt == 1)
{
@@ -104,14 +105,14 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
if(dataLevel->noLevels == 1)
dataLevel->pre_level = 0;
else
- dataLevel->pre_level = (iabs (*m7) * q_params->ScaleComp + q_params->OffsetComp) >> q_bits;
+ dataLevel->pre_level = (int)(((int64)iabs (*m7) * (int64)q_params->ScaleComp + (int64)q_params->OffsetComp) >> q_bits);
dataLevel->sign = isign(*m7);
}
dataLevel++;
@@ -138,7 +139,8 @@
int q_bits = Q_BITS + qp_per + 1;
int q_offset = ( 1 << (q_bits - 1) );
double err;
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
double estErr = (double) estErr4x4[qp_rem][0][0] / currSlice->norm_factor_4x4; // note that we could also use int64
for (coeff_ctr = 0; coeff_ctr < end_coeff_ctr; coeff_ctr++)
@@ -157,11 +159,11 @@
}
else
{
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt=( (scaled_coeff - (level << q_bits)) < q_offset )? 1 : 0;
+ lowerInt=( (scaled_coeff - ((int64)level << q_bits)) < q_offset )? 1 : 0;
dataLevel->level[0] = 0;
if (level == 0)
{
@@ -196,7 +198,7 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
}
Index: lencod/src/quant8x8_around.c
===================================================================
--- lencod/src/quant8x8_around.c (revision 2693)
+++ lencod/src/quant8x8_around.c (revision 2694)
@@ -62,7 +62,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -84,12 +84,12 @@
if (*m7 != 0)
{
q_params = &q_params_8x8[j][i];
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
- level = (scaled_coeff + q_params->OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params->OffsetComp) >> q_bits);
if (level != 0)
{
- *padjust8x8 = rshift_rnd_sf((AdaptRndWeight * (scaled_coeff - (level << q_bits))), (q_bits + 1));
+ *padjust8x8 = rshift_rnd_sf((AdaptRndWeight * (int)(scaled_coeff - (int64)(level << q_bits))), (q_bits + 1));
nonzero = TRUE;
@@ -150,7 +150,7 @@
int i,j, k, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, runs[4] = { 0 };
int nonzero = FALSE;
@@ -179,14 +179,14 @@
m7 = &tblock[j][block_x + i];
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_8x8[j][i].ScaleComp;
- level = (scaled_coeff + q_params_8x8[j][i].OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_8x8[j][i].ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params_8x8[j][i].OffsetComp) >> q_bits);
if (level != 0)
{
level = imin(level, CAVLC_LEVEL_LIMIT);
- *padjust8x8 = rshift_rnd_sf((AdaptRndWeight * (scaled_coeff - (level << q_bits))), (q_bits + 1));
+ *padjust8x8 = rshift_rnd_sf((AdaptRndWeight * (int)(scaled_coeff - (int64)(level << q_bits))), (q_bits + 1));
nonzero=TRUE;
Index: lencod/src/quantChroma_around.c
===================================================================
--- lencod/src/quantChroma_around.c (revision 2693)
+++ lencod/src/quantChroma_around.c (revision 2694)
@@ -42,7 +42,7 @@
int coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -61,8 +61,8 @@
// every 2x2 DC position
if (*m7)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
@@ -116,7 +116,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -136,8 +136,8 @@
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
Index: lencod/src/block.c
===================================================================
--- lencod/src/block.c (revision 2693)
+++ lencod/src/block.c (revision 2694)
@@ -279,7 +279,6 @@
nonzero = currSlice->quant_dc4x4(currMB, &currSlice->tblk4x4[0], qp, DCLevel, DCRun, &quant_methods.q_params[0][0], pos_scan);
-
// inverse DC transform
if (nonzero)
{
@@ -1601,14 +1600,14 @@
// decide prediction
// case 1
- level1 = (iabs (currSlice->tblk16x16[j+block_y][i+block_x]) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp;
- level1 = (level1 << q_bits_sp) / quant_params_sp[j][i].ScaleComp;
+ level1 = (int)(((int64)iabs (currSlice->tblk16x16[j+block_y][i+block_x]) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp);
+ level1 = (int)(((int64)level1 << q_bits_sp) / quant_params_sp[j][i].ScaleComp);
c_err1 = mb_rres[j+block_y][i+block_x] - isignab(level1, currSlice->tblk16x16[j+block_y][i+block_x]);
- level1 = (iabs (c_err1) * q_params_4x4[j][i].ScaleComp + qp_const) >> q_bits;
+ level1 = (int)(((int64)iabs (c_err1) * (int64)q_params_4x4[j][i].ScaleComp + (int64)qp_const) >> q_bits);
// case 2
c_err2 = mb_rres[j+block_y][i+block_x] - currSlice->tblk16x16[j+block_y][i+block_x];
- level2 = (iabs (c_err2) * q_params_4x4[j][i].ScaleComp + qp_const) >> q_bits;
+ level2 = (int)(((int64)iabs (c_err2) * (int64)q_params_4x4[j][i].ScaleComp + (int64)qp_const) >> q_bits);
// select prediction
if ((level1 != level2) && (level1 != 0) && (level2 != 0))
@@ -1664,11 +1663,11 @@
if(currSlice->slice_type != SI_SLICE && !p_Vid->sp2_frame_indicator)//stores the SP frame coefficients in p_Vid->lrec, will be useful to encode these and create SI or SP switching frame
{
p_Vid->lrec[currMB->pix_y+block_y+j][currMB->pix_x+block_x+i]=
- isignab((iabs(ilev) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp, ilev);
+ isignab((int)(((int64)iabs(ilev) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp), ilev);
}
// Yrec Qs and DeQs
- mb_rres[j+block_y][i+block_x] = isignab((iabs(ilev) * quant_params_sp[j][i].ScaleComp + qp_const2)>> q_bits_sp, ilev) * (quant_params_sp[j][i].InvScaleComp >> 4) << qp_per_sp;
+ mb_rres[j+block_y][i+block_x] = isignab((int)(((int64)iabs(ilev) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2)>> q_bits_sp), ilev) * (quant_params_sp[j][i].InvScaleComp >> 4) << qp_per_sp;
}
ACLevel[scan_pos] = 0;
@@ -1798,14 +1797,14 @@
ilev=0;
// case 1
- c_err1 = (iabs (mp1[coeff_ctr]) * quant_params_sp[0][0].ScaleComp + 2 * qp_const2) >> (q_bits_sp + 1);
- c_err1 = (c_err1 << (q_bits_sp + 1)) / quant_params_sp[0][0].ScaleComp;
+ c_err1 = (int)(((int64)iabs (mp1[coeff_ctr]) * (int64)quant_params_sp[0][0].ScaleComp + 2 * (int64)qp_const2) >> (q_bits_sp + 1));
+ c_err1 = (int)(((int64)c_err1 << (q_bits_sp + 1)) / quant_params_sp[0][0].ScaleComp);
c_err1 = m1[coeff_ctr] - isignab(c_err1, mp1[coeff_ctr]);
- level1 = (iabs(c_err1) * q_params_4x4[0][0].ScaleComp + 2 * qp_const) >> (q_bits+1);
+ level1 = (int)(((int64)iabs(c_err1) * (int64)q_params_4x4[0][0].ScaleComp + 2 * (int64)qp_const) >> (q_bits+1));
// case 2
c_err2 = m1[coeff_ctr] - mp1[coeff_ctr];
- level2 = (iabs(c_err2) * q_params_4x4[0][0].ScaleComp + 2 * qp_const) >> (q_bits+1);
+ level2 = (int)(((int64)iabs(c_err2) * (int64)q_params_4x4[0][0].ScaleComp + 2 * (int64)qp_const) >> (q_bits+1));
if (level1 != level2 && level1 != 0 && level2 != 0)
{
@@ -1851,9 +1850,9 @@
}
ilev+= mp1[coeff_ctr];
- m1[coeff_ctr]=isignab((iabs(ilev) * quant_params_sp[0][0].ScaleComp + 2 * qp_const2) >> (q_bits_sp+1), ilev) * (quant_params_sp[0][0].InvScaleComp >> 4) << qp_per_sp;
+ m1[coeff_ctr]=isignab((int)(((int64)iabs(ilev) * (int64)quant_params_sp[0][0].ScaleComp + 2 * (int64)qp_const2) >> (q_bits_sp+1)), ilev) * (quant_params_sp[0][0].InvScaleComp >> 4) << qp_per_sp;
if(currSlice->slice_type != SI_SLICE && !p_Vid->sp2_frame_indicator)
- p_Vid->lrec_uv[uv][currMB->pix_c_y + 4*(coeff_ctr & 0x01)][currMB->pix_c_x + 4*(coeff_ctr>>1)]=isignab((iabs(ilev) * quant_params_sp[0][0].ScaleComp + 2 * qp_const2) >> (q_bits_sp+1), ilev);// stores the SP frames coefficients, will be useful to encode SI or switching SP frame
+ p_Vid->lrec_uv[uv][currMB->pix_c_y + 4*(coeff_ctr & 0x01)][currMB->pix_c_x + 4*(coeff_ctr>>1)]=isignab((int)(((int64)iabs(ilev) * (int64)quant_params_sp[0][0].ScaleComp + 2 * (int64)qp_const2) >> (q_bits_sp+1)), ilev);// stores the SP frames coefficients, will be useful to encode SI or switching SP frame
}
DCLevel[scan_pos] = 0;
@@ -1889,14 +1888,14 @@
ilev=0;
// quantization on prediction
- c_err1 = (iabs(currSlice->tblk16x16[n2+j][n1+i]) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp;
- c_err1 = (c_err1 << q_bits_sp) / quant_params_sp[j][i].ScaleComp;
+ c_err1 = (int)((int64)(iabs(currSlice->tblk16x16[n2+j][n1+i]) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp);
+ c_err1 = (int)(((int64)c_err1 << q_bits_sp) / (int64)quant_params_sp[j][i].ScaleComp);
c_err1 = mb_rres[n2+j][n1+i] - isignab(c_err1, currSlice->tblk16x16[n2+j][n1+i]);
- level1 = (iabs(c_err1) * q_params_4x4[j][i].ScaleComp + qp_const) >> q_bits;
+ level1 = (int)(((int64)iabs(c_err1) * (int64)q_params_4x4[j][i].ScaleComp + (int64)qp_const) >> q_bits);
// no quantization on prediction
c_err2 = mb_rres[n2+j][n1+i] - currSlice->tblk16x16[n2+j][n1+i];
- level2 = (iabs(c_err2) * q_params_4x4[j][i].ScaleComp + qp_const) >> q_bits;
+ level2 = (int)(((int64)iabs(c_err2) * (int64)q_params_4x4[j][i].ScaleComp + (int64)qp_const) >> q_bits);
if (level1 != level2 && level1 != 0 && level2 != 0)
{
@@ -1948,8 +1947,8 @@
ilev+=currSlice->tblk16x16[n2+j][n1+i];
if(currSlice->slice_type != SI_SLICE && !p_Vid->sp2_frame_indicator)
if(!( ((n2+j) & 0x03)==0 && ((n1+i) & 0x03) ==0 ))
- p_Vid->lrec_uv[uv][currMB->pix_c_y + n2+j][currMB->pix_c_x + n1 + i]=isignab((iabs(ilev) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp,ilev);//stores the SP frames coefficients, will be useful to encode SI or switching SP frame
- mb_rres[n2+j][n1+i] = isignab((iabs(ilev) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp,ilev) * (quant_params_sp[j][i].InvScaleComp >> 4) << qp_per_sp;
+ p_Vid->lrec_uv[uv][currMB->pix_c_y + n2+j][currMB->pix_c_x + n1 + i]=isignab( (int)( ( (int64)iabs(ilev) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp),ilev );//stores the SP frames coefficients, will be useful to encode SI or switching SP frame
+ mb_rres[n2+j][n1+i] = (int)((int64)(isignab((int)( ( (int64)iabs(ilev) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp), ilev)) * ((int64)quant_params_sp[j][i].InvScaleComp >> 4) << qp_per_sp);
}
quant_methods.ACLevel[scan_pos] = 0;
}
@@ -2037,12 +2036,12 @@
{
for (i=0; i < BLOCK_SIZE; ++i)
{
- mb_rres[j][i]=isignab((iabs(currSlice->tblk16x16[j][i])* q_params_4x4[j][i].ScaleComp+qp_const2)>> q_bits,
+ mb_rres[j][i]=isignab((int)(((int64)iabs(currSlice->tblk16x16[j][i])* (int64)q_params_4x4[j][i].ScaleComp+(int64)qp_const2)>> q_bits),
currSlice->tblk16x16[j][i]) * (q_params_4x4[j][i].InvScaleComp >> 4) <<qp_per;
if(currSlice->slice_type != SI_SLICE && !p_Vid->sp2_frame_indicator)
{
p_Vid->lrec[currMB->pix_y+block_y+j][currMB->pix_x+block_x+i] =
- isignab((iabs(currSlice->tblk16x16[j][i]) * q_params_4x4[j][i].ScaleComp + qp_const2) >> q_bits, currSlice->tblk16x16[j][i]);// stores the SP frames coefficients, will be useful to encode SI or switching SP frame
+ isignab((int)((int64)(iabs(currSlice->tblk16x16[j][i]) * (int64)q_params_4x4[j][i].ScaleComp + (int64)qp_const2) >> q_bits), currSlice->tblk16x16[j][i]);// stores the SP frames coefficients, will be useful to encode SI or switching SP frame
}
}
}
@@ -2150,7 +2149,7 @@
ilev=0;
//quantization of the predicted block
- level1 = (iabs (currSlice->tblk16x16[j][i]) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp;
+ level1 = (int)(((int64)iabs (currSlice->tblk16x16[j][i]) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp);
//substracted from p_Vid->lrec
c_err = mb_rres[j][i]-isignab(level1, currSlice->tblk16x16[j][i]); //substracting the predicted block
@@ -2278,7 +2277,7 @@
ilev=0;
//quantization of predicted DC coeff
- level1 = (iabs (mp1[coeff_ctr]) * quant_params_sp[0][0].ScaleComp + 2 * qp_const2) >> (q_bits_sp + 1);
+ level1 = (int)(((int64)iabs (mp1[coeff_ctr]) * (int64)quant_params_sp[0][0].ScaleComp + 2 * (int64)qp_const2) >> (q_bits_sp + 1));
//substratcted from lrecUV
c_err = m1[coeff_ctr] - isignab(level1, mp1[coeff_ctr]);
level = iabs(c_err);
@@ -2333,7 +2332,7 @@
++run;
ilev=0;
// quantization on prediction
- level1 = (iabs(currSlice->tblk16x16[n2+j][n1+i]) * quant_params_sp[j][i].ScaleComp + qp_const2) >> q_bits_sp;
+ level1 = (int)(((int64)iabs(currSlice->tblk16x16[n2+j][n1+i]) * (int64)quant_params_sp[j][i].ScaleComp + (int64)qp_const2) >> q_bits_sp);
//substracted from p_Vid->lrec
c_err = mb_rres[n2+j][n1+i] - isignab(level1, currSlice->tblk16x16[n2+j][n1+i]);
level = iabs(c_err) ;
Index: lencod/src/quant4x4_normal.c
===================================================================
--- lencod/src/quant4x4_normal.c (revision 2693)
+++ lencod/src/quant4x4_normal.c (revision 2694)
@@ -57,7 +57,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -76,8 +76,8 @@
if (*m7 != 0)
{
q_params = &q_params_4x4[j][i];
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
- level = (scaled_coeff + q_params->OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params->OffsetComp) >> q_bits);
if (level != 0)
{
@@ -134,7 +134,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -152,8 +152,8 @@
if (*m7 != 0)
{
q_params = &q_params_4x4[j][i];
- scaled_coeff = iabs (*m7) * q_params->ScaleComp;
- level = (scaled_coeff + q_params->OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params->ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params->OffsetComp) >> q_bits);
if (level != 0)
{
@@ -205,7 +205,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -225,8 +225,8 @@
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
Index: lencod/src/quant8x8_normal.c
===================================================================
--- lencod/src/quant8x8_normal.c (revision 2693)
+++ lencod/src/quant8x8_normal.c (revision 2694)
@@ -56,7 +56,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -75,8 +75,8 @@
m7 = &tblock[j][block_x + i];
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_8x8[j][i].ScaleComp;
- level = (scaled_coeff + q_params_8x8[j][i].OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_8x8[j][i].ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params_8x8[j][i].OffsetComp) >> q_bits);
if (level != 0)
{
@@ -135,7 +135,7 @@
int i,j, k, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, runs[4] = { 0 };
int nonzero = FALSE;
@@ -162,8 +162,8 @@
m7 = &tblock[j][block_x + i];
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_8x8[j][i].ScaleComp;
- level = (scaled_coeff + q_params_8x8[j][i].OffsetComp) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_8x8[j][i].ScaleComp;
+ level = (int)((scaled_coeff + (int64)q_params_8x8[j][i].OffsetComp) >> q_bits);
if (level != 0)
{
Index: lencod/src/rdoq_cavlc.c
===================================================================
--- lencod/src/rdoq_cavlc.c (revision 2693)
+++ lencod/src/rdoq_cavlc.c (revision 2694)
@@ -81,7 +81,7 @@
int cmp(const void *arg1, const void *arg2)
{
if(((levelDataStruct *)arg2)->levelDouble != ((levelDataStruct *)arg1)->levelDouble)
- return (((levelDataStruct *)arg2)->levelDouble - ((levelDataStruct *)arg1)->levelDouble);
+ return (int)(((levelDataStruct *)arg2)->levelDouble - ((levelDataStruct *)arg1)->levelDouble);
else
return (((levelDataStruct *)arg1)->coeff_ctr - ((levelDataStruct *)arg2)->coeff_ctr);
}
@@ -473,7 +473,8 @@
int end_coeff_ctr = ( ( type == LUMA_4x4 ) ? 16 : 15 );
int q_bits = Q_BITS + qp_per;
int q_offset = ( 1 << (q_bits - 1) );
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
double err, estErr;
@@ -498,11 +499,11 @@
{
estErr = ((double) estErr4x4[qp_rem][j][i]) / currSlice->norm_factor_4x4;
- scaled_coeff = iabs(*m7) * q_params[j][i].ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params[j][i].ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt = ((scaled_coeff - (level << q_bits)) < q_offset )? 1 : 0;
+ lowerInt = ((scaled_coeff - ((int64)level << q_bits)) < q_offset )? 1 : 0;
dataLevel->level[0] = 0;
if (level == 0 && lowerInt == 1)
@@ -528,14 +529,14 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
if(dataLevel->noLevels == 1)
dataLevel->pre_level = 0;
else
- dataLevel->pre_level = (iabs (*m7) * q_params[j][i].ScaleComp + q_params[j][i].OffsetComp) >> q_bits;
+ dataLevel->pre_level = (int)(((int64)iabs (*m7) * (int64)q_params[j][i].ScaleComp + (int64)q_params[j][i].OffsetComp) >> q_bits);
dataLevel->sign = isign(*m7);
}
dataLevel++;
@@ -556,7 +557,8 @@
int i, j, coeff_ctr, end_coeff_ctr = 16;
int q_bits = Q_BITS + qp_per + 1;
int q_offset = ( 1 << (q_bits - 1) );
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
int *m7;
double err, estErr = (double) estErr4x4[qp_rem][0][0] / currSlice->norm_factor_4x4; // note that we could also use int64
@@ -578,11 +580,11 @@
}
else
{
- scaled_coeff = iabs(*m7) * q_params->ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params->ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt=( (scaled_coeff - (level<<q_bits)) < q_offset )? 1 : 0;
+ lowerInt=( (scaled_coeff - ((int64)level<<q_bits)) < q_offset )? 1 : 0;
dataLevel->level[0] = 0;
if (level == 0 && lowerInt == 1)
@@ -608,14 +610,14 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = (err * err * estErr);
}
if(dataLevel->noLevels == 1)
dataLevel->pre_level = 0;
else
- dataLevel->pre_level = (iabs (*m7) * q_params->ScaleComp + q_params->OffsetComp) >> q_bits;
+ dataLevel->pre_level = (int)(((int64)iabs (*m7) * (int64)q_params->ScaleComp + (int64)q_params->OffsetComp) >> q_bits);
dataLevel->sign = isign(*m7);
}
dataLevel++;
@@ -637,7 +639,8 @@
int q_bits = Q_BITS_8 + qp_per;
int q_offset = ( 1 << (q_bits - 1) );
double err, estErr;
- int scaled_coeff, level, lowerInt, k;
+ int64 scaled_coeff;
+ int level, lowerInt, k;
levelDataStruct *dataLevel = &levelData[0][0];
@@ -665,11 +668,11 @@
{
estErr = (double) estErr8x8[qp_rem][j][i] / currSlice->norm_factor_8x8;
- scaled_coeff = iabs(*m7) * q_params[j][i].ScaleComp;
+ scaled_coeff = (int64)iabs(*m7) * (int64)q_params[j][i].ScaleComp;
dataLevel->levelDouble = scaled_coeff;
- level = (scaled_coeff >> q_bits);
+ level = (int)(scaled_coeff >> q_bits);
- lowerInt = ((scaled_coeff - (level << q_bits)) < q_offset ) ? 1 : 0;
+ lowerInt = ((scaled_coeff - ((int64)level << q_bits)) < q_offset ) ? 1 : 0;
dataLevel->level[0] = 0;
if (level == 0 && lowerInt == 1)
@@ -704,14 +707,14 @@
for (k = 0; k < dataLevel->noLevels; k++)
{
- err = (double)(dataLevel->level[k] << q_bits) - (double)scaled_coeff;
+ err = (double)((int64)dataLevel->level[k] << q_bits) - (double)scaled_coeff;
dataLevel->errLevel[k] = err * err * estErr;
}
if(dataLevel->noLevels == 1)
dataLevel->pre_level = 0;
else
- dataLevel->pre_level = (iabs (*m7) * q_params[j][i].ScaleComp + q_params[j][i].OffsetComp) >> q_bits;
+ dataLevel->pre_level = (int)(((int64)iabs (*m7) * (int64)q_params[j][i].ScaleComp + (int64)q_params[j][i].OffsetComp) >> q_bits);
dataLevel->sign = isign(*m7);
}
}
Index: lencod/src/quantChroma_normal.c
===================================================================
--- lencod/src/quantChroma_normal.c (revision 2693)
+++ lencod/src/quantChroma_normal.c (revision 2694)
@@ -42,7 +42,7 @@
int coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -61,8 +61,8 @@
// every 2x2 DC position
if (*m7)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
@@ -115,7 +115,7 @@
int i,j, coeff_ctr;
int *m7;
- int scaled_coeff;
+ int64 scaled_coeff;
int level, run = 0;
int nonzero = FALSE;
@@ -135,8 +135,8 @@
if (*m7 != 0)
{
- scaled_coeff = iabs (*m7) * q_params_4x4->ScaleComp;
- level = (scaled_coeff + (q_params_4x4->OffsetComp << 1) ) >> q_bits;
+ scaled_coeff = (int64)iabs (*m7) * (int64)q_params_4x4->ScaleComp;
+ level = (int)((scaled_coeff + (int64)(q_params_4x4->OffsetComp << 1) ) >> q_bits);
if (level != 0)
{
Index: lencod/src/mmco.c
===================================================================
--- lencod/src/mmco.c (revision 2693)
+++ lencod/src/mmco.c (revision 2694)
@@ -194,7 +194,7 @@
{
tmp_drpm2->difference_of_pic_nums_minus1 += p_Vid->currentSlice->max_frame_num;
}
- if ( tmp_drpm2->difference_of_pic_nums_minus1 >= p_Vid->currentSlice->max_frame_num )
+ if ( (unsigned int)tmp_drpm2->difference_of_pic_nums_minus1 >= p_Vid->currentSlice->max_frame_num )
{
tmp_drpm2->difference_of_pic_nums_minus1 -= p_Vid->currentSlice->max_frame_num;
}
Index: lencod/inc/rdoq.h
===================================================================
--- lencod/inc/rdoq.h (revision 2693)
+++ lencod/inc/rdoq.h (revision 2694)
@@ -139,7 +139,7 @@
typedef struct level_data_struct
{
int level[3];
- int levelDouble;
+ int64 levelDouble;
double errLevel[3];
int noLevels;
int coeff_ctr;
|
|
|
Actually it stands for a general quantization issue when compiled with 32 bits compiler (truncation part of integer promotion when 32 bit compiled) that IS fixed by the patch. (((This problem does not exist in HM code due to ad hoc int64 cast that overcomes integer promotion truncation for 32 bits binary))). |
|
|
I don't think it is an issue adding this in the code. I will let Karsten decide in any case, but your code seems ok to me. |
| Date Modified | Username | Field | Change |
|---|---|---|---|
| 2012-02-23 09:49 | Pierre Andrivon | New Issue | |
| 2012-02-23 09:49 | Pierre Andrivon | File Added: 20120223-32bits-14bitsContent-bugfix.patch | |
| 2012-02-23 17:39 | Pierre Andrivon | Note Added: 0000522 | |
| 2012-02-23 22:44 | Alexis Michael Tourapis | Note Added: 0000523 |
