color.h: give up clamp for limited value

The [document] referenced in the header is far from being strict in this
respect. The values that were clamed to was Nominal Video Range. At
the same time, the Preferred Min./Max. is significantly lower/higher
(16-235 vs 5-246 for 8 bit). This value can be understood as a "soft"
limit while the Total Video Signal Range (1-254) as a hard limit.

Some decoders (FFmpeg HEVC) overshot the nominal values, anyways.

[document]:
https://tech.ebu.ch/docs/r/r103.pdf
This commit is contained in:
Martin Pulec
2024-09-24 11:20:48 +02:00
parent 4fe65769f0
commit 2f450060cc

View File

@@ -68,9 +68,11 @@ static_assert(sizeof(comp_type_t) * 8 >= COMP_BASE + 18, "comp_type_t not wide e
#define KG(kr,kb) (1.-kr-kb)
#ifdef YCBCR_FULL
#define C_EPS 0 // prevent under/overflows when there is no clip
#define Y_LIMIT(out_depth) 1.0
#define CBCR_LIMIT(out_depth) 1.0
#else
#define C_EPS 0.5
#define Y_LIMIT(out_depth) \
(219. * (1 << ((out_depth) - 8)) / ((1 << (out_depth)) - 1))
#define CBCR_LIMIT(out_depth) \
@@ -88,8 +90,6 @@ static_assert(sizeof(comp_type_t) * 8 >= COMP_BASE + 18, "comp_type_t not wide e
#define D (2.*(KR_709+KG_709))
#define E (2.*(1.-KR_709))
#define C_EPS 0.5
#define Y_R(out_depth) \
((comp_type_t) (((KR_709 * Y_LIMIT(out_depth)) * (1 << COMP_BASE)) + \
C_EPS))
@@ -132,8 +132,9 @@ static_assert(sizeof(comp_type_t) * 8 >= COMP_BASE + 18, "comp_type_t not wide e
#define LIMIT_HI_Y(depth) (235 * (1<<((depth)-8)))
#define LIMIT_HI_CBCR(depth) (240 * (1<<((depth)-8)))
#endif
#define CLAMP_LIMITED_Y(val, depth) CLAMP((val), LIMIT_LO(depth), LIMIT_HI_Y(depth))
#define CLAMP_LIMITED_CBCR(val, depth) CLAMP((val), 1<<(depth-4), LIMIT_HI_CBCR(depth))
// TODO: remove
#define CLAMP_LIMITED_Y(val, depth) (val)
#define CLAMP_LIMITED_CBCR(val, depth) (val)
#define R_CR(in_depth, kr, kb) ((2. * (1. - (kr))) / CBCR_LIMIT(in_depth))
#define G_CB(in_depth, kr, kb) \