[PATCH 1/2] drm/amd/display: fix BT.2020 YCbCr limited output CSC matrix

[Nathan Lucas <nlucasgit@gmail.com>]

COLOR_SPACE_YCBCR2020_TYPE, which is selected for
COLOR_SPACE_2020_YCBCR_LIMITED color_space, has coefficients that are
incorrect for limited-range output. Its luma and chroma scaling is
full-range so output is too bright and colors are incorrect.

COLOR_SPACE_YCBCR2020_TYPE is closer to a full-range conversion matrix with
incorrect luma offset, so correct the luma offset for full-range and rename
it to COLOR_SPACE_YCBCR2020_FULL_TYPE.

Add COLOR_SPACE_YCBCR2020_LIMITED_TYPE with correct scaling and range for
limited-range output.

Fix related functions so COLOR_SPACE_YCBCR2020_LIMITED_TYPE and
COLOR_SPACE_YCBCR2020_FULL_TYPE are correctly selected based on
dc_color_space.

Derivation of both matrices follows ITU-T H.273:

Table 4, MatrixCoefficients 9, BT.2020-NCL weights:
KR = 0.2627, KB = 0.0593, KG = 1 - KR - KB = 0.6780.

Equations 45-47 in matrix form:
            [  KR             KG             KB            0 ]
M2020_NCL = [ -KR/(2(1-KB))  -KG/(2(1-KB))   1/2           0 ]
            [  1/2           -KG/(2(1-KR))  -KB/(2(1-KR))  0 ]
            [  0              0              0             1 ]

Limited and Full transforms based on equations 30-32 and 36-38 with bit
depth 10, normalized by 1023:

            [ 876/1023   0         0         64/1023  ]
MLimited  = [ 0          896/1023  0         512/1023 ]
            [ 0          0         896/1023  512/1023 ]
            [ 0          0         0         1        ]

            [ 1023/1023  0         0         0        ]
    MFull = [ 0          1023/1023 0         512/1023 ]
            [ 0          0         1023/1023 512/1023 ]
            [ 0          0         0         1        ]

M2020_NCL_Limited = MLimited x M2020_NCL
M2020_NCL_Full    = MFull x M2020_NCL

The upper three rows of M2020_NCL_* are stored in CR, Y, CB order. Each
M2020_NCL_* value is stored as Round(value * 8192) in its 16-bit
two's-complement representation.

Fixes: 973a9c810c78 ("drm/amd/display: Fix COLOR_SPACE_YCBCR2020_TYPE matrix")
Assisted-by: OpenAI-Codex:GPT-5.6-Sol
Signed-off-by: Nathan Lucas <nlucasgit@gmail.com>
---
 .../drm/amd/display/dc/core/dc_hw_sequencer.c | 31 ++++++++++++-------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 11411fa94665..e686815f9a0e 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -59,7 +59,8 @@ enum dc_color_space_type {
 	COLOR_SPACE_RGB_LIMITED_TYPE,
 	COLOR_SPACE_YCBCR601_TYPE,
 	COLOR_SPACE_YCBCR709_TYPE,
-	COLOR_SPACE_YCBCR2020_TYPE,
+	COLOR_SPACE_YCBCR2020_LIMITED_TYPE,
+	COLOR_SPACE_YCBCR2020_FULL_TYPE,
 	COLOR_SPACE_YCBCR601_LIMITED_TYPE,
 	COLOR_SPACE_YCBCR709_LIMITED_TYPE,
 	COLOR_SPACE_YCBCR709_BLACK_TYPE,
@@ -111,9 +112,15 @@ static const struct out_csc_color_matrix_type output_csc_matrix[] = {
 		{ 0xE00, 0xF349, 0xFEB7, 0x1000,
 		  0x6CE, 0x16E3, 0x24F,  0x200,
 		  0xFCCB, 0xF535, 0xE00, 0x1000} },
-	{ COLOR_SPACE_YCBCR2020_TYPE,
+	/* Corrected. Not included in the TODO above. */
+	{ COLOR_SPACE_YCBCR2020_LIMITED_TYPE,
+		{ 0x0E04, 0xF31D, 0xFEDF, 0x1004,
+		  0x0733, 0x1294, 0x01A0, 0x0201,
+		  0xFC16, 0xF5E6, 0x0E04, 0x1004} },
+	/* Corrected. Not included in the TODO above. */
+	{ COLOR_SPACE_YCBCR2020_FULL_TYPE,
 		{ 0x1000, 0xF149, 0xFEB7, 0x1004,
-		  0x0868, 0x15B2, 0x01E6, 0x201,
+		  0x0868, 0x15B2, 0x01E6, 0,
 		  0xFB88, 0xF478, 0x1000, 0x1004} },
 	{ COLOR_SPACE_YCBCR709_BLACK_TYPE,
 		{ 0x0000, 0x0000, 0x0000, 0x1000,
@@ -180,14 +187,14 @@ static bool is_ycbcr709_type(
 	return ret;
 }
 
-static bool is_ycbcr2020_type(
-	enum dc_color_space color_space)
+static bool is_ycbcr2020_limited_type(enum dc_color_space color_space)
 {
-	bool ret = false;
+	return color_space == COLOR_SPACE_2020_YCBCR_LIMITED;
+}
 
-	if (color_space == COLOR_SPACE_2020_YCBCR_LIMITED || color_space == COLOR_SPACE_2020_YCBCR_FULL)
-		ret = true;
-	return ret;
+static bool is_ycbcr2020_full_type(enum dc_color_space color_space)
+{
+	return color_space == COLOR_SPACE_2020_YCBCR_FULL;
 }
 
 static bool is_ycbcr709_limited_type(
@@ -216,8 +223,10 @@ static enum dc_color_space_type get_color_space_type(enum dc_color_space color_s
 		type = COLOR_SPACE_YCBCR601_LIMITED_TYPE;
 	else if (is_ycbcr709_limited_type(color_space))
 		type = COLOR_SPACE_YCBCR709_LIMITED_TYPE;
-	else if (is_ycbcr2020_type(color_space))
-		type = COLOR_SPACE_YCBCR2020_TYPE;
+	else if (is_ycbcr2020_limited_type(color_space))
+		type = COLOR_SPACE_YCBCR2020_LIMITED_TYPE;
+	else if (is_ycbcr2020_full_type(color_space))
+		type = COLOR_SPACE_YCBCR2020_FULL_TYPE;
 	else if (color_space == COLOR_SPACE_YCBCR709)
 		type = COLOR_SPACE_YCBCR709_BLACK_TYPE;
 	else if (color_space == COLOR_SPACE_YCBCR709_BLACK)
-- 
2.55.0