* [PATCH v17 18/28] drm/connector: Register color format property on HDMI connectors
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli, Dmitry Baryshkov
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
The drmm_connector_hdmi_init function can figure out what DRM color
formats are supported by a particular connector based on the supported
HDMI format bitmask that's passed in.
Use it to register the drm color format property.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/drm_connector.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 95c65ed18709..26d7449d0feb 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -636,6 +636,10 @@ int drmm_connector_hdmi_init(struct drm_device *dev,
if (max_bpc > 8)
drm_connector_attach_hdr_output_metadata_property(connector);
+ ret = drm_connector_attach_color_format_property(connector, supported_formats);
+ if (ret)
+ return ret;
+
connector->hdmi.funcs = hdmi_funcs;
return 0;
--
2.54.0
^ permalink raw reply related
* [PATCH v17 19/28] drm/tests: hdmi: Add tests for the color_format property
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
Add some KUnit tests to check the color_format property is working as
expected with the HDMI state helper.
Existing tests are extended to also test the
DRM_CONNECTOR_COLOR_FORMAT_AUTO case, in order to avoid duplicating test
cases. For the explicitly selected color format cases, parameterized
tests are added.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c | 236 +++++++++++++++++++++
1 file changed, 236 insertions(+)
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index e89e1af7a811..8eaca9b5b0fe 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -60,6 +60,23 @@ static struct drm_display_mode *find_preferred_mode(struct drm_connector *connec
return preferred;
}
+static struct drm_display_mode *find_420_only_mode(struct drm_connector *connector)
+{
+ struct drm_device *drm = connector->dev;
+ struct drm_display_mode *mode;
+
+ mutex_lock(&drm->mode_config.mutex);
+ list_for_each_entry(mode, &connector->modes, head) {
+ if (drm_mode_is_420_only(&connector->display_info, mode)) {
+ mutex_unlock(&drm->mode_config.mutex);
+ return mode;
+ }
+ }
+ mutex_unlock(&drm->mode_config.mutex);
+
+ return NULL;
+}
+
static int set_connector_edid(struct kunit *test, struct drm_connector *connector,
const void *edid, size_t edid_len)
{
@@ -1547,6 +1564,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_yuv420(struct kunit *test)
* RGB/10bpc
* - The chosen mode has a TMDS character rate lower than the display
* supports in YUV422/12bpc.
+ * - The HDMI connector state's color format property is unset (i.e. AUTO)
*
* Then we will prefer to keep the RGB format with a lower bpc over
* picking YUV422.
@@ -1609,6 +1627,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit
conn_state = conn->state;
KUNIT_ASSERT_NOT_NULL(test, conn_state);
+ KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_bpc, 10);
KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
@@ -1626,6 +1645,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit
* RGB/8bpc
* - The chosen mode has a TMDS character rate lower than the display
* supports in YUV420/12bpc.
+ * - The HDMI connector state's color format property is unset (i.e. AUTO)
*
* Then we will prefer to keep the RGB format with a lower bpc over
* picking YUV420.
@@ -1687,6 +1707,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv420(struct kunit
conn_state = conn->state;
KUNIT_ASSERT_NOT_NULL(test, conn_state);
+ KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_bpc, 8);
KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
@@ -2198,6 +2219,217 @@ static void drm_test_check_disable_connector(struct kunit *test)
drm_modeset_acquire_fini(&ctx);
}
+struct color_format_test_param {
+ enum drm_connector_color_format fmt;
+ enum drm_output_color_format expected;
+ int expected_ret;
+ const char *desc;
+};
+
+/* Test that if:
+ * - an HDMI connector supports RGB, YUV444, YUV422, and YUV420
+ * - the display supports RGB, YUV444, YUV422, and YUV420
+ * - the "color format" property is set
+ * then, for the preferred mode, for a given "color format" option:
+ * - DRM_CONNECTOR_COLOR_FORMAT_AUTO results in an output format of RGB
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR422 results in an output format of YUV422
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR420 results in an output format of YUV420
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR444 results in an output format of YUV444
+ * - DRM_CONNECTOR_COLOR_FORMAT_RGB results in an HDMI output format of RGB
+ */
+static void drm_test_check_hdmi_color_format(struct kunit *test)
+{
+ const struct color_format_test_param *param = test->param_value;
+ struct drm_atomic_helper_connector_hdmi_priv *priv;
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_info *info;
+ struct drm_display_mode *preferred;
+ int ret;
+
+ priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+ BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444),
+ 12,
+ &dummy_connector_hdmi_funcs,
+ test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz);
+ KUNIT_ASSERT_NOT_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ KUNIT_ASSERT_TRUE(test, priv->connector.ycbcr_420_allowed);
+
+ info = &priv->connector.display_info;
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, info);
+ preferred = find_preferred_mode(&priv->connector);
+ KUNIT_ASSERT_TRUE(test, drm_mode_is_420(info, preferred));
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+ conn_state = drm_atomic_get_connector_state(state, &priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ conn_state->color_format = param->fmt;
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, preferred);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_check_only(state);
+ KUNIT_EXPECT_EQ(test, ret, param->expected_ret);
+ KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, param->expected);
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+static const struct color_format_test_param hdmi_color_format_params[] = {
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_AUTO,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+ .expected_ret = 0,
+ .desc = "AUTO -> RGB"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR422,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR422,
+ .expected_ret = 0,
+ .desc = "YCBCR422 -> YUV422"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR420,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR420,
+ .expected_ret = 0,
+ .desc = "YCBCR420 -> YUV420"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR444,
+ .expected_ret = 0,
+ .desc = "YCBCR444 -> YUV444"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_RGB444,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+ .expected_ret = 0,
+ .desc = "RGB -> RGB"
+ },
+};
+
+KUNIT_ARRAY_PARAM_DESC(check_hdmi_color_format, hdmi_color_format_params, desc);
+
+/* Test that if:
+ * - the HDMI connector supports RGB, YUV422, YUV420, and YUV444
+ * - the display has a YUV420-only mode
+ * - the "color format" property is explicitly set (i.e. !AUTO)
+ * then:
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_RGB444 will fail
+ * drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR444 will fail
+ * drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR422 will fail
+ * drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR420 passes
+ * drm_atomic_check_only for the YUV420-only mode
+ */
+static void drm_test_check_hdmi_color_format_420_only(struct kunit *test)
+{
+ const struct color_format_test_param *param = test->param_value;
+ struct drm_atomic_helper_connector_hdmi_priv *priv;
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *dank;
+ int ret;
+
+ priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+ BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444),
+ 12,
+ &dummy_connector_hdmi_funcs,
+ test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+ KUNIT_ASSERT_NOT_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ dank = find_420_only_mode(&priv->connector);
+ KUNIT_ASSERT_NOT_NULL(test, dank);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+ conn_state = drm_atomic_get_connector_state(state, &priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ conn_state->color_format = param->fmt;
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, dank);
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_check_only(state);
+ KUNIT_EXPECT_EQ(test, ret, param->expected_ret);
+ if (!param->expected_ret)
+ KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, param->expected);
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+};
+
+static const struct color_format_test_param hdmi_color_format_420_only_params[] = {
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_RGB444,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+ .expected_ret = -EINVAL,
+ .desc = "RGB should fail"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR444,
+ .expected_ret = -EINVAL,
+ .desc = "YUV444 should fail"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR422,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR422,
+ .expected_ret = -EINVAL,
+ .desc = "YUV422 should fail"
+ },
+ {
+ .fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR420,
+ .expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR420,
+ .expected_ret = 0,
+ .desc = "YUV420 should work"
+ },
+};
+
+KUNIT_ARRAY_PARAM_DESC(check_hdmi_color_format_420_only,
+ hdmi_color_format_420_only_params, desc);
+
static struct kunit_case drm_atomic_helper_connector_hdmi_check_tests[] = {
KUNIT_CASE(drm_test_check_broadcast_rgb_auto_cea_mode),
KUNIT_CASE(drm_test_check_broadcast_rgb_auto_cea_mode_vic_1),
@@ -2227,6 +2459,10 @@ static struct kunit_case drm_atomic_helper_connector_hdmi_check_tests[] = {
KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_8bpc),
KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_10bpc),
KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_12bpc),
+ KUNIT_CASE_PARAM(drm_test_check_hdmi_color_format,
+ check_hdmi_color_format_gen_params),
+ KUNIT_CASE_PARAM(drm_test_check_hdmi_color_format_420_only,
+ check_hdmi_color_format_420_only_gen_params),
/*
* TODO: We should have tests to check that a change in the
* format triggers a CRTC mode change just like we do for the
--
2.54.0
^ permalink raw reply related
* [PATCH v17 20/28] drm/tests: hdmi: Add tests for HDMI helper's mode_valid
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
Add some KUnit tests to verify that the HDMI state helper's mode_valid
implementation does not improperly reject chroma subsampled modes on the
basis of their clock rate not being satisfiable in RGB.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c | 109 +++++++++++++++++++++
1 file changed, 109 insertions(+)
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index 8eaca9b5b0fe..353a261d42da 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -77,6 +77,23 @@ static struct drm_display_mode *find_420_only_mode(struct drm_connector *connect
return NULL;
}
+static struct drm_display_mode *find_420_also_mode(struct drm_connector *connector)
+{
+ struct drm_device *drm = connector->dev;
+ struct drm_display_mode *mode;
+
+ mutex_lock(&drm->mode_config.mutex);
+ list_for_each_entry(mode, &connector->modes, head) {
+ if (drm_mode_is_420_also(&connector->display_info, mode)) {
+ mutex_unlock(&drm->mode_config.mutex);
+ return mode;
+ }
+ }
+ mutex_unlock(&drm->mode_config.mutex);
+
+ return NULL;
+}
+
static int set_connector_edid(struct kunit *test, struct drm_connector *connector,
const void *edid, size_t edid_len)
{
@@ -2745,11 +2762,103 @@ static void drm_test_check_mode_valid_reject_max_clock(struct kunit *test)
KUNIT_EXPECT_EQ(test, preferred->clock, 25200);
}
+/*
+ * Test that drm_hdmi_connector_mode_valid() will accept modes that require a
+ * 4:2:0 chroma subsampling, even if said mode would violate maximum clock
+ * constraints if it used RGB 4:4:4.
+ */
+static void drm_test_check_mode_valid_yuv420_only_max_clock(struct kunit *test)
+{
+ struct drm_atomic_helper_connector_hdmi_priv *priv;
+ struct drm_display_mode *dank;
+ struct drm_connector *conn;
+
+ priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+ BIT(HDMI_COLORSPACE_RGB) |
+ BIT(HDMI_COLORSPACE_YUV420),
+ 8,
+ &dummy_connector_hdmi_funcs,
+ test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+ KUNIT_ASSERT_NOT_NULL(test, priv);
+
+ conn = &priv->connector;
+ KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 200 * 1000);
+
+ dank = find_420_only_mode(conn);
+ KUNIT_ASSERT_NOT_NULL(test, dank);
+ KUNIT_EXPECT_EQ(test, dank->hdisplay, 3840);
+ KUNIT_EXPECT_EQ(test, dank->vdisplay, 2160);
+
+ /*
+ * Note: The mode's "clock" here is not accurate to the actual TMDS
+ * clock that HDMI will use for a subsampled mode. Hence, why the mode's
+ * clock is above the .max_tmds_clock of 200MHz.
+ */
+ KUNIT_EXPECT_EQ(test, dank->clock, 297000);
+}
+
+/*
+ * Test that drm_hdmi_connector_mode_valid() will reject modes that require
+ * 4:2:0 chroma subsampling, if the connector does not support 4:2:0.
+ */
+static void
+drm_test_check_mode_valid_reject_yuv420_only_connector(struct kunit *test)
+{
+ struct drm_atomic_helper_connector_hdmi_priv *priv;
+ struct drm_display_mode *dank;
+ struct drm_connector *conn;
+
+ priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+ BIT(HDMI_COLORSPACE_RGB),
+ 8,
+ &dummy_connector_hdmi_funcs,
+ test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+ KUNIT_ASSERT_NOT_NULL(test, priv);
+
+ conn = &priv->connector;
+ KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 200 * 1000);
+
+ dank = find_420_only_mode(conn);
+ KUNIT_EXPECT_NULL(test, dank);
+}
+
+/*
+ * Test that drm_hdmi_connector_mode_valid() will accept modes that allow (among
+ * other color formats) 4:2:0 chroma subsampling, even if the connector does not
+ * support 4:2:0, but the mode's clock works for RGB 4:4:4.
+ */
+static void
+drm_test_check_mode_valid_accept_yuv420_also_connector_rgb(struct kunit *test)
+{
+ struct drm_atomic_helper_connector_hdmi_priv *priv;
+ struct drm_display_mode *mode;
+ struct drm_connector *conn;
+
+ priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+ BIT(HDMI_COLORSPACE_RGB),
+ 8,
+ &dummy_connector_hdmi_funcs,
+ test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz);
+ KUNIT_ASSERT_NOT_NULL(test, priv);
+
+ conn = &priv->connector;
+ KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 340 * 1000);
+
+ mode = find_420_also_mode(conn);
+ KUNIT_ASSERT_NOT_NULL(test, mode);
+ KUNIT_EXPECT_EQ(test, mode->hdisplay, 3840);
+ KUNIT_EXPECT_EQ(test, mode->vdisplay, 2160);
+ KUNIT_EXPECT_EQ(test, mode->clock, 297000);
+}
+
static struct kunit_case drm_atomic_helper_connector_hdmi_mode_valid_tests[] = {
KUNIT_CASE(drm_test_check_mode_valid),
KUNIT_CASE(drm_test_check_mode_valid_reject),
KUNIT_CASE(drm_test_check_mode_valid_reject_rate),
KUNIT_CASE(drm_test_check_mode_valid_reject_max_clock),
+ KUNIT_CASE(drm_test_check_mode_valid_yuv420_only_max_clock),
+ KUNIT_CASE(drm_test_check_mode_valid_reject_yuv420_only_connector),
+ KUNIT_CASE(drm_test_check_mode_valid_accept_yuv420_also_connector_rgb),
{ }
};
--
2.54.0
^ permalink raw reply related
* [PATCH v17 21/28] drm/tests: bridge: Add KUnit tests for bridge chain format selection
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
With the "color format" property, the bridge chain format selection has
gained increased complexity. Instead of simply finding any sequence of
bus formats that works, the bridge chain format selection needs to pick
a sequence that results in the requested color format.
Add KUnit tests for this new logic. These take the form of some pleasant
preprocessor macros to make it less cumbersome to define test bridges
with a set of possible input and output formats.
The input and output formats are defined for bridges in the form of
tuples, where the first member defines the input format, and the second
member defines the output format that can be produced from this input
format. This means the tests can construct scenarios in which not all
inputs can be converted to all outputs.
Some tests are added to test interesting scenarios to exercise the bus
format selection in the presence of a specific color format request.
Furthermore, tests are added to verify that bridge chains that end in an
HDMI connector will always prefer RGB when the color format is
DRM_CONNECTOR_COLOR_FORMAT_AUTO, as is the behaviour in the HDMI state
helpers.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/tests/drm_bridge_test.c | 787 ++++++++++++++++++++++++++++++++
1 file changed, 787 insertions(+)
diff --git a/drivers/gpu/drm/tests/drm_bridge_test.c b/drivers/gpu/drm/tests/drm_bridge_test.c
index 64b665580a88..92f142ca6695 100644
--- a/drivers/gpu/drm/tests/drm_bridge_test.c
+++ b/drivers/gpu/drm/tests/drm_bridge_test.c
@@ -2,15 +2,23 @@
/*
* Kunit test for drm_bridge functions
*/
+#include <linux/cleanup.h>
+#include <linux/media-bus-format.h>
+
#include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_atomic_uapi.h>
#include <drm/drm_bridge.h>
#include <drm/drm_bridge_connector.h>
#include <drm/drm_bridge_helper.h>
+#include <drm/drm_edid.h>
#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_managed.h>
#include <kunit/device.h>
#include <kunit/test.h>
+#include "drm_kunit_edid.h"
+
/*
* Mimick the typical "private" struct defined by a bridge driver, which
* embeds a bridge plus other fields.
@@ -37,6 +45,21 @@ struct drm_bridge_init_priv {
bool destroyed;
};
+struct drm_bridge_chain_priv {
+ struct drm_device drm;
+ struct drm_encoder encoder;
+ struct drm_plane *plane;
+ struct drm_crtc *crtc;
+ struct drm_connector *connector;
+ unsigned int num_bridges;
+
+ /**
+ * @test_bridges: array of pointers to &struct drm_bridge_priv entries
+ * of which the first @num_bridges entries are valid.
+ */
+ struct drm_bridge_priv **test_bridges;
+};
+
static struct drm_bridge_priv *bridge_to_priv(struct drm_bridge *bridge)
{
return container_of(bridge, struct drm_bridge_priv, bridge);
@@ -95,6 +118,229 @@ static const struct drm_bridge_funcs drm_test_bridge_atomic_funcs = {
.atomic_reset = drm_atomic_helper_bridge_reset,
};
+/**
+ * struct fmt_tuple - a tuple of input/output MEDIA_BUS_FMT_*
+ */
+struct fmt_tuple {
+ u32 in_fmt;
+ u32 out_fmt;
+};
+
+/*
+ * Format mapping that only accepts RGB888, and outputs only RGB888
+ */
+static const struct fmt_tuple rgb8_passthrough[] = {
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+};
+
+/*
+ * Format mapping that only accepts YUV444, and outputs only YUV444
+ */
+static const struct fmt_tuple yuv8_passthrough[] = {
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+};
+
+/*
+ * Format mapping where 8bpc RGB -> 8bpc YUV444, or ID(RGB) or ID(YUV444)
+ */
+static const struct fmt_tuple rgb8_to_yuv8_or_id[] = {
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+};
+
+static const struct fmt_tuple rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420[] = {
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_UYVY8_1X16 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_UYYVYY8_0_5X24 },
+};
+
+/*
+ * Format mapping where 8bpc YUV444 -> 8bpc RGB, or ID(YUV444)
+ */
+static const struct fmt_tuple yuv8_to_rgb8_or_id[] = {
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+};
+
+/*
+ * A format mapping that acts like a video processor that generates an RGB signal
+ */
+static const struct fmt_tuple rgb_producer[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB101010_1X30 },
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB121212_1X36 },
+};
+
+/*
+ * A format mapping that acts like a video processor that generates an 8-bit RGB,
+ * YUV444 or YUV420 signal
+ */
+static const struct fmt_tuple rgb_yuv444_yuv420_producer[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_UYYVYY8_0_5X24 },
+};
+
+static const struct fmt_tuple rgb8_yuv444_yuv422_passthrough[] = {
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_UYVY8_1X16, MEDIA_BUS_FMT_UYVY8_1X16 },
+};
+
+static const struct fmt_tuple yuv444_yuv422_rgb8_passthrough[] = {
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_UYVY8_1X16, MEDIA_BUS_FMT_UYVY8_1X16 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+};
+
+static bool fmt_in_list(const u32 fmt, const u32 *out_fmts, const size_t num_fmts)
+{
+ size_t i;
+
+ for (i = 0; i < num_fmts; i++)
+ if (out_fmts[i] == fmt)
+ return true;
+
+ return false;
+}
+
+/**
+ * get_tuples_out_fmts - Get unique output formats of a &struct fmt_tuple list
+ * @fmt_tuples: array of &struct fmt_tuple
+ * @num_fmt_tuples: number of entries in @fmt_tuples
+ * @out_fmts: target array to store the unique output bus formats
+ *
+ * Returns the number of unique output formats, i.e. the number of entries in
+ * @out_fmts that were populated with sensible values.
+ */
+static size_t get_tuples_out_fmts(const struct fmt_tuple *fmt_tuples,
+ const size_t num_fmt_tuples, u32 *out_fmts)
+{
+ size_t num_unique = 0;
+ size_t i;
+
+ for (i = 0; i < num_fmt_tuples; i++)
+ if (!fmt_in_list(fmt_tuples[i].out_fmt, out_fmts, num_unique))
+ out_fmts[num_unique++] = fmt_tuples[i].out_fmt;
+
+ return num_unique;
+}
+
+#define DEFINE_FMT_FUNCS_FROM_TUPLES(name) \
+static u32 *drm_test_bridge_ ## name ## _out_fmts(struct drm_bridge *bridge, \
+ struct drm_bridge_state *bridge_state, \
+ struct drm_crtc_state *crtc_state, \
+ struct drm_connector_state *conn_state, \
+ unsigned int *num_output_fmts) \
+{ \
+ u32 *out_fmts = kcalloc(ARRAY_SIZE((name)), sizeof(u32), GFP_KERNEL); \
+ \
+ if (out_fmts) \
+ *num_output_fmts = get_tuples_out_fmts((name), ARRAY_SIZE((name)), out_fmts); \
+ else \
+ *num_output_fmts = 0; \
+ \
+ return out_fmts; \
+} \
+ \
+static u32 *drm_test_bridge_ ## name ## _in_fmts(struct drm_bridge *bridge, \
+ struct drm_bridge_state *bridge_state, \
+ struct drm_crtc_state *crtc_state, \
+ struct drm_connector_state *conn_state, \
+ u32 output_fmt, \
+ unsigned int *num_input_fmts) \
+{ \
+ u32 *in_fmts = kcalloc(ARRAY_SIZE((name)), sizeof(u32), GFP_KERNEL); \
+ unsigned int num_fmts = 0; \
+ size_t i; \
+ \
+ if (!in_fmts) { \
+ *num_input_fmts = 0; \
+ return NULL; \
+ } \
+ \
+ for (i = 0; i < ARRAY_SIZE((name)); i++) \
+ if ((name)[i].out_fmt == output_fmt) \
+ in_fmts[num_fmts++] = (name)[i].in_fmt; \
+ \
+ *num_input_fmts = num_fmts; \
+ \
+ return in_fmts; \
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(ident, input_fmts_func, output_fmts_func, \
+ hdmi_write_infoframe_func, \
+ hdmi_clear_infoframe_func) \
+static const struct drm_bridge_funcs (ident) = { \
+ .atomic_enable = drm_test_bridge_atomic_enable, \
+ .atomic_disable = drm_test_bridge_atomic_disable, \
+ .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, \
+ .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, \
+ .atomic_reset = drm_atomic_helper_bridge_reset, \
+ .atomic_get_input_bus_fmts = (input_fmts_func), \
+ .atomic_get_output_bus_fmts = (output_fmts_func), \
+ .hdmi_write_avi_infoframe = (hdmi_write_infoframe_func), \
+ .hdmi_clear_avi_infoframe = (hdmi_clear_infoframe_func), \
+ .hdmi_write_hdmi_infoframe = (hdmi_write_infoframe_func), \
+ .hdmi_clear_hdmi_infoframe = (hdmi_clear_infoframe_func), \
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_FUNC(ident, input_fmts_func, output_fmts_func) \
+ DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(ident, input_fmts_func, output_fmts_func, \
+ NULL, NULL)
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(_name) \
+ DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_FUNC(_name ## _funcs, \
+ drm_test_bridge_ ## _name ## _in_fmts, \
+ drm_test_bridge_ ## _name ## _out_fmts)
+
+static int drm_test_bridge_write_infoframe_stub(struct drm_bridge *bridge,
+ const u8 *buffer, size_t len)
+{
+ return 0;
+}
+
+static int drm_test_bridge_clear_infoframe_stub(struct drm_bridge *bridge)
+{
+ return 0;
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI(_name) \
+ DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(_name ## _hdmi ## _funcs, \
+ drm_test_bridge_ ## _name ## _in_fmts, \
+ drm_test_bridge_ ## _name ## _out_fmts, \
+ drm_test_bridge_write_infoframe_stub, \
+ drm_test_bridge_clear_infoframe_stub)
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv8_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_to_yuv8_or_id)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_to_yuv8_or_id);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv8_to_rgb8_or_id)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv8_to_rgb8_or_id);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb_producer)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb_producer);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb_yuv444_yuv420_producer)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb_yuv444_yuv420_producer);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_yuv444_yuv422_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_yuv444_yuv422_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv444_yuv422_rgb8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv444_yuv422_rgb8_passthrough);
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI(yuv444_yuv422_rgb8_passthrough);
+
KUNIT_DEFINE_ACTION_WRAPPER(drm_bridge_remove_wrapper,
drm_bridge_remove,
struct drm_bridge *);
@@ -178,6 +424,119 @@ drm_test_bridge_init(struct kunit *test, const struct drm_bridge_funcs *funcs)
return priv;
}
+static struct drm_bridge_chain_priv *
+drm_test_bridge_chain_init(struct kunit *test, unsigned int num_bridges,
+ const struct drm_bridge_funcs **funcs)
+{
+ struct drm_bridge_chain_priv *priv;
+ const struct drm_edid *edid;
+ struct drm_bridge *prev;
+ struct drm_encoder *enc;
+ struct drm_bridge *bridge;
+ struct drm_device *drm;
+ bool has_hdmi = false;
+ struct device *dev;
+ unsigned int i;
+ int ret;
+
+ dev = drm_kunit_helper_alloc_device(test);
+ if (IS_ERR(dev))
+ return ERR_CAST(dev);
+
+ priv = drm_kunit_helper_alloc_drm_device(test, dev, struct drm_bridge_chain_priv,
+ drm, DRIVER_MODESET | DRIVER_ATOMIC);
+ if (IS_ERR(priv))
+ return ERR_CAST(priv);
+
+ drm = &priv->drm;
+
+ priv->test_bridges = drmm_kmalloc_array(drm, num_bridges, sizeof(*priv->test_bridges),
+ GFP_KERNEL);
+ if (!priv->test_bridges)
+ return ERR_PTR(-ENOMEM);
+
+ priv->num_bridges = num_bridges;
+
+ for (i = 0; i < num_bridges; i++) {
+ priv->test_bridges[i] = devm_drm_bridge_alloc(dev, struct drm_bridge_priv,
+ bridge, funcs[i]);
+ if (IS_ERR(priv->test_bridges[i]))
+ return ERR_CAST(priv->test_bridges[i]);
+
+ priv->test_bridges[i]->data = priv;
+ }
+
+ priv->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, NULL,
+ NULL, 0, NULL);
+ if (IS_ERR(priv->plane))
+ return ERR_CAST(priv->plane);
+
+ priv->crtc = drm_kunit_helper_create_crtc(test, drm, priv->plane, NULL,
+ NULL, NULL);
+ if (IS_ERR(priv->crtc))
+ return ERR_CAST(priv->crtc);
+
+ enc = &priv->encoder;
+ ret = drmm_encoder_init(drm, enc, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+ if (ret)
+ return ERR_PTR(ret);
+
+ enc->possible_crtcs = drm_crtc_mask(priv->crtc);
+
+ prev = NULL;
+ for (i = 0; i < num_bridges; i++) {
+ bridge = &priv->test_bridges[i]->bridge;
+ bridge->type = DRM_MODE_CONNECTOR_VIRTUAL;
+
+ if (bridge->funcs->hdmi_write_hdmi_infoframe) {
+ has_hdmi = true;
+ bridge->ops |= DRM_BRIDGE_OP_HDMI;
+ bridge->type = DRM_MODE_CONNECTOR_HDMIA;
+ bridge->vendor = "LNX";
+ bridge->product = "KUnit";
+ bridge->supported_formats = (BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420));
+ }
+
+ ret = drm_kunit_bridge_add(test, bridge);
+ if (ret)
+ return ERR_PTR(ret);
+
+ ret = drm_bridge_attach(enc, bridge, prev, 0);
+ if (ret)
+ return ERR_PTR(ret);
+
+ prev = bridge;
+ }
+
+ priv->connector = drm_bridge_connector_init(drm, enc);
+ if (IS_ERR(priv->connector))
+ return ERR_CAST(priv->connector);
+
+ drm_connector_attach_encoder(priv->connector, enc);
+
+ drm_mode_config_reset(drm);
+
+ if (!has_hdmi)
+ return priv;
+
+ scoped_guard(mutex, &drm->mode_config.mutex) {
+ edid = drm_edid_alloc(test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz,
+ ARRAY_SIZE(test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz));
+ if (!edid)
+ return ERR_PTR(-EINVAL);
+
+ drm_edid_connector_update(priv->connector, edid);
+ KUNIT_ASSERT_GT(test, drm_edid_connector_add_modes(priv->connector), 0);
+
+ ret = priv->connector->funcs->fill_modes(priv->connector, 4096, 4096);
+ }
+
+ return priv;
+}
+
/*
* Test that drm_bridge_get_current_state() returns the last committed
* state for an atomic bridge.
@@ -506,14 +865,442 @@ static struct kunit_suite drm_bridge_alloc_test_suite = {
.test_cases = drm_bridge_alloc_tests,
};
+/**
+ * drm_test_bridge_chain_verify_fmt - Verify bridge chain format selection
+ * @test: pointer to KUnit test object
+ * @priv: pointer to a &struct drm_bridge_chain_priv for this chain
+ * @expected: constant array of &struct fmt_tuple describing the expected
+ * input and output bus formats
+ * @num_expected: number of entries in @expected
+ *
+ * Runs the KUNIT_EXPECT clauses to verify the bridge chain format selection
+ * resulted in the expected formats. If %0 is given as a format in a
+ * &struct fmt_tuple, then it is understood to mean "any".
+ *
+ * Must be called with the modeset lock held.
+ */
+static void drm_test_bridge_chain_verify_fmt(struct kunit *test,
+ struct drm_bridge_chain_priv *priv,
+ const struct fmt_tuple *const expected,
+ const unsigned int num_expected)
+{
+ struct drm_bridge_state *bstate;
+ unsigned int i = 0;
+
+ drm_for_each_bridge_in_chain_scoped(&priv->encoder, bridge) {
+ KUNIT_ASSERT_LT(test, i, num_expected);
+
+ bstate = drm_bridge_get_current_state(bridge);
+ if (expected[i].in_fmt)
+ KUNIT_EXPECT_EQ(test, bstate->input_bus_cfg.format,
+ expected[i].in_fmt);
+ if (expected[i].out_fmt)
+ KUNIT_EXPECT_EQ(test, bstate->output_bus_cfg.format,
+ expected[i].out_fmt);
+
+ i++;
+ }
+
+ KUNIT_ASSERT_EQ_MSG(test, i, num_expected,
+ "Fewer bridges (%u) than expected (%u)\n", i, num_expected);
+}
+
+/*
+ * Test that constructs a bridge chain in which an RGB888 producer is chained to
+ * two bridges that will convert from RGB to YUV and from YUV to RGB respectively.
+ *
+ * The test requests an output color_format of RGB using the color_format property,
+ * so to satisfy this request, the bridge chain must take a detour over YUV.
+ */
+static void drm_test_bridge_rgb_yuv_rgb(struct kunit *test)
+{
+ static const struct drm_bridge_funcs *funcs[] = {
+ &rgb_producer_funcs,
+ &rgb8_to_yuv8_or_id_funcs,
+ &yuv8_to_rgb8_or_id_funcs,
+ };
+ static const struct fmt_tuple expected[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ };
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_chain_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ int ret;
+
+ priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ conn_state->color_format = DRM_CONNECTOR_COLOR_FORMAT_RGB444;
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_commit(state);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test in which a bridge capable of producing RGB, YUV444 and YUV420 has to
+ * produce RGB and convert with a downstream bridge in the chain to reach the
+ * requested YUV444 color format, as no direct path exists between its YUV444
+ * and the last bridge.
+ *
+ * The rationale behind this test is to devise a scenario in which naively
+ * assuming any format the video processor can output, and the connector
+ * requests, is the right format to pick, does not work.
+ */
+static void drm_test_bridge_must_convert_to_yuv444(struct kunit *test)
+{
+ static const struct drm_bridge_funcs *funcs[] = {
+ &rgb_yuv444_yuv420_producer_funcs,
+ &rgb8_passthrough_funcs,
+ &rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420_funcs,
+ &rgb8_yuv444_yuv422_passthrough_funcs,
+ };
+ static const struct fmt_tuple expected[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ };
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_chain_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ int ret;
+
+ priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ conn_state->color_format = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444;
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_commit(state);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that no matter the order of bus formats returned by an
+ * HDMI bridge, RGB is preferred on DRM_CONNECTOR_COLOR_FORMAT_AUTO if it's
+ * available.
+ */
+static void drm_test_bridge_hdmi_auto_rgb(struct kunit *test)
+{
+ static const struct drm_bridge_funcs *funcs[] = {
+ &rgb_yuv444_yuv420_producer_funcs,
+ &yuv444_yuv422_rgb8_passthrough_hdmi_funcs,
+ };
+ static const struct fmt_tuple expected[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+ { MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+ };
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_chain_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ int ret;
+
+ priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_commit(state);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
+
+ drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that DRM_CONNECTOR_COLOR_FORMAT_AUTO on non-HDMI connectors
+ * will result in the first bus format on the output to be picked.
+ */
+static void drm_test_bridge_auto_first(struct kunit *test)
+{
+ static const struct drm_bridge_funcs *funcs[] = {
+ &rgb_yuv444_yuv420_producer_funcs,
+ &yuv444_yuv422_rgb8_passthrough_funcs,
+ };
+ static const struct fmt_tuple expected[] = {
+ { MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_YUV8_1X24 },
+ { MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+ };
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_chain_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ int ret;
+
+ priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_commit(state);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that in a configuration of bridge chains where an RGB
+ * producer is hooked to a YUV444-only pass-through, the atomic commit fails as
+ * the bridge format selection cannot find a valid sequence of bus formats.
+ */
+static void drm_test_bridge_rgb_yuv_no_path(struct kunit *test)
+{
+ static const struct drm_bridge_funcs *funcs[] = {
+ &rgb_producer_funcs,
+ &yuv8_passthrough_funcs,
+ &rgb8_yuv444_yuv422_passthrough_funcs,
+ };
+ struct drm_connector_state *conn_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_chain_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ int ret;
+
+ priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ ret = drm_atomic_commit(state);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_EXPECT_EQ(test, ret, -ENOTSUPP);
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+}
+
+static struct kunit_case drm_bridge_bus_fmt_tests[] = {
+ KUNIT_CASE(drm_test_bridge_rgb_yuv_rgb),
+ KUNIT_CASE(drm_test_bridge_must_convert_to_yuv444),
+ KUNIT_CASE(drm_test_bridge_hdmi_auto_rgb),
+ KUNIT_CASE(drm_test_bridge_auto_first),
+ KUNIT_CASE(drm_test_bridge_rgb_yuv_no_path),
+ { }
+};
+
+static struct kunit_suite drm_bridge_bus_fmt_test_suite = {
+ .name = "drm_bridge_bus_fmt",
+ .test_cases = drm_bridge_bus_fmt_tests,
+};
+
kunit_test_suites(
&drm_bridge_get_current_state_test_suite,
&drm_bridge_helper_reset_crtc_test_suite,
&drm_bridge_alloc_test_suite,
+ &drm_bridge_bus_fmt_test_suite,
);
MODULE_AUTHOR("Maxime Ripard <mripard@kernel.org>");
MODULE_AUTHOR("Luca Ceresoli <luca.ceresoli@bootlin.com>");
+MODULE_AUTHOR("Nicolas Frattaroli <nicolas.frattaroli@collabora.com>");
MODULE_DESCRIPTION("Kunit test for drm_bridge functions");
MODULE_LICENSE("GPL");
--
2.54.0
^ permalink raw reply related
* [PATCH v17 22/28] drm/tests: bridge: Add test for HDMI output bus formats helper
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
The common atomic_get_output_bus_fmts helper for HDMI bridge connectors,
called drm_atomic_helper_bridge_get_hdmi_output_bus_fmts, should return
an array of output bus formats depending on the supported formats of the
connector, and the current output BPC.
Add a test to exercise some of this helper.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/tests/drm_bridge_test.c | 184 ++++++++++++++++++++++++++++++++
1 file changed, 184 insertions(+)
diff --git a/drivers/gpu/drm/tests/drm_bridge_test.c b/drivers/gpu/drm/tests/drm_bridge_test.c
index 92f142ca6695..9d4fbe709df4 100644
--- a/drivers/gpu/drm/tests/drm_bridge_test.c
+++ b/drivers/gpu/drm/tests/drm_bridge_test.c
@@ -5,6 +5,7 @@
#include <linux/cleanup.h>
#include <linux/media-bus-format.h>
+#include <drm/drm_atomic_helper.h>
#include <drm/drm_atomic_state_helper.h>
#include <drm/drm_atomic_uapi.h>
#include <drm/drm_bridge.h>
@@ -118,6 +119,28 @@ static const struct drm_bridge_funcs drm_test_bridge_atomic_funcs = {
.atomic_reset = drm_atomic_helper_bridge_reset,
};
+static int dummy_clear_infoframe(struct drm_bridge *bridge)
+{
+ return 0;
+}
+
+static int dummy_write_infoframe(struct drm_bridge *bridge, const u8 *buffer,
+ size_t len)
+{
+ return 0;
+}
+
+static const struct drm_bridge_funcs drm_test_bridge_bus_fmts_funcs = {
+ .atomic_get_output_bus_fmts = drm_atomic_helper_bridge_get_hdmi_output_bus_fmts,
+ .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+ .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+ .atomic_reset = drm_atomic_helper_bridge_reset,
+ .hdmi_write_avi_infoframe = dummy_write_infoframe,
+ .hdmi_write_hdmi_infoframe = dummy_write_infoframe,
+ .hdmi_clear_avi_infoframe = dummy_clear_infoframe,
+ .hdmi_clear_hdmi_infoframe = dummy_clear_infoframe,
+};
+
/**
* struct fmt_tuple - a tuple of input/output MEDIA_BUS_FMT_*
*/
@@ -537,6 +560,83 @@ drm_test_bridge_chain_init(struct kunit *test, unsigned int num_bridges,
return priv;
}
+static struct drm_bridge_init_priv *
+drm_test_bridge_hdmi_init(struct kunit *test, const struct drm_bridge_funcs *funcs,
+ unsigned int supported_formats, int max_bpc)
+{
+ struct drm_bridge_init_priv *priv;
+ struct drm_encoder *enc;
+ struct drm_bridge *bridge;
+ struct drm_device *drm;
+ struct device *dev;
+ int ret;
+
+ dev = drm_kunit_helper_alloc_device(test);
+ if (IS_ERR(dev))
+ return ERR_CAST(dev);
+
+ priv = drm_kunit_helper_alloc_drm_device(test, dev,
+ struct drm_bridge_init_priv, drm,
+ DRIVER_MODESET | DRIVER_ATOMIC);
+ if (IS_ERR(priv))
+ return ERR_CAST(priv);
+
+ priv->test_bridge = devm_drm_bridge_alloc(dev, struct drm_bridge_priv, bridge, funcs);
+ if (IS_ERR(priv->test_bridge))
+ return ERR_CAST(priv->test_bridge);
+
+ priv->test_bridge->data = priv;
+
+ drm = &priv->drm;
+ priv->plane = drm_kunit_helper_create_primary_plane(test, drm,
+ NULL,
+ NULL,
+ NULL, 0,
+ NULL);
+ if (IS_ERR(priv->plane))
+ return ERR_CAST(priv->plane);
+
+ priv->crtc = drm_kunit_helper_create_crtc(test, drm,
+ priv->plane, NULL,
+ NULL,
+ NULL);
+ if (IS_ERR(priv->crtc))
+ return ERR_CAST(priv->crtc);
+
+ enc = &priv->encoder;
+ ret = drmm_encoder_init(drm, enc, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+ if (ret)
+ return ERR_PTR(ret);
+
+ enc->possible_crtcs = drm_crtc_mask(priv->crtc);
+
+ bridge = &priv->test_bridge->bridge;
+ bridge->type = DRM_MODE_CONNECTOR_HDMIA;
+ bridge->supported_formats = supported_formats;
+ bridge->max_bpc = max_bpc;
+ bridge->ops |= DRM_BRIDGE_OP_HDMI;
+ bridge->vendor = "LNX";
+ bridge->product = "KUnit";
+
+ ret = drm_kunit_bridge_add(test, bridge);
+ if (ret)
+ return ERR_PTR(ret);
+
+ ret = drm_bridge_attach(enc, bridge, NULL, 0);
+ if (ret)
+ return ERR_PTR(ret);
+
+ priv->connector = drm_bridge_connector_init(drm, enc);
+ if (IS_ERR(priv->connector))
+ return ERR_CAST(priv->connector);
+
+ drm_connector_attach_encoder(priv->connector, enc);
+
+ drm_mode_config_reset(drm);
+
+ return priv;
+}
+
/*
* Test that drm_bridge_get_current_state() returns the last committed
* state for an atomic bridge.
@@ -784,10 +884,94 @@ static void drm_test_drm_bridge_helper_reset_crtc_legacy(struct kunit *test)
KUNIT_EXPECT_EQ(test, bridge_priv->disable_count, 1);
}
+/*
+ * Test that a bridge using the drm_atomic_helper_bridge_get_hdmi_output_bus_fmts()
+ * function for &drm_bridge_funcs.atomic_get_output_bus_fmts behaves as expected
+ * for an HDMI connector bridge. Does so by creating an HDMI bridge connector
+ * with RGB444, YCBCR444, and YCBCR420 (but not YCBCR422) as supported formats,
+ * sets the output depth to 8 bits per component, and then validates the returned
+ * list of bus formats.
+ */
+static void drm_test_drm_bridge_helper_hdmi_output_bus_fmts(struct kunit *test)
+{
+ struct drm_connector_state *conn_state;
+ struct drm_bridge_state *bridge_state;
+ struct drm_modeset_acquire_ctx ctx;
+ struct drm_bridge_init_priv *priv;
+ struct drm_crtc_state *crtc_state;
+ struct drm_atomic_commit *state;
+ struct drm_display_mode *mode;
+ unsigned int num_output_fmts;
+ struct drm_bridge *bridge;
+ u32 *out_bus_fmts;
+ int ret;
+
+ priv = drm_test_bridge_hdmi_init(test, &drm_test_bridge_bus_fmts_funcs,
+ BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+ BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420),
+ 12);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+ bridge = &priv->test_bridge->bridge;
+
+ drm_modeset_acquire_init(&ctx, 0);
+
+ state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+ conn_state = drm_atomic_get_connector_state(state, priv->connector);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+ conn_state->hdmi.output_bpc = 8;
+
+ mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+ ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+ ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+ if (ret == -EDEADLK) {
+ drm_modeset_backoff(&ctx);
+ goto retry_commit;
+ }
+ KUNIT_ASSERT_EQ(test, ret, 0);
+
+ crtc_state->enable = true;
+ crtc_state->active = true;
+
+ bridge_state = drm_atomic_get_bridge_state(state, bridge);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bridge_state);
+
+ out_bus_fmts = bridge->funcs->atomic_get_output_bus_fmts(
+ bridge, bridge_state, crtc_state, conn_state, &num_output_fmts);
+ KUNIT_EXPECT_NOT_NULL(test, out_bus_fmts);
+ KUNIT_EXPECT_EQ(test, num_output_fmts, 3);
+
+ KUNIT_EXPECT_EQ(test, out_bus_fmts[0], MEDIA_BUS_FMT_RGB888_1X24);
+ KUNIT_EXPECT_EQ(test, out_bus_fmts[1], MEDIA_BUS_FMT_YUV8_1X24);
+ KUNIT_EXPECT_EQ(test, out_bus_fmts[2], MEDIA_BUS_FMT_UYYVYY8_0_5X24);
+
+ drm_modeset_drop_locks(&ctx);
+ drm_modeset_acquire_fini(&ctx);
+
+ kfree(out_bus_fmts);
+}
+
static struct kunit_case drm_bridge_helper_reset_crtc_tests[] = {
KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_atomic),
KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_atomic_disabled),
KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_legacy),
+ KUNIT_CASE(drm_test_drm_bridge_helper_hdmi_output_bus_fmts),
{ }
};
--
2.54.0
^ permalink raw reply related
* [PATCH v17 23/28] drm/bridge: Document bridge chain format selection
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
The bridge chain format selection behaviour was, until now,
undocumented. With the addition of the "color format" DRM property, it's
not sufficiently complex enough that documentation is warranted,
especially for driver authors trying to do the right thing.
Add a high-level overview of how the process is supposed to work, and
mention what the display driver is supposed to do if it wants to make
use of this functionality.
Reviewed-by: Maxime Ripard <mripard@kernel.org>
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
Documentation/gpu/drm-kms-helpers.rst | 6 ++++++
drivers/gpu/drm/drm_bridge.c | 40 +++++++++++++++++++++++++++++++++++
2 files changed, 46 insertions(+)
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 80453dda33b8..94cfc26acecc 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -171,6 +171,12 @@ Bridge Operations
.. kernel-doc:: drivers/gpu/drm/drm_bridge.c
:doc: bridge operations
+Bridge Chain Format Selection
+-----------------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_bridge.c
+ :doc: bridge chain format selection
+
Bridge Connector Helper
-----------------------
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 9d6fcf986022..1daac8a7f4c9 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -198,6 +198,46 @@
* driver.
*/
+/**
+ * DOC: bridge chain format selection
+ *
+ * A bridge chain, from display output processor to connector, may contain
+ * bridges capable of converting between bus formats on their inputs, and
+ * output formats on their outputs. For example, a bridge may be able to convert
+ * from RGB to YCbCr 4:4:4, and pass through YCbCr 4:2:0 as-is, but not convert
+ * from RGB to YCbCr 4:2:0. This means not all input formats map to all output
+ * formats.
+ *
+ * Further adding to this, a desired output color format, as specified with the
+ * "color format" DRM property, might not correspond 1:1 to what the display
+ * driver should set at its output. The bridge chain it feeds into may only be
+ * able to reach the desired output format, if a conversion from a different
+ * starting format is performed.
+ *
+ * To deal with this complexity, the recursive bridge chain bus format selection
+ * logic starts with the last bridge in the chain, usually the connector, and
+ * then recursively walks the chain of bridges backwards to the first bridge,
+ * trying to find a path.
+ *
+ * For a display driver to work in such a scenario, it should read the first
+ * bridge's bridge state to figure out which bus format the chain resolved to.
+ * If the first bridge's input format resolved to %MEDIA_BUS_FMT_FIXED, then its
+ * output format should be used.
+ *
+ * Special handling is done for HDMI as it relates to format selection. Instead
+ * of directly using the "color format" DRM property for bridge chains that end
+ * in HDMI bridges, the bridge chain format selection logic will trust the logic
+ * that set the HDMI output format. For the common HDMI state helper
+ * functionality, this means that %DRM_CONNECTOR_COLOR_FORMAT_AUTO will allow
+ * fallbacks to YCBCr 4:2:0 if the bandwidth requirements would otherwise be too
+ * high but the mode and connector allow it.
+ *
+ * For bridge chains that do not end in an HDMI bridge,
+ * %DRM_CONNECTOR_COLOR_FORMAT_AUTO will be satisfied with the first output
+ * format on the last bridge for which it can find a path back to the first
+ * bridge.
+ */
+
/* Protect bridge_list and bridge_lingering_list */
static DEFINE_MUTEX(bridge_lock);
static LIST_HEAD(bridge_list);
--
2.54.0
^ permalink raw reply related
* [PATCH v17 24/28] drm/connector: Update docs of "colorspace" for color format prop
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
The colorspace property's documentation states that BT2020_RGB and
BT2020_YCC are equivalent, and the output format depends on the driver.
Now that there is a "color format" property that userspace can use to
explicitly set a format, update the colorspace docs to mention this.
The behaviour here is not changed for userspace that doesn't know about
the color format property yet, as the color format property defaults to
"AUTO", where the choice of output format is left up to drivers.
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/drm_connector.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 26d7449d0feb..cbb067d02cb9 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -2582,7 +2582,8 @@ EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
* conversion matrix and convert to the appropriate quantization
* range.
* The variants BT2020_RGB and BT2020_YCC are equivalent and the
- * driver chooses between RGB and YCbCr on its own.
+ * driver chooses between RGB and YCbCr based on the color format
+ * property.
*
* SMPTE_170M_YCC:
* BT709_YCC:
--
2.54.0
^ permalink raw reply related
* [PATCH v17 25/28] drm/i915/hdmi: Add YCBCR444 handling for sink formats
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
In anticipation of userspace being able to explicitly select supported
sink formats, add handling of the YCBCR444 sink format. The AUTO path
does not choose this format, but with explicit format selection added to
the driver, it becomes a possibility.
Check for YCBCR444 support on the sink in sink_bpc_possible, and on the
source and sink in sink_format_valid.
Acked-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/i915/display/intel_hdmi.c | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 9076c2b176ec..6c6a3a0677fe 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -1966,6 +1966,8 @@ static bool intel_hdmi_sink_bpc_possible(struct drm_connector *_connector,
if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR420)
return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_36;
+ else if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR444)
+ return info->edid_hdmi_ycbcr444_dc_modes & DRM_EDID_HDMI_DC_36;
else
return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_36;
case 10:
@@ -1974,6 +1976,8 @@ static bool intel_hdmi_sink_bpc_possible(struct drm_connector *_connector,
if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR420)
return hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_30;
+ else if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR444)
+ return info->edid_hdmi_ycbcr444_dc_modes & DRM_EDID_HDMI_DC_30;
else
return info->edid_hdmi_rgb444_dc_modes & DRM_EDID_HDMI_DC_30;
case 8:
@@ -2021,6 +2025,17 @@ intel_hdmi_mode_clock_valid(struct drm_connector *_connector, int clock,
return status;
}
+static bool
+intel_hdmi_can_ycbcr444(struct intel_connector *connector)
+{
+ const struct intel_display *display = to_intel_display(connector);
+
+ if (!HAS_GMCH(display))
+ return true;
+
+ return false;
+}
+
static enum drm_mode_status
intel_hdmi_sink_format_valid(struct intel_connector *connector,
const struct drm_display_mode *mode,
@@ -2036,6 +2051,15 @@ intel_hdmi_sink_format_valid(struct intel_connector *connector,
!drm_mode_is_420(info, mode))
return MODE_NO_420;
+ return MODE_OK;
+ case INTEL_OUTPUT_FORMAT_YCBCR444:
+ if (!has_hdmi_sink)
+ return MODE_BAD;
+ if (!intel_hdmi_can_ycbcr444(connector))
+ return MODE_BAD;
+ if (!(info->color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444)))
+ return MODE_BAD;
+
return MODE_OK;
case INTEL_OUTPUT_FORMAT_RGB:
return MODE_OK;
--
2.54.0
^ permalink raw reply related
* [PATCH v17 26/28] drm/i915/dp: Add YCBCR444 handling for sink formats
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
In anticipation of userspace being able to explicitly select supported
sink formats, add handling of the YCBCR444 sink format. The AUTO path
does not choose this format, but with explicit format selection added to
the driver, it becomes a possibility.
Check for both source and sink support of YCBCR444 in
intel_dp_sink_format_valid.
Acked-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/i915/display/intel_dp.c | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index f01a6eed3839..7c69d3badf9f 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -1344,6 +1344,16 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector,
8, sink_format, true);
}
+static bool
+intel_dp_can_ycbcr444(struct intel_dp *intel_dp)
+{
+ if (source_can_output(intel_dp, INTEL_OUTPUT_FORMAT_YCBCR444) &&
+ !drm_dp_is_branch(intel_dp->dpcd))
+ return true;
+
+ return false;
+}
+
static enum drm_mode_status
intel_dp_sink_format_valid(struct intel_connector *connector,
const struct drm_display_mode *mode,
@@ -1362,6 +1372,16 @@ intel_dp_sink_format_valid(struct intel_connector *connector,
!drm_mode_is_420(info, mode))
return MODE_NO_420;
+ return MODE_OK;
+ case INTEL_OUTPUT_FORMAT_YCBCR444:
+ if (intel_dp->dfp.min_tmds_clock &&
+ !intel_dp_has_hdmi_sink(intel_dp))
+ return MODE_BAD;
+ if (!intel_dp_can_ycbcr444(intel_dp))
+ return MODE_BAD;
+ if (!(info->color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444)))
+ return MODE_BAD;
+
return MODE_OK;
case INTEL_OUTPUT_FORMAT_RGB:
return MODE_OK;
--
2.54.0
^ permalink raw reply related
* [PATCH v17 27/28] drm/i915/hdmi: Implement "color format" DRM property
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
Implement the "color format" DRM property for HDMI. The values of the
property include RGB, YCbCr420, YCbCr444 and Auto. Auto will pick RGB,
with a fallback to YCbCr420.
The mask of supported formats by the source exposed by the property is
an optimistic scenario, as specific DFP-related caveats can't be
established before an EDID is present.
Should the explicitly requested color format not be supported by the
sink (or by the source in combination with the sink), then an error is
returned to userspace, so that it can make a better choice.
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/i915/display/intel_hdmi.c | 52 ++++++++++++++++++++++++++++---
1 file changed, 48 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 6c6a3a0677fe..9e5d88f07262 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -2322,10 +2322,10 @@ static int intel_hdmi_compute_output_format(struct intel_encoder *encoder,
return intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits);
}
-static int intel_hdmi_compute_formats(struct intel_encoder *encoder,
- struct intel_crtc_state *crtc_state,
- const struct drm_connector_state *conn_state,
- bool respect_downstream_limits)
+static int intel_hdmi_compute_formats_auto(struct intel_encoder *encoder,
+ struct intel_crtc_state *crtc_state,
+ const struct drm_connector_state *conn_state,
+ bool respect_downstream_limits)
{
struct intel_display *display = to_intel_display(encoder);
struct intel_connector *connector = to_intel_connector(conn_state->connector);
@@ -2360,6 +2360,35 @@ static int intel_hdmi_compute_formats(struct intel_encoder *encoder,
return ret;
}
+static int intel_hdmi_compute_formats(struct intel_encoder *encoder,
+ struct intel_crtc_state *crtc_state,
+ const struct drm_connector_state *conn_state,
+ bool respect_downstream_limits)
+{
+ struct intel_connector *connector = to_intel_connector(conn_state->connector);
+
+ switch (conn_state->color_format) {
+ case DRM_CONNECTOR_COLOR_FORMAT_RGB444:
+ return intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_RGB);
+ case DRM_CONNECTOR_COLOR_FORMAT_YCBCR444:
+ return intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_YCBCR444);
+ case DRM_CONNECTOR_COLOR_FORMAT_YCBCR420:
+ return intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_YCBCR420);
+ case DRM_CONNECTOR_COLOR_FORMAT_AUTO:
+ return intel_hdmi_compute_formats_auto(encoder, crtc_state, conn_state,
+ respect_downstream_limits);
+ default:
+ MISSING_CASE(conn_state->color_format);
+ return -EINVAL;
+ }
+}
+
static bool intel_hdmi_is_cloned(const struct intel_crtc_state *crtc_state)
{
return crtc_state->uapi.encoder_mask &&
@@ -2732,6 +2761,20 @@ static const struct drm_connector_helper_funcs intel_hdmi_connector_helper_funcs
.atomic_check = intel_hdmi_connector_atomic_check,
};
+static void
+intel_hdmi_attach_color_format_property(struct intel_connector *connector)
+{
+ unsigned long fmts = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444);
+
+ if (connector->base.ycbcr_420_allowed)
+ fmts |= BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+
+ if (intel_hdmi_can_ycbcr444(connector))
+ fmts |= BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444);
+
+ drm_connector_attach_color_format_property(&connector->base, fmts);
+}
+
static void
intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *_connector)
{
@@ -2744,6 +2787,7 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *_
intel_attach_hdmi_colorspace_property(&connector->base);
drm_connector_attach_content_type_property(&connector->base);
+ intel_hdmi_attach_color_format_property(connector);
if (DISPLAY_VER(display) >= 10)
drm_connector_attach_hdr_output_metadata_property(&connector->base);
--
2.54.0
^ permalink raw reply related
* [PATCH v17 28/28] drm/i915/dp: Implement "color format" DRM property
From: Nicolas Frattaroli @ 2026-06-09 12:44 UTC (permalink / raw)
To: Harry Wentland, Leo Li, Rodrigo Siqueira, Alex Deucher,
Christian König, David Airlie, Simona Vetter,
Maarten Lankhorst, Maxime Ripard, Thomas Zimmermann,
Andrzej Hajda, Neil Armstrong, Robert Foss, Laurent Pinchart,
Jonas Karlman, Jernej Skrabec, Sandy Huang, Heiko Stübner,
Andy Yan, Jani Nikula, Rodrigo Vivi, Joonas Lahtinen,
Tvrtko Ursulin, Dmitry Baryshkov, Sascha Hauer, Rob Herring,
Jonathan Corbet, Shuah Khan, Daniel Stone
Cc: kernel, amd-gfx, dri-devel, linux-kernel, linux-arm-kernel,
linux-rockchip, intel-gfx, intel-xe, linux-doc, wayland-devel,
Nicolas Frattaroli
In-Reply-To: <20260609-color-format-v17-0-35739b5782cc@collabora.com>
Implement the "color format" DRM property for DP. The values of the
property include RGB, YCbCr420, YCbCr444 and Auto. Auto will pick RGB,
with a fallback to YCbCr420.
The mask of supported formats by the source exposed by the property is
an optimistic scenario, as specific DFP-related caveats can't be
established before an EDID is present.
Should the explicitly requested color format not be supported by the
sink (or by the source in combination with the sink), then an error is
returned to userspace, so that it can make a better choice.
Reviewed-by: Daniel Stone <daniel@fooishbar.org>
Signed-off-by: Nicolas Frattaroli <nicolas.frattaroli@collabora.com>
---
drivers/gpu/drm/i915/display/intel_dp.c | 61 ++++++++++++++++++++++++++++++---
1 file changed, 57 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 7c69d3badf9f..8c189e784adb 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -3434,10 +3434,10 @@ intel_dp_compute_output_format(struct intel_encoder *encoder,
}
static int
-intel_dp_compute_formats(struct intel_encoder *encoder,
- struct intel_crtc_state *crtc_state,
- struct drm_connector_state *conn_state,
- bool respect_downstream_limits)
+intel_dp_compute_formats_auto(struct intel_encoder *encoder,
+ struct intel_crtc_state *crtc_state,
+ struct drm_connector_state *conn_state,
+ bool respect_downstream_limits)
{
struct intel_display *display = to_intel_display(encoder);
struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -3473,6 +3473,34 @@ intel_dp_compute_formats(struct intel_encoder *encoder,
return ret;
}
+static int
+intel_dp_compute_formats(struct intel_encoder *encoder,
+ struct intel_crtc_state *crtc_state,
+ struct drm_connector_state *conn_state,
+ bool respect_downstream_limits)
+{
+ switch (conn_state->color_format) {
+ case DRM_CONNECTOR_COLOR_FORMAT_RGB444:
+ return intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_RGB);
+ case DRM_CONNECTOR_COLOR_FORMAT_YCBCR444:
+ return intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_YCBCR444);
+ case DRM_CONNECTOR_COLOR_FORMAT_YCBCR420:
+ return intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+ respect_downstream_limits,
+ INTEL_OUTPUT_FORMAT_YCBCR420);
+ case DRM_CONNECTOR_COLOR_FORMAT_AUTO:
+ return intel_dp_compute_formats_auto(encoder, crtc_state, conn_state,
+ respect_downstream_limits);
+ default:
+ MISSING_CASE(conn_state->color_format);
+ return -EINVAL;
+ }
+}
+
void
intel_dp_audio_compute_config(struct intel_encoder *encoder,
struct intel_crtc_state *pipe_config,
@@ -7027,6 +7055,29 @@ intel_dp_has_gamut_metadata_dip(struct intel_encoder *encoder)
return false;
}
+static void
+intel_dp_attach_color_format_property(struct intel_dp *intel_dp)
+{
+ struct intel_connector *connector = intel_dp->attached_connector;
+ struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+ unsigned long fmts = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444);
+ bool has_pcon = false;
+
+ if (!connector)
+ return;
+
+ if (dig_port && intel_bios_encoder_is_lspcon(dig_port->base.devdata))
+ has_pcon = true;
+
+ if (has_pcon || source_can_output(intel_dp, INTEL_OUTPUT_FORMAT_YCBCR420))
+ fmts |= BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+
+ if (has_pcon || source_can_output(intel_dp, INTEL_OUTPUT_FORMAT_YCBCR444))
+ fmts |= BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444);
+
+ drm_connector_attach_color_format_property(&connector->base, fmts);
+}
+
static void
intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *_connector)
{
@@ -7059,6 +7110,8 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *_connec
if (HAS_VRR(display))
drm_connector_attach_vrr_capable_property(&connector->base);
+
+ intel_dp_attach_color_format_property(intel_dp);
}
static void
--
2.54.0
^ permalink raw reply related
* Re: [PATCH v2] hwmon: add a driver for the temp/voltage sensor on PolarFire SoC
From: Guenter Roeck @ 2026-06-09 12:56 UTC (permalink / raw)
To: Conor Dooley
Cc: linux-hwmon, Lars Randers, Conor Dooley, Jonathan Corbet,
Shuah Khan, Daire McNamara, linux-doc, linux-kernel, linux-riscv,
Valentina.FernandezAlanis
In-Reply-To: <20260609-estrogen-entangled-da00ac932481@spud>
On 6/9/26 02:43, Conor Dooley wrote:
> On Mon, Jun 08, 2026 at 10:03:48AM -0700, Guenter Roeck wrote:
>> On 6/3/26 06:19, Conor Dooley wrote:
>>> From: Lars Randers <lranders@mail.dk>
>>>
>>> Add a driver for the temperature and voltage sensors on PolarFire SoC.
>>> The temperature reports how hot the die is, and the voltages are the
>>> SoC's 1.05, 1.8 and 2.5 volt rails respectively.
>>>
>>> The hardware supports alarms in theory, but there is an erratum that
>>> prevents clearing them once triggered, so no support is added for them.
>>>
>>> The hardware measures voltage with 16 bits, of which 1 is a sign bit and
>>> the remainder holds the voltage as a fixed point integer value. It's
>>> improbable that the hardware will work if the voltages are negative, so
>>> the driver ignores the sign bits.
>>>
>>> There's no dt support etc here because this is the child of a simple-mfd
>>> syscon.
>>>
>>> Signed-off-by: Lars Randers <lranders@mail.dk>
>>> Co-developed-by: Conor Dooley <conor.dooley@microchip.com>
>>> Signed-off-by: Conor Dooley <conor.dooley@microchip.com>
>>
>> Comments inline.
>
> Cheers.
>
...
>>
>> If regmap never returns errors this needs to be documented in the driver.
>
> It's an mmio regmap via a syscon, it evaluates to readl()/writel() so
> there's nothing that can fail /and/ return an error.
> I mean, I can add if (ret) return ret, there's not a clean place to put
> a comment about it I don't think.
>
No, there is no need to add dummy error checks. Just add a comment
either at the top of the driver or maybe even better where the regmap
is initialized and explain the above.
Guenter
^ permalink raw reply
* Re: [PATCH v3 1/3] PM: core: Rename module parameters prefix to "power"
From: Tzung-Bi Shih @ 2026-06-09 13:03 UTC (permalink / raw)
To: Tomasz Figa
Cc: Rafael J. Wysocki, Jonathan Corbet, Greg Kroah-Hartman,
Danilo Krummrich, Shuah Khan, Pavel Machek, Len Brown, linux-doc,
linux-kernel, linux-pm, driver-core, senozhatsky, Randy Dunlap
In-Reply-To: <CAAFQd5Bmm4VeJ2ygOJeQYmdTq1DLBsQdvDkhstA-tUPZr0rB1A@mail.gmail.com>
On Tue, Jun 09, 2026 at 08:17:03PM +0900, Tomasz Figa wrote:
> On Tue, Jun 9, 2026 at 8:10 PM Rafael J. Wysocki <rafael@kernel.org> wrote:
> >
> > On Tue, Jun 9, 2026 at 11:03 AM Tzung-Bi Shih <tzungbi@kernel.org> wrote:
> > >
> > > On Mon, Jun 08, 2026 at 04:11:30PM +0200, Rafael J. Wysocki wrote:
> > > > On Mon, Jun 8, 2026 at 4:16 AM Tzung-Bi Shih <tzungbi@kernel.org> wrote:
> > > > >
> > > > > Currently, the module parameters defined in drivers/base/power/main.c
> > > > > use the default prefix "main" (derived from the filename). The prefix
> > > > > "main" is too generic and non-descriptive for power management
> > > > > parameters.
> > > > >
> > > > > Redefine MODULE_PARAM_PREFIX to "power." at the beginning of the file
> > > > > to group the module parameters under the "power" namespace instead.
> > > > > This makes the parameters more descriptive.
> > > > >
> > > > > Signed-off-by: Tzung-Bi Shih <tzungbi@kernel.org>
> > > > > ---
> > > > > v3:
> > > > > - No changes.
> > > > >
> > > > > v2: https://lore.kernel.org/all/20260604090756.2884671-2-tzungbi@kernel.org
> > > > > - New to the series.
> > > > >
> > > > > v1: Doesn't exist.
> > > > >
> > > > > drivers/base/power/main.c | 3 +++
> > > > > 1 file changed, 3 insertions(+)
> > > > >
> > > > > diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
> > > > > index ed48c292f575..cd864f3a2799 100644
> > > > > --- a/drivers/base/power/main.c
> > > > > +++ b/drivers/base/power/main.c
> > > > > @@ -40,6 +40,9 @@
> > > > > #include "../base.h"
> > > > > #include "power.h"
> > > > >
> > > > > +#undef MODULE_PARAM_PREFIX
> > > > > +#define MODULE_PARAM_PREFIX "power."
> > > >
> > > > "power" may be confused with the power supply support, so I'd rather
> > > > use "pm" or even "pm_sleep" (in which case the "dpm_" prefix could be
> > > > dropped from the new module param name in the next patch).
> > >
> > > Ack, will use "pm_sleep" in the next version.
> > >
> > > Regarding dropping the "dpm_" prefix, should this also apply to the existing
> > > dpm_watchdog_all_cpu_backtrace parameter? Or should we leave it as-is to
> > > avoid breaking existing configurations?
> >
> > Breaking things for someone would be unfortunate.
> >
> > For consistency, let's retain the "dpm_watchdog" part in this name and
> > use it in the new one.
>
> Hmm, doesn't the prefix change already break things?
Theoretically yes, though it's worth noting that this parameter is
relatively new. It was introduced in v6.19 (Oct 2025) via commit
a67818f74512 ("PM: dpm_watchdog: add module param to backtrace all CPUs").
To my knowledge, the flag was originally introduced for ChromeOS [1], which
doesn't actually rely on this module parameter in its production
configuration yet [2].
That said, I can't know the status of all other Linux distributions. If we
want to be 100% safe against breaking existing setups, retaining the "main"
prefix is definitely the safest.
Given its recent introduction and limited known usage, this might be the
best window to rename the prefix before it gains wider adoption.
What are your thoughts?
[1] https://chromium-review.git.corp.google.com/c/chromiumos/third_party/kernel/+/7414781
[2] https://chromium-review.git.corp.google.com/c/chromiumos/third_party/kernel/+/7414721
^ permalink raw reply
* Re: [PATCH v2] platform/x86: thinkpad_acpi: Add USB-C Security (USCS) support
From: Vishnu Sankar @ 2026-06-09 13:03 UTC (permalink / raw)
To: Ilpo Järvinen
Cc: Mark Pearson, hmh, Hans de Goede, corbet, derekjohn.clark, skhan,
LKML, ibm-acpi-devel, linux-doc, platform-driver-x86, vsankar
In-Reply-To: <b6a08dcf-c7ce-d85a-5c18-18371f6a7cbf@linux.intel.com>
Hi Ilpo,
Thanks a lot for the review comments.
On Tue, Jun 9, 2026 at 6:12 PM Ilpo Järvinen
<ilpo.jarvinen@linux.intel.com> wrote:
>
> On Tue, 9 Jun 2026, Vishnu Sankar wrote:
>
> > Newer ThinkPad systems expose a USB-C Security (Restricted Mode) feature.
> > When active, USB-C data connections are disabled while power delivery is
> > preserved. This is useful for kiosk and physically-secured deployments.
> >
> > Hardware interface:
> >
> > The HKEY device exposes a read-only ACPI method USCS():
> >
> > Return value bit layout:
> > Bit 16 : Capability flag (1 = feature present on this SKU)
> > Bit 0 : Current state (0 = security OFF, 1 = security ON)
> >
> > The sysfs attribute is read-only.
> >
> > The Fn+U followed by Fn+S hotkey chord is the only way to toggle the
> > hardware state.
> >
> > Hotkey:
> >
> > Fn+U followed by Fn+S generates HKEY event 0x131e.
> >
> > sysfs interface:
> >
> > /sys/devices/platform/thinkpad_acpi/usb_c_security (read-only)
> > "enabled\n" -- data connections are currently blocked
> > "disabled\n" -- data connections are currently allowed
> >
> > The attribute is hidden on SKUs where the USCS capability bit (bit 16)
> > is not set, so there is no ABI impact on unsupported hardware.
> >
> > Suggested-by: Mark Pearson <mpearson-lenovo@squebb.ca>
> > Signed-off-by: Vishnu Sankar <vishnuocv@gmail.com>
> > ---
> > Changes since v1:
> > - Use guard(mutex) from cleanup.h instead of manual mutex_lock/unlock
> > - Revert usbc_security_query() to return int (-EIO/-ENODEV/0) instead
> > of bool to avoid uninitialized *enabled bug on unsupported platforms
> > - Remove !! when assigning to bool in usbc_security_query()
> > - Remove dead tp_features.usbc_security_supported check in show()
> > since is_visible() already gates the attribute on unsupported SKUs
> > - Use str_enabled_disabled() from string_choices.h in show()
> > - Fix uninitialized *enabled bug in tpacpi_usbc_security_init() by
> > only assigning usbc_security_enabled after a successful query
> > ---
> > .../admin-guide/laptops/thinkpad-acpi.rst | 24 ++++
> > drivers/platform/x86/lenovo/thinkpad_acpi.c | 118 ++++++++++++++++++
> > 2 files changed, 142 insertions(+)
> >
> > diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
> > index f874db31801d..db4588af0278 100644
> > --- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst
> > +++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
> > @@ -1543,6 +1543,30 @@ Values:
> >
> > This setting can also be toggled via the Fn+doubletap hotkey.
> >
> > +USB-C Security
> > +--------------
> > +
> > +sysfs: usb_c_security
> > +
> > +Reports the current state of the USB-C Security (Restricted Mode) feature
> > +on supported ThinkPad systems. When enabled, USB-C data connections are
> > +disabled while power delivery is preserved.
> > +
> > +The available command is::
> > +
> > + cat /sys/devices/platform/thinkpad_acpi/usb_c_security
> > +
> > +Values:
> > +
> > + * ``enabled`` - USB-C data connections are currently blocked
> > + * ``disabled`` - USB-C data connections are currently allowed
> > +
> > +The attribute is read-only. The USB-C Security state can only be toggled
> > +via the Fn+U followed by Fn+S hotkey chord.
> > +
> > +The sysfs attribute is not created on platforms that do not support this
> > +feature.
> > +
> > Auxmac
> > ------
> >
> > diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c
> > index e1cee42a1683..379769b62c80 100644
> > --- a/drivers/platform/x86/lenovo/thinkpad_acpi.c
> > +++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c
> > @@ -38,6 +38,7 @@
> > #include <linux/backlight.h>
> > #include <linux/bitfield.h>
> > #include <linux/bitops.h>
> > +#include <linux/cleanup.h>
> > #include <linux/delay.h>
> > #include <linux/dmi.h>
> > #include <linux/freezer.h>
> > @@ -66,6 +67,7 @@
> > #include <linux/seq_file.h>
> > #include <linux/slab.h>
> > #include <linux/string.h>
> > +#include <linux/string_choices.h>
> > #include <linux/string_helpers.h>
> > #include <linux/sysfs.h>
> > #include <linux/types.h>
> > @@ -185,6 +187,7 @@ enum tpacpi_hkey_event_t {
> > TP_HKEY_EV_AMT_TOGGLE = 0x131a, /* Toggle AMT on/off */
> > TP_HKEY_EV_CAMERASHUTTER_TOGGLE = 0x131b, /* Toggle Camera Shutter */
> > TP_HKEY_EV_DOUBLETAP_TOGGLE = 0x131c, /* Toggle trackpoint doubletap on/off */
> > + TP_HKEY_EV_USB_C_SECURITY = 0x131e, /* Toggle USB C Security ON/OFF */
> > TP_HKEY_EV_PROFILE_TOGGLE = 0x131f, /* Toggle platform profile in 2024 systems */
> > TP_HKEY_EV_PROFILE_TOGGLE2 = 0x1401, /* Toggle platform profile in 2025 + systems */
> >
> > @@ -373,6 +376,8 @@ static struct {
> > u32 has_adaptive_kbd:1;
> > u32 kbd_lang:1;
> > u32 trackpoint_doubletap_enable:1;
> > + u32 usbc_security_supported:1;
> > + u32 usbc_security_enabled:1;
>
> Sashiko (sashiko.dev) warned that there may be concurrent, unprotected
> updates to these bitfields (changing bitfields require unsafe RMW). It
> looks pre-existing problem at least for trackpoint_doubletap_enable (maybe
> others).
>
> To avoid adding to the problems, usbc_security_enabled should be added
> outside the bitfield to avoid need to do locking for this bitfield.
>
> And trackpoint_doubletap_enable (and possibly others) which are touched in
> the notify context or in sysfs write should be fixed in a separate patch
> (can be done after this series as it's pre-existing problem for them).
>
> Anything that is only written during init is fine inside the bitfield.
Agreed.
usbc_security_enabled will be moved outside tp_features as a
static bool since it is written from both init and the hotkey notify
context. usbc_security_supported remains inside the bitfield as it is
only written during init.
I note that trackpoint_doubletap_enable and possibly others have the
same pre-existing issue and will address those in a separate patch as
you suggested.
>
> > struct quirk_entry *quirks;
> > } tp_features;
> >
> > @@ -11265,6 +11270,112 @@ static struct ibm_struct hwdd_driver_data = {
> > .name = "hwdd",
> > };
> >
> > +/*************************************************************************
> > + * USB-C Security subdriver
> > + *
> > + * HKEY.USCS(0) is a read-only ACPI method; its argument is ignored.
> > + * It always returns:
> > + * bit 16 - USB-C security capability present on this SKU or not
> > + * bit 0 - USB-C Security state (enable or disable)
> > + *
> > + * Hotkey
> > + * ------
> > + * 0x131e (Fn+U, Fn+S): firmware toggles USBS before firing the event.
> > + * The driver reads back the new state and notifies the sysfs attribute.
> > + *
>
> Remove the extra line.
Acked
>
> > + */
> > +
> > +/* USCS() return word bit layout */
> > +#define USCS_CAP_BIT BIT(16) /* capability: feature present on SKU */
> > +#define USCS_STATUS_BIT BIT(0) /* current security state */
> > +
> > +static DEFINE_MUTEX(usbc_security_mutex);
> > +
> > +/*
> > + * usbc_security_query - read current USB-C security state via USCS()
> > + * @enabled: out - true when security is ON (data connections blocked)
> > + *
> > + * Returns true if the feature is supported and query succeeded,
>
> Kerneldoc doc compatible syntax is:
>
> Returns:
>
Acked
> > + * false otherwise (feature absent or ACPI call failed).
>
> Please rewrite this as this function no longer returns true/false. :-)
>
Acked
> > + */
> > +static int usbc_security_query(bool *enabled)
> > +{
> > + int status;
> > +
> > + guard(mutex)(&usbc_security_mutex);
> > + if (!acpi_evalf(hkey_handle, &status, "USCS", "dd", 0))
> > + return -EIO;
> > +
> > + if (!(status & USCS_CAP_BIT)) {
> > + pr_debug("USCS cap bit absent (raw=0x%x)\n", status);
> > + return -ENODEV;
> > + }
> > +
> > + *enabled = status & USCS_STATUS_BIT;
> > + return 0;
> > +}
> > +
> > +/* sysfs: /sys/devices/platform/thinkpad_acpi/usb_c_security ---------- */
> > +static ssize_t usb_c_security_show(struct device *dev,
> > + struct device_attribute *attr,
> > + char *buf)
> > +{
> > + return sysfs_emit(buf, "%s\n",
> > + str_enabled_disabled(tp_features.usbc_security_enabled));
> > +}
> > +
> > +static DEVICE_ATTR_RO(usb_c_security);
> > +
> > +static struct attribute *usbc_security_attributes[] = {
> > + &dev_attr_usb_c_security.attr,
> > + NULL,
> > +};
> > +
> > +static umode_t usbc_security_attr_is_visible(struct kobject *kobj,
> > + struct attribute *attr, int n)
> > +{
> > + return tp_features.usbc_security_supported ? attr->mode : 0;
> > +}
> > +
> > +static const struct attribute_group usbc_security_attr_group = {
> > + .is_visible = usbc_security_attr_is_visible,
> > + .attrs = usbc_security_attributes,
> > +};
> > +
> > +static int tpacpi_usbc_security_init(struct ibm_init_struct *iibm)
> > +{
> > + bool enabled;
> > + int err;
> > +
> > + err = usbc_security_query(&enabled);
> > + if (err)
> > + return err == -ENODEV ? 0 : err;
>
> Just split this to two if () + returns for clarity.
>
Acked.
> > +
> > + tp_features.usbc_security_supported = true;
> > + tp_features.usbc_security_enabled = enabled;
> > + return 0;
> > +}
> > +
> > +/* tpacpi_usbc_security_hotkey - handle Fn+U Fn+S hotkey (0x131e) */
> > +static bool tpacpi_usbc_security_hotkey(void)
> > +{
> > + bool enabled;
> > +
> > + if (!tp_features.usbc_security_supported)
> > + return false;
> > +
> > + if (usbc_security_query(&enabled))
> > + return false;
> > +
> > + tp_features.usbc_security_enabled = enabled;
> > + sysfs_notify(&tpacpi_pdev->dev.kobj, NULL, "usb_c_security");
> > + return true;
> > +}
> > +
> > +static struct ibm_struct usbc_security_driver_data = {
> > + .name = "usbc_security",
> > +};
> > +
> > /* --------------------------------------------------------------------- */
> >
> > static struct attribute *tpacpi_driver_attributes[] = {
> > @@ -11325,6 +11436,7 @@ static const struct attribute_group *tpacpi_groups[] = {
> > &dprc_attr_group,
> > &auxmac_attr_group,
> > &hwdd_attr_group,
> > + &usbc_security_attr_group,
> > NULL,
> > };
> >
> > @@ -11479,6 +11591,8 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
> > case TP_HKEY_EV_PROFILE_TOGGLE2:
> > platform_profile_cycle();
> > return true;
> > + case TP_HKEY_EV_USB_C_SECURITY:
> > + return tpacpi_usbc_security_hotkey();
> > }
> >
> > return false;
> > @@ -11930,6 +12044,10 @@ static struct ibm_init_struct ibms_init[] __initdata = {
> > .init = tpacpi_hwdd_init,
> > .data = &hwdd_driver_data,
> > },
> > + {
> > + .init = tpacpi_usbc_security_init,
> > + .data = &usbc_security_driver_data,
> > + },
> > };
> >
> > static int __init set_ibm_param(const char *val, const struct kernel_param *kp)
> >
>
> --
> i.
>
Thank you!
--
Regards,
Vishnu Sankar
^ permalink raw reply
* Re: [PATCH v3 1/3] PM: core: Rename module parameters prefix to "power"
From: Rafael J. Wysocki @ 2026-06-09 13:40 UTC (permalink / raw)
To: Tzung-Bi Shih
Cc: Tomasz Figa, Rafael J. Wysocki, Jonathan Corbet,
Greg Kroah-Hartman, Danilo Krummrich, Shuah Khan, Pavel Machek,
Len Brown, linux-doc, linux-kernel, linux-pm, driver-core,
senozhatsky, Randy Dunlap
In-Reply-To: <aigPNvkxRIz36dWm@google.com>
On Tue, Jun 9, 2026 at 3:03 PM Tzung-Bi Shih <tzungbi@kernel.org> wrote:
>
> On Tue, Jun 09, 2026 at 08:17:03PM +0900, Tomasz Figa wrote:
> > On Tue, Jun 9, 2026 at 8:10 PM Rafael J. Wysocki <rafael@kernel.org> wrote:
> > >
> > > On Tue, Jun 9, 2026 at 11:03 AM Tzung-Bi Shih <tzungbi@kernel.org> wrote:
> > > >
> > > > On Mon, Jun 08, 2026 at 04:11:30PM +0200, Rafael J. Wysocki wrote:
> > > > > On Mon, Jun 8, 2026 at 4:16 AM Tzung-Bi Shih <tzungbi@kernel.org> wrote:
> > > > > >
> > > > > > Currently, the module parameters defined in drivers/base/power/main.c
> > > > > > use the default prefix "main" (derived from the filename). The prefix
> > > > > > "main" is too generic and non-descriptive for power management
> > > > > > parameters.
> > > > > >
> > > > > > Redefine MODULE_PARAM_PREFIX to "power." at the beginning of the file
> > > > > > to group the module parameters under the "power" namespace instead.
> > > > > > This makes the parameters more descriptive.
> > > > > >
> > > > > > Signed-off-by: Tzung-Bi Shih <tzungbi@kernel.org>
> > > > > > ---
> > > > > > v3:
> > > > > > - No changes.
> > > > > >
> > > > > > v2: https://lore.kernel.org/all/20260604090756.2884671-2-tzungbi@kernel.org
> > > > > > - New to the series.
> > > > > >
> > > > > > v1: Doesn't exist.
> > > > > >
> > > > > > drivers/base/power/main.c | 3 +++
> > > > > > 1 file changed, 3 insertions(+)
> > > > > >
> > > > > > diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
> > > > > > index ed48c292f575..cd864f3a2799 100644
> > > > > > --- a/drivers/base/power/main.c
> > > > > > +++ b/drivers/base/power/main.c
> > > > > > @@ -40,6 +40,9 @@
> > > > > > #include "../base.h"
> > > > > > #include "power.h"
> > > > > >
> > > > > > +#undef MODULE_PARAM_PREFIX
> > > > > > +#define MODULE_PARAM_PREFIX "power."
> > > > >
> > > > > "power" may be confused with the power supply support, so I'd rather
> > > > > use "pm" or even "pm_sleep" (in which case the "dpm_" prefix could be
> > > > > dropped from the new module param name in the next patch).
> > > >
> > > > Ack, will use "pm_sleep" in the next version.
> > > >
> > > > Regarding dropping the "dpm_" prefix, should this also apply to the existing
> > > > dpm_watchdog_all_cpu_backtrace parameter? Or should we leave it as-is to
> > > > avoid breaking existing configurations?
> > >
> > > Breaking things for someone would be unfortunate.
> > >
> > > For consistency, let's retain the "dpm_watchdog" part in this name and
> > > use it in the new one.
> >
> > Hmm, doesn't the prefix change already break things?
>
> Theoretically yes, though it's worth noting that this parameter is
> relatively new. It was introduced in v6.19 (Oct 2025) via commit
> a67818f74512 ("PM: dpm_watchdog: add module param to backtrace all CPUs").
>
> To my knowledge, the flag was originally introduced for ChromeOS [1], which
> doesn't actually rely on this module parameter in its production
> configuration yet [2].
>
> That said, I can't know the status of all other Linux distributions. If we
> want to be 100% safe against breaking existing setups, retaining the "main"
> prefix is definitely the safest.
>
> Given its recent introduction and limited known usage, this might be the
> best window to rename the prefix before it gains wider adoption.
>
> What are your thoughts?
If somebody complains about it, we can reconsider, but using "main" as
a prefix for power management module parameters is definitely
confusing.
> [1] https://chromium-review.git.corp.google.com/c/chromiumos/third_party/kernel/+/7414781
> [2] https://chromium-review.git.corp.google.com/c/chromiumos/third_party/kernel/+/7414721
^ permalink raw reply
* Re: [PATCH ipsec-next v9 00/16] xfrm: XFRM_MSG_MIGRATE_STATE new netlink message
From: Steffen Klassert @ 2026-06-09 14:05 UTC (permalink / raw)
To: Antony Antony
Cc: Herbert Xu, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Simon Horman, David Ahern, Masahide NAKAMURA,
Paul Moore, Stephen Smalley, Ondrej Mosnacek, Jonathan Corbet,
Shuah Khan, Sabrina Dubroca, netdev, linux-kernel, selinux,
linux-doc, Chiachang Wang, Yan Yan, devel
In-Reply-To: <migrate-state-v9-0-ad9947e4ae74@secunet.com>
On Tue, May 26, 2026 at 09:05:38PM +0200, Antony Antony wrote:
> The current XFRM_MSG_MIGRATE interface is tightly coupled to policy and
> SA migration, and it lacks the information required to reliably migrate
> individual SAs. This makes it unsuitable for IKEv2 deployments,
> dual-stack setups (IPv4/IPv6), and scenarios where policies are managed
> externally (e.g., by daemons other than the IKE daemon).
>
> Mandatory SA selector list
> The current API requires a non-empty SA selector list, which does not
> reflect the IKEv2 use case.
> A single Child SA may correspond to multiple policies,
> and SA discovery already occurs via address and reqid matching. With
> dual-stack Child SAs this leads to excessive churn: the current method
> would have to be called up to six times (in/out/fwd × v4/v6) on SA,
> while the new method only requires two calls.
>
> Selectors lack SPI (and marks)
> XFRM_MSG_MIGRATE cannot uniquely identify an SA when multiple SAs share
> the same policies (per-CPU SAs, SELinux label-based SAs, etc.). Without
> the SPI, the kernel may update the wrong SA instance.
>
> Reqid cannot be changed
> Some implementations allocate reqids based on traffic selectors. In
> host-to-host or selector-changing scenarios, the reqid must change,
> which the current API cannot express.
>
> Because strongSwan and other implementations manage policies
> independently of the kernel, an interface that updates only a specific
> SA - with complete and unambiguous identification - is required.
>
> SA Selector, x->sel, can't be changed, especially Transport mode.
>
> XFRM_MSG_MIGRATE_STATE provides that interface. It supports migration
> of a single SA via xfrm_usersa_id (including SPI) and we fix
> encap removal in this patch set, reqid updates, address changes,
> and other SA-specific parameters. It avoids the structural limitations
> of XFRM_MSG_MIGRATE and provides a simpler, extensible mechanism for
> precise per-SA migration without involving policies.
> This method also allows migtrating SA selectors typically used with
> host-to-host in Transport mode.
>
> New migration steps: first install block policy, remove the old policy,
> call XFRM_MSG_MIGRATE_STATE for each state, then re-install the
> policies and remove the block policy.
>
> If the target SA tuple (daddr, SPI, proto, family) is already
> occupied, the operation returns -EEXIST. In this case the original
> SA is not preserved. Userspace must handle -EEXIST by
> re-establishing the SA at the IKE level and manage policies.
>
> ---
> v8->v9: address v8 review Sabrina and Steffen
> - split const xuo cleanup
> - input validation
>
> Link to v8: https://lore.kernel.org/all/migrate-state-v8-0-4578fb016965@secunet.com/
> v7->v8: - removed the unknown-flags validation block
>
> Link to v7: https://patch.msgid.link/migrate-state-v7-14-44eb2440b91c@secunet.com
> v6->v7: - add SA selectoor migration
> - fixes to commit messages
> - white space removal
>
> Link to v6: https://lore.kernel.org/r/migrate-state-v6-0-9df9764ddb9e@secunet.com
> v5->v6: - add mark to look up SA.
> - restrict netlink attributes in new method
> - address review feedback from Sabrina
> - add new patch to fix existing inter-family address comparison
> - add extack xfrm_state_init()
> - Feedback from Yan : omit-to-inherit add migrating marks
> - Drop missing __rcu annotation on nlsk, Sabrina has a better patch
>
> Link to v5: https://lore.kernel.org/all/cover.1769509130.git.antony.antony@secunet.com/
> v4->v5: add synchronize after migrate and delete it inside a lock
> - split xfrm_state_migrate into create and install functions
>
> Link to v4: https://lore.kernel.org/all/cover.1768811736.git.antony.antony@secunet.com/
> v3->v4: add patch to fix pre-existing missing __rcu annotation on nlsk
>
> v2->v3: - fix commit message formatting
>
> Link to v2: https://lore.kernel.org/all/cover.1768462955.git.antony.antony@secunet.com/
>
> Lint to v1: https://lore.kernel.org/all/cover.1767964254.git.antony@moon.secunet.de/
> v1->v2: dropped 6/6. That check is already there where the func is called
> - merged patch 4/6 and 5/6, to fix use uninitialized value
> - fix commit messages
>
> ---
> Antony Antony (16):
> xfrm: remove redundant assignments
> xfrm: add extack to xfrm_init_state
> xfrm: allow migration from UDP encapsulated to non-encapsulated ESP
> xfrm: fix NAT-related field inheritance in SA migration
> xfrm: rename reqid in xfrm_migrate
> xfrm: split xfrm_state_migrate into create and install functions
> xfrm: check family before comparing addresses in migrate
> xfrm: add state synchronization after migration
> xfrm: add error messages to state migration
> xfrm: move encap and xuo into struct xfrm_migrate
> xfrm: refactor XFRMA_MTIMER_THRESH validation into a helper
> xfrm: extract address family and selector validation helpers
> xfrm: make xfrm_dev_state_add xuo parameter const
> xfrm: add XFRM_MSG_MIGRATE_STATE for single SA migration
> xfrm: restrict netlink attributes for XFRM_MSG_MIGRATE_STATE
> xfrm: add documentation for XFRM_MSG_MIGRATE_STATE
Series appled, thanks a lot for your effort Antony!
^ permalink raw reply
* [PATCH net-next 1/2] tls: remove tls_toe and the related driver
From: Sabrina Dubroca @ 2026-06-09 14:21 UTC (permalink / raw)
To: netdev
Cc: Sabrina Dubroca, Ayush Sawal, John Fastabend, Jakub Kicinski,
David S. Miller, Alexander Gordeev, Andrew Lunn,
Christian Borntraeger, Eric Dumazet, Heiko Carstens, Paolo Abeni,
Simon Horman, Sven Schnelle, Vasily Gorbik, linux-s390, linux-doc,
Jonathan Corbet, Shuah Khan
In-Reply-To: <cover.1781013810.git.sd@queasysnail.net>
The tls_toe feature and its single user (chelsio chtls) have been
unmaintained for multiple years. It also hooks into the core of the
TCP implementation, and bypasses most of the networking stack.
Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
---
Documentation/networking/tls-offload.rst | 6 -
arch/s390/configs/debug_defconfig | 1 -
arch/s390/configs/defconfig | 1 -
.../ethernet/chelsio/inline_crypto/Kconfig | 12 -
.../ethernet/chelsio/inline_crypto/Makefile | 1 -
.../chelsio/inline_crypto/chtls/Makefile | 6 -
.../chelsio/inline_crypto/chtls/chtls.h | 584 -----
.../chelsio/inline_crypto/chtls/chtls_cm.c | 2336 -----------------
.../chelsio/inline_crypto/chtls/chtls_cm.h | 218 --
.../chelsio/inline_crypto/chtls/chtls_hw.c | 462 ----
.../chelsio/inline_crypto/chtls/chtls_io.c | 1836 -------------
.../chelsio/inline_crypto/chtls/chtls_main.c | 642 -----
include/net/tls.h | 1 -
include/net/tls_toe.h | 77 -
include/uapi/linux/tls.h | 2 +-
net/tls/Kconfig | 10 -
net/tls/Makefile | 1 -
net/tls/tls_main.c | 17 -
net/tls/tls_toe.c | 141 -
19 files changed, 1 insertion(+), 6353 deletions(-)
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
delete mode 100644 drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
delete mode 100644 include/net/tls_toe.h
delete mode 100644 net/tls/tls_toe.c
diff --git a/Documentation/networking/tls-offload.rst b/Documentation/networking/tls-offload.rst
index c173f537bf4d..9f08f8f06ca8 100644
--- a/Documentation/networking/tls-offload.rst
+++ b/Documentation/networking/tls-offload.rst
@@ -26,12 +26,6 @@ documentation in :ref:`Documentation/networking/tls.rst <kernel_tls>`.
This mode integrates best with the kernel stack and is described in detail
in the remaining part of this document
(``ethtool`` flags ``tls-hw-tx-offload`` and ``tls-hw-rx-offload``).
- * Full TCP NIC offload mode (``TLS_HW_RECORD``) - mode of operation where
- NIC driver and firmware replace the kernel networking stack
- with its own TCP handling, it is not usable in production environments
- making use of the Linux networking stack for example any firewalling
- abilities or QoS and packet scheduling (``ethtool`` flag ``tls-hw-record``).
-
The operation mode is selected automatically based on device configuration,
offload opt-in or opt-out on per-connection basis is not currently supported.
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 730c90b4a876..fa517117b275 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -125,7 +125,6 @@ CONFIG_UNIX=y
CONFIG_UNIX_DIAG=m
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
-CONFIG_TLS_TOE=y
CONFIG_XFRM_USER=m
CONFIG_NET_KEY=m
CONFIG_SMC=m
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index dd5fc1426c88..86c19649d6a4 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -116,7 +116,6 @@ CONFIG_UNIX=y
CONFIG_UNIX_DIAG=m
CONFIG_TLS=m
CONFIG_TLS_DEVICE=y
-CONFIG_TLS_TOE=y
CONFIG_XFRM_USER=m
CONFIG_NET_KEY=m
CONFIG_SMC=m
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig
index 521955e1f894..b7d452f4a7f1 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig
+++ b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig
@@ -13,18 +13,6 @@ config CHELSIO_INLINE_CRYPTO
if CHELSIO_INLINE_CRYPTO
-config CRYPTO_DEV_CHELSIO_TLS
- tristate "Chelsio Crypto Inline TLS Driver"
- depends on CHELSIO_T4
- depends on TLS
- depends on TLS_TOE
- help
- Support Chelsio Inline TLS with Chelsio crypto accelerator.
- Enable inline TLS support for Tx and Rx.
-
- To compile this driver as a module, choose M here: the module
- will be called chtls.
-
config CHELSIO_IPSEC_INLINE
tristate "Chelsio IPSec XFRM Tx crypto offload"
depends on CHELSIO_T4
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/Makefile
index 27e6d7e2f1eb..ca6548adc6a7 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/Makefile
+++ b/drivers/net/ethernet/chelsio/inline_crypto/Makefile
@@ -1,4 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/
obj-$(CONFIG_CHELSIO_IPSEC_INLINE) += ch_ipsec/
obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls/
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile
deleted file mode 100644
index bc11495acdb3..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 \
- -I $(srctree)/drivers/crypto/chelsio
-
-obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls.o
-chtls-objs := chtls_main.o chtls_cm.o chtls_io.o chtls_hw.o
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
deleted file mode 100644
index 1de5744a49b0..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h
+++ /dev/null
@@ -1,584 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- */
-
-#ifndef __CHTLS_H__
-#define __CHTLS_H__
-
-#include <crypto/aes.h>
-#include <crypto/hash.h>
-#include <crypto/sha1.h>
-#include <crypto/sha2.h>
-#include <crypto/authenc.h>
-#include <crypto/ctr.h>
-#include <crypto/gf128mul.h>
-#include <crypto/internal/aead.h>
-#include <crypto/null.h>
-#include <crypto/internal/skcipher.h>
-#include <crypto/aead.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/internal/hash.h>
-#include <linux/tls.h>
-#include <net/tls.h>
-#include <net/tls_prot.h>
-#include <net/tls_toe.h>
-
-#include "t4fw_api.h"
-#include "t4_msg.h"
-#include "cxgb4.h"
-#include "cxgb4_uld.h"
-#include "l2t.h"
-#include "chcr_algo.h"
-#include "chcr_core.h"
-#include "chcr_crypto.h"
-
-#define CHTLS_DRV_VERSION "1.0.0.0-ko"
-
-#define TLS_KEYCTX_RXFLIT_CNT_S 24
-#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
-
-#define TLS_KEYCTX_RXPROT_VER_S 20
-#define TLS_KEYCTX_RXPROT_VER_M 0xf
-#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
-
-#define TLS_KEYCTX_RXCIPH_MODE_S 16
-#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
-#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
-
-#define TLS_KEYCTX_RXAUTH_MODE_S 12
-#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
-#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
-
-#define TLS_KEYCTX_RXCIAU_CTRL_S 11
-#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
-
-#define TLS_KEYCTX_RX_SEQCTR_S 9
-#define TLS_KEYCTX_RX_SEQCTR_M 0x3
-#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
-
-#define TLS_KEYCTX_RX_VALID_S 8
-#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
-
-#define TLS_KEYCTX_RXCK_SIZE_S 3
-#define TLS_KEYCTX_RXCK_SIZE_M 0x7
-#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
-
-#define TLS_KEYCTX_RXMK_SIZE_S 0
-#define TLS_KEYCTX_RXMK_SIZE_M 0x7
-#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
-
-#define KEYCTX_TX_WR_IV_S 55
-#define KEYCTX_TX_WR_IV_M 0x1ffULL
-#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
-#define KEYCTX_TX_WR_IV_G(x) \
- (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
-
-#define KEYCTX_TX_WR_AAD_S 47
-#define KEYCTX_TX_WR_AAD_M 0xffULL
-#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
-#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
- KEYCTX_TX_WR_AAD_M)
-
-#define KEYCTX_TX_WR_AADST_S 39
-#define KEYCTX_TX_WR_AADST_M 0xffULL
-#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
-#define KEYCTX_TX_WR_AADST_G(x) \
- (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
-
-#define KEYCTX_TX_WR_CIPHER_S 30
-#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
-#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
-#define KEYCTX_TX_WR_CIPHER_G(x) \
- (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
-
-#define KEYCTX_TX_WR_CIPHERST_S 23
-#define KEYCTX_TX_WR_CIPHERST_M 0x7f
-#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
-#define KEYCTX_TX_WR_CIPHERST_G(x) \
- (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
-
-#define KEYCTX_TX_WR_AUTH_S 14
-#define KEYCTX_TX_WR_AUTH_M 0x1ff
-#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
-#define KEYCTX_TX_WR_AUTH_G(x) \
- (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
-
-#define KEYCTX_TX_WR_AUTHST_S 7
-#define KEYCTX_TX_WR_AUTHST_M 0x7f
-#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
-#define KEYCTX_TX_WR_AUTHST_G(x) \
- (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
-
-#define KEYCTX_TX_WR_AUTHIN_S 0
-#define KEYCTX_TX_WR_AUTHIN_M 0x7f
-#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
-#define KEYCTX_TX_WR_AUTHIN_G(x) \
- (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
-
-struct sge_opaque_hdr {
- void *dev;
- dma_addr_t addr[MAX_SKB_FRAGS + 1];
-};
-
-#define MAX_IVS_PAGE 256
-#define TLS_KEY_CONTEXT_SZ 64
-#define CIPHER_BLOCK_SIZE 16
-#define GCM_TAG_SIZE 16
-#define KEY_ON_MEM_SZ 16
-#define AEAD_EXPLICIT_DATA_SIZE 8
-#define TLS_HEADER_LENGTH 5
-#define SCMD_CIPH_MODE_AES_GCM 2
-/* Any MFS size should work and come from openssl */
-#define TLS_MFS 16384
-
-#define RSS_HDR sizeof(struct rss_header)
-#define TLS_WR_CPL_LEN \
- (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
-
-enum {
- CHTLS_KEY_CONTEXT_DSGL,
- CHTLS_KEY_CONTEXT_IMM,
- CHTLS_KEY_CONTEXT_DDR,
-};
-
-enum {
- CHTLS_LISTEN_START,
- CHTLS_LISTEN_STOP,
-};
-
-/* Flags for return value of CPL message handlers */
-enum {
- CPL_RET_BUF_DONE = 1, /* buffer processing done */
- CPL_RET_BAD_MSG = 2, /* bad CPL message */
- CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
-};
-
-#define LISTEN_INFO_HASH_SIZE 32
-#define RSPQ_HASH_BITS 5
-struct listen_info {
- struct listen_info *next; /* Link to next entry */
- struct sock *sk; /* The listening socket */
- unsigned int stid; /* The server TID */
-};
-
-enum {
- T4_LISTEN_START_PENDING,
- T4_LISTEN_STARTED
-};
-
-enum csk_flags {
- CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */
- CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */
- CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */
- CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */
- CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */
- CSK_ABORT_RPL_PENDING, /* expecting an abort reply */
- CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
- CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */
- CSK_TX_FAILOVER, /* Tx traffic failing over */
- CSK_UPDATE_RCV_WND, /* Need to update rcv window */
- CSK_RST_ABORTED, /* outgoing RST was aborted */
- CSK_TLS_HANDSHK, /* TLS Handshake */
- CSK_CONN_INLINE, /* Connection on HW */
-};
-
-enum chtls_cdev_state {
- CHTLS_CDEV_STATE_UP = 1
-};
-
-struct listen_ctx {
- struct sock *lsk;
- struct chtls_dev *cdev;
- struct sk_buff_head synq;
- u32 state;
-};
-
-struct key_map {
- unsigned long *addr;
- unsigned int start;
- unsigned int available;
- unsigned int size;
- spinlock_t lock; /* lock for key id request from map */
-} __packed;
-
-struct tls_scmd {
- u32 seqno_numivs;
- u32 ivgen_hdrlen;
-};
-
-struct chtls_dev {
- struct tls_toe_device tlsdev;
- struct list_head list;
- struct cxgb4_lld_info *lldi;
- struct pci_dev *pdev;
- struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
- spinlock_t listen_lock; /* lock for listen list */
- struct net_device **ports;
- struct tid_info *tids;
- unsigned int pfvf;
- const unsigned short *mtus;
-
- struct idr hwtid_idr;
- struct idr stid_idr;
-
- spinlock_t idr_lock ____cacheline_aligned_in_smp;
-
- struct net_device *egr_dev[NCHAN * 2];
- struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
- struct sk_buff *askb;
-
- struct sk_buff_head deferq;
- struct work_struct deferq_task;
-
- struct list_head list_node;
- struct list_head rcu_node;
- struct list_head na_node;
- unsigned int send_page_order;
- int max_host_sndbuf;
- u32 round_robin_cnt;
- struct key_map kmap;
- unsigned int cdev_state;
-};
-
-struct chtls_listen {
- struct chtls_dev *cdev;
- struct sock *sk;
-};
-
-struct chtls_hws {
- struct sk_buff_head sk_recv_queue;
- u8 txqid;
- u8 ofld;
- u16 type;
- u16 rstate;
- u16 keyrpl;
- u16 pldlen;
- u16 rcvpld;
- u16 compute;
- u16 expansion;
- u16 keylen;
- u16 pdus;
- u16 adjustlen;
- u16 ivsize;
- u16 txleft;
- u32 mfs;
- s32 txkey;
- s32 rxkey;
- u32 fcplenmax;
- u32 copied_seq;
- u64 tx_seq_no;
- struct tls_scmd scmd;
- union {
- struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
- struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
- } crypto_info;
-};
-
-struct chtls_sock {
- struct sock *sk;
- struct chtls_dev *cdev;
- struct l2t_entry *l2t_entry; /* pointer to the L2T entry */
- struct net_device *egress_dev; /* TX_CHAN for act open retry */
-
- struct sk_buff_head txq;
- struct sk_buff *wr_skb_head;
- struct sk_buff *wr_skb_tail;
- struct sk_buff *ctrl_skb_cache;
- struct sk_buff *txdata_skb_cache; /* abort path messages */
- struct kref kref;
- unsigned long flags;
- u32 opt2;
- u32 wr_credits;
- u32 wr_unacked;
- u32 wr_max_credits;
- u32 wr_nondata;
- u32 hwtid; /* TCP Control Block ID */
- u32 txq_idx;
- u32 rss_qid;
- u32 tid;
- u32 idr;
- u32 mss;
- u32 ulp_mode;
- u32 tx_chan;
- u32 rx_chan;
- u32 sndbuf;
- u32 txplen_max;
- u32 mtu_idx; /* MTU table index */
- u32 smac_idx;
- u8 port_id;
- u8 tos;
- u16 resv2;
- u32 delack_mode;
- u32 delack_seq;
- u32 snd_win;
- u32 rcv_win;
-
- void *passive_reap_next; /* placeholder for passive */
- struct chtls_hws tlshws;
- struct synq {
- struct sk_buff *next;
- struct sk_buff *prev;
- } synq;
- struct listen_ctx *listen_ctx;
-};
-
-struct tls_hdr {
- u8 type;
- u16 version;
- u16 length;
-} __packed;
-
-struct tlsrx_cmp_hdr {
- u8 type;
- u16 version;
- u16 length;
-
- u64 tls_seq;
- u16 reserved1;
- u8 res_to_mac_error;
-} __packed;
-
-/* res_to_mac_error fields */
-#define TLSRX_HDR_PKT_INT_ERROR_S 4
-#define TLSRX_HDR_PKT_INT_ERROR_M 0x1
-#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
- ((x) << TLSRX_HDR_PKT_INT_ERROR_S)
-#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
- (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
-#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_SPP_ERROR_S 3
-#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
-#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
-#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
- (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
-#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_CCDX_ERROR_S 2
-#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
-#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
-#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
- (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
-#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_PAD_ERROR_S 1
-#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
-#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
-#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
- (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
-#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_MAC_ERROR_S 0
-#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
-#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
-#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
- (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
-#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
-
-#define TLSRX_HDR_PKT_ERROR_M 0x1F
-#define CONTENT_TYPE_ERROR 0x7F
-
-struct ulp_mem_rw {
- __be32 cmd;
- __be32 len16; /* command length */
- __be32 dlen; /* data length in 32-byte units */
- __be32 lock_addr;
-};
-
-struct tls_key_wr {
- __be32 op_to_compl;
- __be32 flowid_len16;
- __be32 ftid;
- u8 reneg_to_write_rx;
- u8 protocol;
- __be16 mfs;
-};
-
-struct tls_key_req {
- struct tls_key_wr wr;
- struct ulp_mem_rw req;
- struct ulptx_idata sc_imm;
-};
-
-/*
- * This lives in skb->cb and is used to chain WRs in a linked list.
- */
-struct wr_skb_cb {
- struct l2t_skb_cb l2t; /* reserve space for l2t CB */
- struct sk_buff *next_wr; /* next write request */
-};
-
-/* Per-skb backlog handler. Run when a socket's backlog is processed. */
-struct blog_skb_cb {
- void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
- struct chtls_dev *cdev;
-};
-
-/*
- * Similar to tcp_skb_cb but with ULP elements added to support TLS,
- * etc.
- */
-struct ulp_skb_cb {
- struct wr_skb_cb wr; /* reserve space for write request */
- u16 flags; /* TCP-like flags */
- u8 psh;
- u8 ulp_mode; /* ULP mode/submode of sk_buff */
- u32 seq; /* TCP sequence number */
- union { /* ULP-specific fields */
- struct {
- u8 type;
- u8 ofld;
- u8 iv;
- } tls;
- } ulp;
-};
-
-#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
-#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
-
-/*
- * Flags for ulp_skb_cb.flags.
- */
-enum {
- ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */
- ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */
- ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */
- ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
- ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
- ULPCB_FLAG_URG = 1 << 5, /* urgent data */
- ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
- ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
-};
-
-/* The ULP mode/submode of an skbuff */
-#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
-#define TCP_PAGE(sk) (sk->sk_frag.page)
-#define TCP_OFF(sk) (sk->sk_frag.offset)
-
-static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
-{
- return container_of(tlsdev, struct chtls_dev, tlsdev);
-}
-
-static inline void csk_set_flag(struct chtls_sock *csk,
- enum csk_flags flag)
-{
- __set_bit(flag, &csk->flags);
-}
-
-static inline void csk_reset_flag(struct chtls_sock *csk,
- enum csk_flags flag)
-{
- __clear_bit(flag, &csk->flags);
-}
-
-static inline bool csk_conn_inline(const struct chtls_sock *csk)
-{
- return test_bit(CSK_CONN_INLINE, &csk->flags);
-}
-
-static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- if (!csk_conn_inline(csk))
- return 0;
- return test_bit(flag, &csk->flags);
-}
-
-static inline int csk_flag_nochk(const struct chtls_sock *csk,
- enum csk_flags flag)
-{
- return test_bit(flag, &csk->flags);
-}
-
-static inline void *cplhdr(struct sk_buff *skb)
-{
- return skb->data;
-}
-
-static inline int is_neg_adv(unsigned int status)
-{
- return status == CPL_ERR_RTX_NEG_ADVICE ||
- status == CPL_ERR_KEEPALV_NEG_ADVICE ||
- status == CPL_ERR_PERSIST_NEG_ADVICE;
-}
-
-static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
- struct sock *sk,
- struct sk_buff *skb)
-{
- skb_reset_mac_header(skb);
- skb_reset_network_header(skb);
- skb_reset_transport_header(skb);
-
- bh_lock_sock(sk);
- if (unlikely(sock_owned_by_user(sk))) {
- BLOG_SKB_CB(skb)->backlog_rcv = fn;
- __sk_add_backlog(sk, skb);
- } else {
- fn(sk, skb);
- }
- bh_unlock_sock(sk);
-}
-
-static inline void chtls_sock_free(struct kref *ref)
-{
- struct chtls_sock *csk = container_of(ref, struct chtls_sock,
- kref);
- kfree(csk);
-}
-
-static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
-{
- kref_put(&csk->kref, chtls_sock_free);
-}
-
-static inline void __chtls_sock_get(const char *fn,
- struct chtls_sock *csk)
-{
- kref_get(&csk->kref);
-}
-
-static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
- struct sk_buff *skb, int through_l2t)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- if (through_l2t) {
- /* send through L2T */
- cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
- } else {
- /* send directly */
- cxgb4_ofld_send(csk->egress_dev, skb);
- }
-}
-
-typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
-extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
-void chtls_install_cpl_ops(struct sock *sk);
-int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
-void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
-int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
-void chtls_close(struct sock *sk, long timeout);
-int chtls_disconnect(struct sock *sk, int flags);
-void chtls_shutdown(struct sock *sk, int how);
-void chtls_destroy_sock(struct sock *sk);
-int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
-int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
- size_t len, int flags);
-void chtls_splice_eof(struct socket *sock);
-int send_tx_flowc_wr(struct sock *sk, int compl,
- u32 snd_nxt, u32 rcv_nxt);
-void chtls_tcp_push(struct sock *sk, int flags);
-int chtls_push_frames(struct chtls_sock *csk, int comp);
-void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
- u64 mask, u64 val, u8 cookie,
- int through_l2t);
-int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
-void chtls_set_quiesce_ctrl(struct sock *sk, int val);
-void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
-unsigned int keyid_to_addr(int start_addr, int keyid);
-void free_tls_keyid(struct sock *sk);
-#endif
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
deleted file mode 100644
index 0e3e5cf52c2c..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+++ /dev/null
@@ -1,2336 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/sched/signal.h>
-#include <linux/kallsyms.h>
-#include <linux/kprobes.h>
-#include <linux/if_vlan.h>
-#include <linux/ipv6.h>
-#include <net/ipv6.h>
-#include <net/transp_v6.h>
-#include <net/ip6_route.h>
-#include <net/inet_common.h>
-#include <net/tcp.h>
-#include <net/dst.h>
-#include <net/tls.h>
-#include <net/addrconf.h>
-#include <net/secure_seq.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-#include "clip_tbl.h"
-#include "t4_tcb.h"
-
-/*
- * State transitions and actions for close. Note that if we are in SYN_SENT
- * we remain in that state as we cannot control a connection while it's in
- * SYN_SENT; such connections are allowed to establish and are then aborted.
- */
-static unsigned char new_state[16] = {
- /* current state: new state: action: */
- /* (Invalid) */ TCP_CLOSE,
- /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
- /* TCP_SYN_SENT */ TCP_SYN_SENT,
- /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
- /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
- /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
- /* TCP_TIME_WAIT */ TCP_CLOSE,
- /* TCP_CLOSE */ TCP_CLOSE,
- /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
- /* TCP_LAST_ACK */ TCP_LAST_ACK,
- /* TCP_LISTEN */ TCP_CLOSE,
- /* TCP_CLOSING */ TCP_CLOSING,
-};
-
-static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
-{
- struct chtls_sock *csk = kzalloc_obj(*csk, GFP_ATOMIC);
-
- if (!csk)
- return NULL;
-
- csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
- if (!csk->txdata_skb_cache) {
- kfree(csk);
- return NULL;
- }
-
- kref_init(&csk->kref);
- csk->cdev = cdev;
- skb_queue_head_init(&csk->txq);
- csk->wr_skb_head = NULL;
- csk->wr_skb_tail = NULL;
- csk->mss = MAX_MSS;
- csk->tlshws.ofld = 1;
- csk->tlshws.txkey = -1;
- csk->tlshws.rxkey = -1;
- csk->tlshws.mfs = TLS_MFS;
- skb_queue_head_init(&csk->tlshws.sk_recv_queue);
- return csk;
-}
-
-static void chtls_sock_release(struct kref *ref)
-{
- struct chtls_sock *csk =
- container_of(ref, struct chtls_sock, kref);
-
- kfree(csk);
-}
-
-static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
- struct sock *sk)
-{
- struct adapter *adap = pci_get_drvdata(cdev->pdev);
- struct net_device *ndev = cdev->ports[0];
-#if IS_ENABLED(CONFIG_IPV6)
- struct net_device *temp;
- int addr_type;
-#endif
- int i;
-
- switch (sk->sk_family) {
- case PF_INET:
- if (likely(!inet_sk(sk)->inet_rcv_saddr))
- return ndev;
- ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
- break;
-#if IS_ENABLED(CONFIG_IPV6)
- case PF_INET6:
- addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
- if (likely(addr_type == IPV6_ADDR_ANY))
- return ndev;
-
- for_each_netdev_rcu(&init_net, temp) {
- if (ipv6_chk_addr(&init_net, (struct in6_addr *)
- &sk->sk_v6_rcv_saddr, temp, 1)) {
- ndev = temp;
- break;
- }
- }
- break;
-#endif
- default:
- return NULL;
- }
-
- if (!ndev)
- return NULL;
-
- if (is_vlan_dev(ndev))
- ndev = vlan_dev_real_dev(ndev);
-
- for_each_port(adap, i)
- if (cdev->ports[i] == ndev)
- return ndev;
- return NULL;
-}
-
-static void assign_rxopt(struct sock *sk, unsigned int opt)
-{
- const struct chtls_dev *cdev;
- struct chtls_sock *csk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
- tp = tcp_sk(sk);
-
- cdev = csk->cdev;
- tp->tcp_header_len = sizeof(struct tcphdr);
- tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
- tp->mss_cache = tp->rx_opt.mss_clamp;
- tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
- tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt);
- tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
- SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
- if (!tp->rx_opt.wscale_ok)
- tp->rx_opt.rcv_wscale = 0;
- if (tp->rx_opt.tstamp_ok) {
- tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
- tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
- } else if (csk->opt2 & TSTAMPS_EN_F) {
- csk->opt2 &= ~TSTAMPS_EN_F;
- csk->mtu_idx = TCPOPT_MSS_G(opt);
- }
-}
-
-static void chtls_purge_receive_queue(struct sock *sk)
-{
- struct sk_buff *skb;
-
- while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
- skb_dstref_steal(skb);
- kfree_skb(skb);
- }
-}
-
-static void chtls_purge_write_queue(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct sk_buff *skb;
-
- while ((skb = __skb_dequeue(&csk->txq))) {
- sk->sk_wmem_queued -= skb->truesize;
- __kfree_skb(skb);
- }
-}
-
-static void chtls_purge_recv_queue(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_hws *tlsk = &csk->tlshws;
- struct sk_buff *skb;
-
- while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
- skb_dstref_steal(skb);
- kfree_skb(skb);
- }
-}
-
-static void abort_arp_failure(void *handle, struct sk_buff *skb)
-{
- struct cpl_abort_req *req = cplhdr(skb);
- struct chtls_dev *cdev;
-
- cdev = (struct chtls_dev *)handle;
- req->cmd = CPL_ABORT_NO_RST;
- cxgb4_ofld_send(cdev->lldi->ports[0], skb);
-}
-
-static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
-{
- if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
- __skb_trim(skb, 0);
- refcount_inc(&skb->users);
- } else {
- skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
- }
- return skb;
-}
-
-static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
-{
- struct cpl_abort_req *req;
- struct chtls_sock *csk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
- tp = tcp_sk(sk);
-
- if (!skb)
- skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
-
- req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
- INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
- skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
- req->rsvd0 = htonl(tp->snd_nxt);
- req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
- req->cmd = mode;
- t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
- send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
-}
-
-static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
- !csk->cdev)) {
- if (sk->sk_state == TCP_SYN_RECV)
- csk_set_flag(csk, CSK_RST_ABORTED);
- goto out;
- }
-
- if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
- struct tcp_sock *tp = tcp_sk(sk);
-
- if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
- WARN_ONCE(1, "send tx flowc error");
- csk_set_flag(csk, CSK_TX_DATA_SENT);
- }
-
- csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
- chtls_purge_write_queue(sk);
-
- csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
- if (sk->sk_state != TCP_SYN_RECV)
- chtls_send_abort(sk, mode, skb);
- else
- chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W,
- TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0,
- TCB_FIELD_COOKIE_TFLAG, 1);
-
- return;
-out:
- kfree_skb(skb);
-}
-
-static void release_tcp_port(struct sock *sk)
-{
- if (inet_csk(sk)->icsk_bind_hash)
- inet_put_port(sk);
-}
-
-static void tcp_uncork(struct sock *sk)
-{
- struct tcp_sock *tp = tcp_sk(sk);
-
- if (tp->nonagle & TCP_NAGLE_CORK) {
- tp->nonagle &= ~TCP_NAGLE_CORK;
- chtls_tcp_push(sk, 0);
- }
-}
-
-static void chtls_close_conn(struct sock *sk)
-{
- struct cpl_close_con_req *req;
- struct chtls_sock *csk;
- struct sk_buff *skb;
- unsigned int tid;
- unsigned int len;
-
- len = roundup(sizeof(struct cpl_close_con_req), 16);
- csk = rcu_dereference_sk_user_data(sk);
- tid = csk->tid;
-
- skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
- req = (struct cpl_close_con_req *)__skb_put(skb, len);
- memset(req, 0, len);
- req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
- FW_WR_IMMDLEN_V(sizeof(*req) -
- sizeof(req->wr)));
- req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
- FW_WR_FLOWID_V(tid));
-
- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
-
- tcp_uncork(sk);
- skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
- if (sk->sk_state != TCP_SYN_SENT)
- chtls_push_frames(csk, 1);
-}
-
-/*
- * Perform a state transition during close and return the actions indicated
- * for the transition. Do not make this function inline, the main reason
- * it exists at all is to avoid multiple inlining of tcp_set_state.
- */
-static int make_close_transition(struct sock *sk)
-{
- int next = (int)new_state[sk->sk_state];
-
- tcp_set_state(sk, next & TCP_STATE_MASK);
- return next & TCP_ACTION_FIN;
-}
-
-void chtls_close(struct sock *sk, long timeout)
-{
- int data_lost, prev_state;
- struct chtls_sock *csk;
-
- csk = rcu_dereference_sk_user_data(sk);
-
- lock_sock(sk);
- sk->sk_shutdown |= SHUTDOWN_MASK;
-
- data_lost = skb_queue_len(&sk->sk_receive_queue);
- data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
- chtls_purge_recv_queue(sk);
- chtls_purge_receive_queue(sk);
-
- if (sk->sk_state == TCP_CLOSE) {
- goto wait;
- } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
- chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
- release_tcp_port(sk);
- goto unlock;
- } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
- sk->sk_prot->disconnect(sk, 0);
- } else if (make_close_transition(sk)) {
- chtls_close_conn(sk);
- }
-wait:
- if (timeout)
- sk_stream_wait_close(sk, timeout);
-
-unlock:
- prev_state = sk->sk_state;
- sock_hold(sk);
- sock_orphan(sk);
-
- release_sock(sk);
-
- local_bh_disable();
- bh_lock_sock(sk);
-
- if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
- goto out;
-
- if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
- !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
- struct sk_buff *skb;
-
- skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
- if (skb)
- chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
- }
-
- if (sk->sk_state == TCP_CLOSE)
- inet_csk_destroy_sock(sk);
-
-out:
- bh_unlock_sock(sk);
- local_bh_enable();
- sock_put(sk);
-}
-
-/*
- * Wait until a socket enters on of the given states.
- */
-static int wait_for_states(struct sock *sk, unsigned int states)
-{
- DECLARE_WAITQUEUE(wait, current);
- struct socket_wq _sk_wq;
- long current_timeo;
- int err = 0;
-
- current_timeo = 200;
-
- /*
- * We want this to work even when there's no associated struct socket.
- * In that case we provide a temporary wait_queue_head_t.
- */
- if (!sk->sk_wq) {
- init_waitqueue_head(&_sk_wq.wait);
- _sk_wq.fasync_list = NULL;
- init_rcu_head_on_stack(&_sk_wq.rcu);
- RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
- }
-
- add_wait_queue(sk_sleep(sk), &wait);
- while (!sk_in_state(sk, states)) {
- if (!current_timeo) {
- err = -EBUSY;
- break;
- }
- if (signal_pending(current)) {
- err = sock_intr_errno(current_timeo);
- break;
- }
- set_current_state(TASK_UNINTERRUPTIBLE);
- release_sock(sk);
- if (!sk_in_state(sk, states))
- current_timeo = schedule_timeout(current_timeo);
- __set_current_state(TASK_RUNNING);
- lock_sock(sk);
- }
- remove_wait_queue(sk_sleep(sk), &wait);
-
- if (rcu_dereference(sk->sk_wq) == &_sk_wq)
- sk->sk_wq = NULL;
- return err;
-}
-
-int chtls_disconnect(struct sock *sk, int flags)
-{
- struct tcp_sock *tp;
- int err;
-
- tp = tcp_sk(sk);
- chtls_purge_recv_queue(sk);
- chtls_purge_receive_queue(sk);
- chtls_purge_write_queue(sk);
-
- if (sk->sk_state != TCP_CLOSE) {
- sk->sk_err = ECONNRESET;
- chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
- err = wait_for_states(sk, TCPF_CLOSE);
- if (err)
- return err;
- }
- chtls_purge_recv_queue(sk);
- chtls_purge_receive_queue(sk);
- tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
- return tcp_disconnect(sk, flags);
-}
-
-#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
- TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
-void chtls_shutdown(struct sock *sk, int how)
-{
- if ((how & SEND_SHUTDOWN) &&
- sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
- make_close_transition(sk))
- chtls_close_conn(sk);
-}
-
-void chtls_destroy_sock(struct sock *sk)
-{
- struct chtls_sock *csk;
-
- csk = rcu_dereference_sk_user_data(sk);
- chtls_purge_recv_queue(sk);
- csk->ulp_mode = ULP_MODE_NONE;
- chtls_purge_write_queue(sk);
- free_tls_keyid(sk);
- kref_put(&csk->kref, chtls_sock_release);
- if (sk->sk_family == AF_INET)
- sk->sk_prot = &tcp_prot;
-#if IS_ENABLED(CONFIG_IPV6)
- else
- sk->sk_prot = &tcpv6_prot;
-#endif
- sk->sk_prot->destroy(sk);
-}
-
-static void reset_listen_child(struct sock *child)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
- struct sk_buff *skb;
-
- skb = alloc_ctrl_skb(csk->txdata_skb_cache,
- sizeof(struct cpl_abort_req));
-
- chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
- sock_orphan(child);
- tcp_orphan_count_inc();
- if (child->sk_state == TCP_CLOSE)
- inet_csk_destroy_sock(child);
-}
-
-static void chtls_disconnect_acceptq(struct sock *listen_sk)
-{
- struct request_sock **pprev;
-
- pprev = ACCEPT_QUEUE(listen_sk);
- while (*pprev) {
- struct request_sock *req = *pprev;
-
- if (req->rsk_ops == &chtls_rsk_ops ||
- req->rsk_ops == &chtls_rsk_opsv6) {
- struct sock *child = req->sk;
-
- *pprev = req->dl_next;
- sk_acceptq_removed(listen_sk);
- reqsk_put(req);
- sock_hold(child);
- local_bh_disable();
- bh_lock_sock(child);
- release_tcp_port(child);
- reset_listen_child(child);
- bh_unlock_sock(child);
- local_bh_enable();
- sock_put(child);
- } else {
- pprev = &req->dl_next;
- }
- }
-}
-
-static int listen_hashfn(const struct sock *sk)
-{
- return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
-}
-
-static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
- struct sock *sk,
- unsigned int stid)
-{
- struct listen_info *p = kmalloc_obj(*p);
-
- if (p) {
- int key = listen_hashfn(sk);
-
- p->sk = sk;
- p->stid = stid;
- spin_lock(&cdev->listen_lock);
- p->next = cdev->listen_hash_tab[key];
- cdev->listen_hash_tab[key] = p;
- spin_unlock(&cdev->listen_lock);
- }
- return p;
-}
-
-static int listen_hash_find(struct chtls_dev *cdev,
- struct sock *sk)
-{
- struct listen_info *p;
- int stid = -1;
- int key;
-
- key = listen_hashfn(sk);
-
- spin_lock(&cdev->listen_lock);
- for (p = cdev->listen_hash_tab[key]; p; p = p->next)
- if (p->sk == sk) {
- stid = p->stid;
- break;
- }
- spin_unlock(&cdev->listen_lock);
- return stid;
-}
-
-static int listen_hash_del(struct chtls_dev *cdev,
- struct sock *sk)
-{
- struct listen_info *p, **prev;
- int stid = -1;
- int key;
-
- key = listen_hashfn(sk);
- prev = &cdev->listen_hash_tab[key];
-
- spin_lock(&cdev->listen_lock);
- for (p = *prev; p; prev = &p->next, p = p->next)
- if (p->sk == sk) {
- stid = p->stid;
- *prev = p->next;
- kfree(p);
- break;
- }
- spin_unlock(&cdev->listen_lock);
- return stid;
-}
-
-static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
-{
- struct request_sock *req;
- struct chtls_sock *csk;
-
- csk = rcu_dereference_sk_user_data(child);
- req = csk->passive_reap_next;
-
- reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
- __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
- chtls_reqsk_free(req);
- csk->passive_reap_next = NULL;
-}
-
-static void chtls_reset_synq(struct listen_ctx *listen_ctx)
-{
- struct sock *listen_sk = listen_ctx->lsk;
-
- while (!skb_queue_empty(&listen_ctx->synq)) {
- struct chtls_sock *csk =
- container_of((struct synq *)skb_peek
- (&listen_ctx->synq), struct chtls_sock, synq);
- struct sock *child = csk->sk;
-
- cleanup_syn_rcv_conn(child, listen_sk);
- sock_hold(child);
- local_bh_disable();
- bh_lock_sock(child);
- release_tcp_port(child);
- reset_listen_child(child);
- bh_unlock_sock(child);
- local_bh_enable();
- sock_put(child);
- }
-}
-
-int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
-{
- struct net_device *ndev;
-#if IS_ENABLED(CONFIG_IPV6)
- bool clip_valid = false;
-#endif
- struct listen_ctx *ctx;
- struct adapter *adap;
- struct port_info *pi;
- int ret = 0;
- int stid;
-
- rcu_read_lock();
- ndev = chtls_find_netdev(cdev, sk);
- rcu_read_unlock();
- if (!ndev)
- return -EBADF;
-
- pi = netdev_priv(ndev);
- adap = pi->adapter;
- if (!(adap->flags & CXGB4_FULL_INIT_DONE))
- return -EBADF;
-
- if (listen_hash_find(cdev, sk) >= 0) /* already have it */
- return -EADDRINUSE;
-
- ctx = kmalloc_obj(*ctx);
- if (!ctx)
- return -ENOMEM;
-
- __module_get(THIS_MODULE);
- ctx->lsk = sk;
- ctx->cdev = cdev;
- ctx->state = T4_LISTEN_START_PENDING;
- skb_queue_head_init(&ctx->synq);
-
- stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
- if (stid < 0)
- goto free_ctx;
-
- sock_hold(sk);
- if (!listen_hash_add(cdev, sk, stid))
- goto free_stid;
-
- if (sk->sk_family == PF_INET) {
- ret = cxgb4_create_server(ndev, stid,
- inet_sk(sk)->inet_rcv_saddr,
- inet_sk(sk)->inet_sport, 0,
- cdev->lldi->rxq_ids[0]);
-#if IS_ENABLED(CONFIG_IPV6)
- } else {
- int addr_type;
-
- addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
- if (addr_type != IPV6_ADDR_ANY) {
- ret = cxgb4_clip_get(ndev, (const u32 *)
- &sk->sk_v6_rcv_saddr, 1);
- if (ret)
- goto del_hash;
- clip_valid = true;
- }
- ret = cxgb4_create_server6(ndev, stid,
- &sk->sk_v6_rcv_saddr,
- inet_sk(sk)->inet_sport,
- cdev->lldi->rxq_ids[0]);
-#endif
- }
- if (ret > 0)
- ret = net_xmit_errno(ret);
- if (ret)
- goto del_hash;
- return 0;
-del_hash:
-#if IS_ENABLED(CONFIG_IPV6)
- if (clip_valid)
- cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
-#endif
- listen_hash_del(cdev, sk);
-free_stid:
- cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
- sock_put(sk);
-free_ctx:
- kfree(ctx);
- module_put(THIS_MODULE);
- return -EBADF;
-}
-
-void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
-{
- struct listen_ctx *listen_ctx;
- int stid;
-
- stid = listen_hash_del(cdev, sk);
- if (stid < 0)
- return;
-
- listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
- chtls_reset_synq(listen_ctx);
-
- cxgb4_remove_server(cdev->lldi->ports[0], stid,
- cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
-
-#if IS_ENABLED(CONFIG_IPV6)
- if (sk->sk_family == PF_INET6) {
- struct net_device *ndev = chtls_find_netdev(cdev, sk);
- int addr_type = 0;
-
- addr_type = ipv6_addr_type((const struct in6_addr *)
- &sk->sk_v6_rcv_saddr);
- if (addr_type != IPV6_ADDR_ANY)
- cxgb4_clip_release(ndev, (const u32 *)
- &sk->sk_v6_rcv_saddr, 1);
- }
-#endif
- chtls_disconnect_acceptq(sk);
-}
-
-static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
- unsigned int stid = GET_TID(rpl);
- struct listen_ctx *listen_ctx;
-
- listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
- if (!listen_ctx)
- return CPL_RET_BUF_DONE;
-
- if (listen_ctx->state == T4_LISTEN_START_PENDING) {
- listen_ctx->state = T4_LISTEN_STARTED;
- return CPL_RET_BUF_DONE;
- }
-
- if (rpl->status != CPL_ERR_NONE) {
- pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
- rpl->status, stid);
- } else {
- cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
- sock_put(listen_ctx->lsk);
- kfree(listen_ctx);
- module_put(THIS_MODULE);
- }
- return CPL_RET_BUF_DONE;
-}
-
-static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
- struct listen_ctx *listen_ctx;
- unsigned int stid;
- void *data;
-
- stid = GET_TID(rpl);
- data = lookup_stid(cdev->tids, stid);
- listen_ctx = (struct listen_ctx *)data;
-
- if (rpl->status != CPL_ERR_NONE) {
- pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
- rpl->status, stid);
- } else {
- cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
- sock_put(listen_ctx->lsk);
- kfree(listen_ctx);
- module_put(THIS_MODULE);
- }
- return CPL_RET_BUF_DONE;
-}
-
-static void chtls_purge_wr_queue(struct sock *sk)
-{
- struct sk_buff *skb;
-
- while ((skb = dequeue_wr(sk)) != NULL)
- kfree_skb(skb);
-}
-
-static void chtls_release_resources(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_dev *cdev = csk->cdev;
- unsigned int tid = csk->tid;
- struct tid_info *tids;
-
- if (!cdev)
- return;
-
- tids = cdev->tids;
- kfree_skb(csk->txdata_skb_cache);
- csk->txdata_skb_cache = NULL;
-
- if (csk->wr_credits != csk->wr_max_credits) {
- chtls_purge_wr_queue(sk);
- chtls_reset_wr_list(csk);
- }
-
- if (csk->l2t_entry) {
- cxgb4_l2t_release(csk->l2t_entry);
- csk->l2t_entry = NULL;
- }
-
- if (sk->sk_state != TCP_SYN_SENT) {
- cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
- sock_put(sk);
- }
-}
-
-static void chtls_conn_done(struct sock *sk)
-{
- if (sock_flag(sk, SOCK_DEAD))
- chtls_purge_receive_queue(sk);
- sk_wakeup_sleepers(sk, 0);
- tcp_done(sk);
-}
-
-static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
-{
- /*
- * If the server is still open we clean up the child connection,
- * otherwise the server already did the clean up as it was purging
- * its SYN queue and the skb was just sitting in its backlog.
- */
- if (likely(parent->sk_state == TCP_LISTEN)) {
- cleanup_syn_rcv_conn(child, parent);
- /* Without the below call to sock_orphan,
- * we leak the socket resource with syn_flood test
- * as inet_csk_destroy_sock will not be called
- * in tcp_done since SOCK_DEAD flag is not set.
- * Kernel handles this differently where new socket is
- * created only after 3 way handshake is done.
- */
- sock_orphan(child);
- tcp_orphan_count_inc();
- chtls_release_resources(child);
- chtls_conn_done(child);
- } else {
- if (csk_flag(child, CSK_RST_ABORTED)) {
- chtls_release_resources(child);
- chtls_conn_done(child);
- }
- }
-}
-
-static void pass_open_abort(struct sock *child, struct sock *parent,
- struct sk_buff *skb)
-{
- do_abort_syn_rcv(child, parent);
- kfree_skb(skb);
-}
-
-static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
-{
- pass_open_abort(skb->sk, lsk, skb);
-}
-
-static void chtls_pass_open_arp_failure(struct sock *sk,
- struct sk_buff *skb)
-{
- const struct request_sock *oreq;
- struct chtls_sock *csk;
- struct chtls_dev *cdev;
- struct sock *parent;
- void *data;
-
- csk = rcu_dereference_sk_user_data(sk);
- cdev = csk->cdev;
-
- /*
- * If the connection is being aborted due to the parent listening
- * socket going away there's nothing to do, the ABORT_REQ will close
- * the connection.
- */
- if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
- kfree_skb(skb);
- return;
- }
-
- oreq = csk->passive_reap_next;
- data = lookup_stid(cdev->tids, oreq->ts_recent);
- parent = ((struct listen_ctx *)data)->lsk;
-
- bh_lock_sock(parent);
- if (!sock_owned_by_user(parent)) {
- pass_open_abort(sk, parent, skb);
- } else {
- BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
- __sk_add_backlog(parent, skb);
- }
- bh_unlock_sock(parent);
-}
-
-static void chtls_accept_rpl_arp_failure(void *handle,
- struct sk_buff *skb)
-{
- struct sock *sk = (struct sock *)handle;
-
- sock_hold(sk);
- process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
- sock_put(sk);
-}
-
-static unsigned int chtls_select_mss(const struct chtls_sock *csk,
- unsigned int pmtu,
- struct cpl_pass_accept_req *req)
-{
- struct chtls_dev *cdev;
- struct dst_entry *dst;
- unsigned int tcpoptsz;
- unsigned int iphdrsz;
- unsigned int mtu_idx;
- struct tcp_sock *tp;
- unsigned int mss;
- struct sock *sk;
- u16 user_mss;
-
- mss = ntohs(req->tcpopt.mss);
- sk = csk->sk;
- dst = __sk_dst_get(sk);
- cdev = csk->cdev;
- tp = tcp_sk(sk);
- tcpoptsz = 0;
-
-#if IS_ENABLED(CONFIG_IPV6)
- if (sk->sk_family == AF_INET6)
- iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
- else
-#endif
- iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
- if (req->tcpopt.tstamp)
- tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
-
- tp->advmss = dst_metric_advmss(dst);
- user_mss = USER_MSS(tp);
- if (user_mss && tp->advmss > user_mss)
- tp->advmss = user_mss;
- if (tp->advmss > pmtu - iphdrsz)
- tp->advmss = pmtu - iphdrsz;
- if (mss && tp->advmss > mss)
- tp->advmss = mss;
-
- tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
- iphdrsz + tcpoptsz,
- tp->advmss - tcpoptsz,
- 8, &mtu_idx);
- tp->advmss -= iphdrsz;
-
- inet_csk(sk)->icsk_pmtu_cookie = pmtu;
- return mtu_idx;
-}
-
-static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
-{
- int wscale = 0;
-
- if (space > MAX_RCV_WND)
- space = MAX_RCV_WND;
- if (win_clamp && win_clamp < space)
- space = win_clamp;
-
- if (wscale_ok) {
- while (wscale < 14 && (65535 << wscale) < space)
- wscale++;
- }
- return wscale;
-}
-
-static void chtls_pass_accept_rpl(struct sk_buff *skb,
- struct cpl_pass_accept_req *req,
- unsigned int tid)
-
-{
- struct cpl_t5_pass_accept_rpl *rpl5;
- struct cxgb4_lld_info *lldi;
- const struct tcphdr *tcph;
- const struct tcp_sock *tp;
- struct chtls_sock *csk;
- unsigned int len;
- struct sock *sk;
- u32 opt2, hlen;
- u64 opt0;
-
- sk = skb->sk;
- tp = tcp_sk(sk);
- csk = sk->sk_user_data;
- csk->tid = tid;
- lldi = csk->cdev->lldi;
- len = roundup(sizeof(*rpl5), 16);
-
- rpl5 = __skb_put_zero(skb, len);
- INIT_TP_WR(rpl5, tid);
-
- OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
- csk->tid));
- csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
- req);
- opt0 = TCAM_BYPASS_F |
- WND_SCALE_V(RCV_WSCALE(tp)) |
- MSS_IDX_V(csk->mtu_idx) |
- L2T_IDX_V(csk->l2t_entry->idx) |
- NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
- TX_CHAN_V(csk->tx_chan) |
- SMAC_SEL_V(csk->smac_idx) |
- DSCP_V(csk->tos >> 2) |
- ULP_MODE_V(ULP_MODE_TLS) |
- RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
-
- opt2 = RX_CHANNEL_V(0) |
- RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
-
- if (!is_t5(lldi->adapter_type))
- opt2 |= RX_FC_DISABLE_F;
- if (req->tcpopt.tstamp)
- opt2 |= TSTAMPS_EN_F;
- if (req->tcpopt.sack)
- opt2 |= SACK_EN_F;
- hlen = ntohl(req->hdr_len);
-
- tcph = (struct tcphdr *)((u8 *)(req + 1) +
- T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
- if (tcph->ece && tcph->cwr)
- opt2 |= CCTRL_ECN_V(1);
- opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
- opt2 |= T5_ISS_F;
- opt2 |= T5_OPT_2_VALID_F;
- opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
- rpl5->opt0 = cpu_to_be64(opt0);
- rpl5->opt2 = cpu_to_be32(opt2);
- rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
- set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
- t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
- cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
-}
-
-static void inet_inherit_port(struct sock *lsk, struct sock *newsk)
-{
- local_bh_disable();
- __inet_inherit_port(lsk, newsk);
- local_bh_enable();
-}
-
-static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
-{
- if (skb->protocol) {
- kfree_skb(skb);
- return 0;
- }
- BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
- return 0;
-}
-
-static void chtls_set_tcp_window(struct chtls_sock *csk)
-{
- struct net_device *ndev = csk->egress_dev;
- struct port_info *pi = netdev_priv(ndev);
- unsigned int linkspeed;
- u8 scale;
-
- linkspeed = pi->link_cfg.speed;
- scale = linkspeed / SPEED_10000;
-#define CHTLS_10G_RCVWIN (256 * 1024)
- csk->rcv_win = CHTLS_10G_RCVWIN;
- if (scale)
- csk->rcv_win *= scale;
-#define CHTLS_10G_SNDWIN (256 * 1024)
- csk->snd_win = CHTLS_10G_SNDWIN;
- if (scale)
- csk->snd_win *= scale;
-}
-
-static struct sock *chtls_recv_sock(struct sock *lsk,
- struct request_sock *oreq,
- void *network_hdr,
- const struct cpl_pass_accept_req *req,
- struct chtls_dev *cdev)
-{
- struct adapter *adap = pci_get_drvdata(cdev->pdev);
- struct neighbour *n = NULL;
- struct inet_sock *newinet;
- const struct iphdr *iph;
- struct tls_context *ctx;
- struct net_device *ndev;
- struct chtls_sock *csk;
- struct dst_entry *dst;
- struct tcp_sock *tp;
- struct sock *newsk;
- bool found = false;
- u16 port_id;
- int rxq_idx;
- int step, i;
-
- iph = (const struct iphdr *)network_hdr;
- newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
- if (!newsk)
- goto free_oreq;
-
- if (lsk->sk_family == AF_INET) {
- dst = inet_csk_route_child_sock(lsk, newsk, oreq);
- if (!dst)
- goto free_sk;
-
- n = dst_neigh_lookup(dst, &iph->saddr);
-#if IS_ENABLED(CONFIG_IPV6)
- } else {
- const struct ipv6hdr *ip6h;
- struct flowi6 fl6;
-
- ip6h = (const struct ipv6hdr *)network_hdr;
- memset(&fl6, 0, sizeof(fl6));
- fl6.flowi6_proto = IPPROTO_TCP;
- fl6.saddr = ip6h->daddr;
- fl6.daddr = ip6h->saddr;
- fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
- fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
- security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6));
- dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
- if (IS_ERR(dst))
- goto free_sk;
- n = dst_neigh_lookup(dst, &ip6h->saddr);
-#endif
- }
- if (!n || !n->dev)
- goto free_dst;
-
- ndev = n->dev;
- if (is_vlan_dev(ndev))
- ndev = vlan_dev_real_dev(ndev);
-
- for_each_port(adap, i)
- if (cdev->ports[i] == ndev)
- found = true;
-
- if (!found)
- goto free_dst;
-
- port_id = cxgb4_port_idx(ndev);
-
- csk = chtls_sock_create(cdev);
- if (!csk)
- goto free_dst;
-
- csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
- if (!csk->l2t_entry)
- goto free_csk;
-
- newsk->sk_user_data = csk;
- newsk->sk_backlog_rcv = chtls_backlog_rcv;
-
- tp = tcp_sk(newsk);
- newinet = inet_sk(newsk);
-
- if (iph->version == 0x4) {
- newinet->inet_daddr = iph->saddr;
- newinet->inet_rcv_saddr = iph->daddr;
- newinet->inet_saddr = iph->daddr;
-#if IS_ENABLED(CONFIG_IPV6)
- } else {
- struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
- struct inet_request_sock *treq = inet_rsk(oreq);
- struct ipv6_pinfo *newnp = inet6_sk(newsk);
- struct ipv6_pinfo *np = inet6_sk(lsk);
-
- newinet->pinet6 = &newtcp6sk->inet6;
- newinet->ipv6_fl_list = NULL;
- memcpy(newnp, np, sizeof(struct ipv6_pinfo));
- newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
- newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
- inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
- newnp->pktoptions = NULL;
- newsk->sk_bound_dev_if = treq->ir_iif;
- newinet->inet_opt = NULL;
- newinet->inet_daddr = LOOPBACK4_IPV6;
- newinet->inet_saddr = LOOPBACK4_IPV6;
-#endif
- }
-
- oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
- sk_setup_caps(newsk, dst);
- ctx = tls_get_ctx(lsk);
- newsk->sk_destruct = ctx->sk_destruct;
- newsk->sk_prot_creator = lsk->sk_prot_creator;
- csk->sk = newsk;
- csk->passive_reap_next = oreq;
- csk->tx_chan = cxgb4_port_chan(ndev);
- csk->port_id = port_id;
- csk->egress_dev = ndev;
- csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
- chtls_set_tcp_window(csk);
- tp->rcv_wnd = csk->rcv_win;
- csk->sndbuf = csk->snd_win;
- csk->ulp_mode = ULP_MODE_TLS;
- step = cdev->lldi->nrxq / cdev->lldi->nchan;
- rxq_idx = port_id * step;
- rxq_idx += cdev->round_robin_cnt++ % step;
- csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx];
- csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
- port_id * step;
- csk->sndbuf = newsk->sk_sndbuf;
- csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
- RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
- READ_ONCE(sock_net(newsk)->
- ipv4.sysctl_tcp_window_scaling),
- tp->window_clamp);
- neigh_release(n);
- inet_inherit_port(lsk, newsk);
- csk_set_flag(csk, CSK_CONN_INLINE);
- bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
-
- return newsk;
-free_csk:
- chtls_sock_release(&csk->kref);
-free_dst:
- if (n)
- neigh_release(n);
- dst_release(dst);
-free_sk:
- inet_csk_prepare_forced_close(newsk);
- tcp_done(newsk);
-free_oreq:
- chtls_reqsk_free(oreq);
- return NULL;
-}
-
-/*
- * Populate a TID_RELEASE WR. The skb must be already propely sized.
- */
-static void mk_tid_release(struct sk_buff *skb,
- unsigned int chan, unsigned int tid)
-{
- struct cpl_tid_release *req;
- unsigned int len;
-
- len = roundup(sizeof(struct cpl_tid_release), 16);
- req = (struct cpl_tid_release *)__skb_put(skb, len);
- memset(req, 0, len);
- set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
- INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
-}
-
-static int chtls_get_module(struct sock *sk)
-{
- struct inet_connection_sock *icsk = inet_csk(sk);
-
- if (!try_module_get(icsk->icsk_ulp_ops->owner))
- return -1;
-
- return 0;
-}
-
-static void chtls_pass_accept_request(struct sock *sk,
- struct sk_buff *skb)
-{
- struct cpl_t5_pass_accept_rpl *rpl;
- struct cpl_pass_accept_req *req;
- struct listen_ctx *listen_ctx;
- struct vlan_ethhdr *vlan_eh;
- struct request_sock *oreq;
- struct sk_buff *reply_skb;
- struct chtls_sock *csk;
- struct chtls_dev *cdev;
- struct ipv6hdr *ip6h;
- struct tcphdr *tcph;
- struct sock *newsk;
- struct ethhdr *eh;
- struct iphdr *iph;
- void *network_hdr;
- unsigned int stid;
- unsigned int len;
- unsigned int tid;
- bool th_ecn, ect;
- __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
- u16 eth_hdr_len;
- bool ecn_ok;
-
- req = cplhdr(skb) + RSS_HDR;
- tid = GET_TID(req);
- cdev = BLOG_SKB_CB(skb)->cdev;
- newsk = lookup_tid(cdev->tids, tid);
- stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
- if (newsk) {
- pr_info("tid (%d) already in use\n", tid);
- return;
- }
-
- len = roundup(sizeof(*rpl), 16);
- reply_skb = alloc_skb(len, GFP_ATOMIC);
- if (!reply_skb) {
- cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
- kfree_skb(skb);
- return;
- }
-
- if (sk->sk_state != TCP_LISTEN)
- goto reject;
-
- if (inet_csk_reqsk_queue_is_full(sk))
- goto reject;
-
- if (sk_acceptq_is_full(sk))
- goto reject;
-
-
- eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
- if (eth_hdr_len == ETH_HLEN) {
- eh = (struct ethhdr *)(req + 1);
- iph = (struct iphdr *)(eh + 1);
- ip6h = (struct ipv6hdr *)(eh + 1);
- network_hdr = (void *)(eh + 1);
- } else {
- vlan_eh = (struct vlan_ethhdr *)(req + 1);
- iph = (struct iphdr *)(vlan_eh + 1);
- ip6h = (struct ipv6hdr *)(vlan_eh + 1);
- network_hdr = (void *)(vlan_eh + 1);
- }
-
- if (iph->version == 0x4) {
- tcph = (struct tcphdr *)(iph + 1);
- skb_set_network_header(skb, (void *)iph - (void *)req);
- oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
- } else {
- tcph = (struct tcphdr *)(ip6h + 1);
- skb_set_network_header(skb, (void *)ip6h - (void *)req);
- oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
- }
-
- if (!oreq)
- goto reject;
-
- oreq->rsk_rcv_wnd = 0;
- oreq->rsk_window_clamp = 0;
- oreq->syncookie = 0;
- oreq->mss = 0;
- oreq->ts_recent = 0;
-
- tcp_rsk(oreq)->tfo_listener = false;
- tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
- chtls_set_req_port(oreq, tcph->source, tcph->dest);
- if (iph->version == 0x4) {
- chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
- ip_dsfield = ipv4_get_dsfield(iph);
-#if IS_ENABLED(CONFIG_IPV6)
- } else {
- inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
- inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
- ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
-#endif
- }
- if (req->tcpopt.wsf <= 14 &&
- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
- inet_rsk(oreq)->wscale_ok = 1;
- inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
- }
- inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
- th_ecn = tcph->ece && tcph->cwr;
- if (th_ecn) {
- ect = !INET_ECN_is_not_ect(ip_dsfield);
- ecn_ok = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn);
- if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
- inet_rsk(oreq)->ecn_ok = 1;
- }
-
- newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
- if (!newsk)
- goto reject;
-
- if (chtls_get_module(newsk))
- goto reject;
- inet_csk_reqsk_queue_added(sk);
- reply_skb->sk = newsk;
- chtls_install_cpl_ops(newsk);
- cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
- csk = rcu_dereference_sk_user_data(newsk);
- listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
- csk->listen_ctx = listen_ctx;
- __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
- chtls_pass_accept_rpl(reply_skb, req, tid);
- kfree_skb(skb);
- return;
-
-reject:
- mk_tid_release(reply_skb, 0, tid);
- cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
- kfree_skb(skb);
-}
-
-/*
- * Handle a CPL_PASS_ACCEPT_REQ message.
- */
-static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
- struct listen_ctx *ctx;
- unsigned int stid;
- unsigned int tid;
- struct sock *lsk;
- void *data;
-
- stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
- tid = GET_TID(req);
-
- data = lookup_stid(cdev->tids, stid);
- if (!data)
- return 1;
-
- ctx = (struct listen_ctx *)data;
- lsk = ctx->lsk;
-
- if (unlikely(tid_out_of_range(cdev->tids, tid))) {
- pr_info("passive open TID %u too large\n", tid);
- return 1;
- }
-
- BLOG_SKB_CB(skb)->cdev = cdev;
- process_cpl_msg(chtls_pass_accept_request, lsk, skb);
- return 0;
-}
-
-/*
- * Completes some final bits of initialization for just established connections
- * and changes their state to TCP_ESTABLISHED.
- *
- * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
- */
-static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
-{
- struct tcp_sock *tp = tcp_sk(sk);
-
- tp->pushed_seq = snd_isn;
- tp->write_seq = snd_isn;
- tp->snd_nxt = snd_isn;
- tp->snd_una = snd_isn;
- atomic_set(&inet_sk(sk)->inet_id, get_random_u16());
- assign_rxopt(sk, opt);
-
- if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
- tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
-
- smp_mb();
- tcp_set_state(sk, TCP_ESTABLISHED);
-}
-
-static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
-{
- struct sk_buff *abort_skb;
-
- abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
- if (abort_skb)
- chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
-}
-
-static struct sock *reap_list;
-static DEFINE_SPINLOCK(reap_list_lock);
-
-/*
- * Process the reap list.
- */
-DECLARE_TASK_FUNC(process_reap_list, task_param)
-{
- spin_lock_bh(&reap_list_lock);
- while (reap_list) {
- struct sock *sk = reap_list;
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- reap_list = csk->passive_reap_next;
- csk->passive_reap_next = NULL;
- spin_unlock(&reap_list_lock);
- sock_hold(sk);
-
- bh_lock_sock(sk);
- chtls_abort_conn(sk, NULL);
- sock_orphan(sk);
- if (sk->sk_state == TCP_CLOSE)
- inet_csk_destroy_sock(sk);
- bh_unlock_sock(sk);
- sock_put(sk);
- spin_lock(&reap_list_lock);
- }
- spin_unlock_bh(&reap_list_lock);
-}
-
-static DECLARE_WORK(reap_task, process_reap_list);
-
-static void add_to_reap_list(struct sock *sk)
-{
- struct chtls_sock *csk = sk->sk_user_data;
-
- local_bh_disable();
- release_tcp_port(sk); /* release the port immediately */
-
- spin_lock(&reap_list_lock);
- csk->passive_reap_next = reap_list;
- reap_list = sk;
- if (!csk->passive_reap_next)
- schedule_work(&reap_task);
- spin_unlock(&reap_list_lock);
- local_bh_enable();
-}
-
-static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
- struct chtls_dev *cdev)
-{
- struct request_sock *oreq;
- struct chtls_sock *csk;
-
- if (lsk->sk_state != TCP_LISTEN)
- return;
-
- csk = child->sk_user_data;
- oreq = csk->passive_reap_next;
- csk->passive_reap_next = NULL;
-
- reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
- __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
-
- if (sk_acceptq_is_full(lsk)) {
- chtls_reqsk_free(oreq);
- add_to_reap_list(child);
- } else {
- refcount_set(&oreq->rsk_refcnt, 1);
- inet_csk_reqsk_queue_add(lsk, oreq, child);
- lsk->sk_data_ready(lsk);
- }
-}
-
-static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
-{
- struct sock *child = skb->sk;
-
- skb->sk = NULL;
- add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
- kfree_skb(skb);
-}
-
-static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk;
- struct sock *lsk, *sk;
- unsigned int hwtid;
-
- hwtid = GET_TID(req);
- sk = lookup_tid(cdev->tids, hwtid);
- if (!sk)
- return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
-
- bh_lock_sock(sk);
- if (unlikely(sock_owned_by_user(sk))) {
- kfree_skb(skb);
- } else {
- unsigned int stid;
- void *data;
-
- csk = sk->sk_user_data;
- csk->wr_max_credits = 64;
- csk->wr_credits = 64;
- csk->wr_unacked = 0;
- make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
- stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
- sk->sk_state_change(sk);
- if (unlikely(sk->sk_socket))
- sk_wake_async(sk, 0, POLL_OUT);
-
- data = lookup_stid(cdev->tids, stid);
- if (!data) {
- /* listening server close */
- kfree_skb(skb);
- goto unlock;
- }
- lsk = ((struct listen_ctx *)data)->lsk;
-
- bh_lock_sock(lsk);
- if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
- /* removed from synq */
- bh_unlock_sock(lsk);
- kfree_skb(skb);
- goto unlock;
- }
-
- if (likely(!sock_owned_by_user(lsk))) {
- kfree_skb(skb);
- add_pass_open_to_parent(sk, lsk, cdev);
- } else {
- skb->sk = sk;
- BLOG_SKB_CB(skb)->cdev = cdev;
- BLOG_SKB_CB(skb)->backlog_rcv =
- bl_add_pass_open_to_parent;
- __sk_add_backlog(lsk, skb);
- }
- bh_unlock_sock(lsk);
- }
-unlock:
- bh_unlock_sock(sk);
- return 0;
-}
-
-/*
- * Handle receipt of an urgent pointer.
- */
-static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
-{
- struct tcp_sock *tp = tcp_sk(sk);
-
- urg_seq--;
- if (tp->urg_data && !after(urg_seq, tp->urg_seq))
- return; /* duplicate pointer */
-
- sk_send_sigurg(sk);
- if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
- !sock_flag(sk, SOCK_URGINLINE) &&
- tp->copied_seq != tp->rcv_nxt) {
- struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
-
- tp->copied_seq++;
- if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
- chtls_free_skb(sk, skb);
- }
-
- tp->urg_data = TCP_URG_NOTYET;
- tp->urg_seq = urg_seq;
-}
-
-static void check_sk_callbacks(struct chtls_sock *csk)
-{
- struct sock *sk = csk->sk;
-
- if (unlikely(sk->sk_user_data &&
- !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
- csk_set_flag(csk, CSK_CALLBACKS_CHKD);
-}
-
-/*
- * Handles Rx data that arrives in a state where the socket isn't accepting
- * new data.
- */
-static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
-{
- if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
- chtls_abort_conn(sk, skb);
-
- kfree_skb(skb);
-}
-
-static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
-{
- struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
- tp = tcp_sk(sk);
-
- if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
- handle_excess_rx(sk, skb);
- return;
- }
-
- ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
- ULP_SKB_CB(skb)->psh = hdr->psh;
- skb_ulp_mode(skb) = ULP_MODE_NONE;
-
- skb_reset_transport_header(skb);
- __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
- if (!skb->data_len)
- __skb_trim(skb, ntohs(hdr->len));
-
- if (unlikely(hdr->urg))
- handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
- if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
- tp->urg_seq - tp->rcv_nxt < skb->len))
- tp->urg_data = TCP_URG_VALID |
- skb->data[tp->urg_seq - tp->rcv_nxt];
-
- if (unlikely(hdr->dack_mode != csk->delack_mode)) {
- csk->delack_mode = hdr->dack_mode;
- csk->delack_seq = tp->rcv_nxt;
- }
-
- tcp_hdr(skb)->fin = 0;
- tp->rcv_nxt += skb->len;
-
- __skb_queue_tail(&sk->sk_receive_queue, skb);
-
- if (!sock_flag(sk, SOCK_DEAD)) {
- check_sk_callbacks(csk);
- sk->sk_data_ready(sk);
- }
-}
-
-static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
- unsigned int hwtid = GET_TID(req);
- struct sock *sk;
-
- sk = lookup_tid(cdev->tids, hwtid);
- if (unlikely(!sk)) {
- pr_err("can't find conn. for hwtid %u.\n", hwtid);
- return -EINVAL;
- }
- skb_dstref_steal(skb);
- process_cpl_msg(chtls_recv_data, sk, skb);
- return 0;
-}
-
-static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
-{
- struct cpl_tls_data *hdr = cplhdr(skb);
- struct chtls_sock *csk;
- struct chtls_hws *tlsk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
- tlsk = &csk->tlshws;
- tp = tcp_sk(sk);
-
- if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
- handle_excess_rx(sk, skb);
- return;
- }
-
- ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
- ULP_SKB_CB(skb)->flags = 0;
- skb_ulp_mode(skb) = ULP_MODE_TLS;
-
- skb_reset_transport_header(skb);
- __skb_pull(skb, sizeof(*hdr));
- if (!skb->data_len)
- __skb_trim(skb,
- CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
-
- if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
- tp->rcv_nxt < skb->len))
- tp->urg_data = TCP_URG_VALID |
- skb->data[tp->urg_seq - tp->rcv_nxt];
-
- tcp_hdr(skb)->fin = 0;
- tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
- __skb_queue_tail(&tlsk->sk_recv_queue, skb);
-}
-
-static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_tls_data *req = cplhdr(skb);
- unsigned int hwtid = GET_TID(req);
- struct sock *sk;
-
- sk = lookup_tid(cdev->tids, hwtid);
- if (unlikely(!sk)) {
- pr_err("can't find conn. for hwtid %u.\n", hwtid);
- return -EINVAL;
- }
- skb_dstref_steal(skb);
- process_cpl_msg(chtls_recv_pdu, sk, skb);
- return 0;
-}
-
-static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
-{
- struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
-
- skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
- tls_cmp_hdr->length = ntohs((__force __be16)nlen);
-}
-
-static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
-{
- struct tlsrx_cmp_hdr *tls_hdr_pkt;
- struct cpl_rx_tls_cmp *cmp_cpl;
- struct sk_buff *skb_rec;
- struct chtls_sock *csk;
- struct chtls_hws *tlsk;
- struct tcp_sock *tp;
-
- cmp_cpl = cplhdr(skb);
- csk = rcu_dereference_sk_user_data(sk);
- tlsk = &csk->tlshws;
- tp = tcp_sk(sk);
-
- ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
- ULP_SKB_CB(skb)->flags = 0;
-
- skb_reset_transport_header(skb);
- __skb_pull(skb, sizeof(*cmp_cpl));
- tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
- if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
- tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
- if (!skb->data_len)
- __skb_trim(skb, TLS_HEADER_LENGTH);
-
- tp->rcv_nxt +=
- CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
-
- ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
- skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
- if (!skb_rec) {
- __skb_queue_tail(&sk->sk_receive_queue, skb);
- } else {
- chtls_set_hdrlen(skb, tlsk->pldlen);
- tlsk->pldlen = 0;
- __skb_queue_tail(&sk->sk_receive_queue, skb);
- __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
- }
-
- if (!sock_flag(sk, SOCK_DEAD)) {
- check_sk_callbacks(csk);
- sk->sk_data_ready(sk);
- }
-}
-
-static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_rx_tls_cmp *req = cplhdr(skb);
- unsigned int hwtid = GET_TID(req);
- struct sock *sk;
-
- sk = lookup_tid(cdev->tids, hwtid);
- if (unlikely(!sk)) {
- pr_err("can't find conn. for hwtid %u.\n", hwtid);
- return -EINVAL;
- }
- skb_dstref_steal(skb);
- process_cpl_msg(chtls_rx_hdr, sk, skb);
-
- return 0;
-}
-
-static void chtls_timewait(struct sock *sk)
-{
- struct tcp_sock *tp = tcp_sk(sk);
-
- tp->rcv_nxt++;
- tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
- tp->srtt_us = 0;
- tcp_time_wait(sk, TCP_TIME_WAIT, 0);
-}
-
-static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
- goto out;
-
- sk->sk_shutdown |= RCV_SHUTDOWN;
- sock_set_flag(sk, SOCK_DONE);
-
- switch (sk->sk_state) {
- case TCP_SYN_RECV:
- case TCP_ESTABLISHED:
- tcp_set_state(sk, TCP_CLOSE_WAIT);
- break;
- case TCP_FIN_WAIT1:
- tcp_set_state(sk, TCP_CLOSING);
- break;
- case TCP_FIN_WAIT2:
- chtls_release_resources(sk);
- if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
- chtls_conn_done(sk);
- else
- chtls_timewait(sk);
- break;
- default:
- pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
- }
-
- if (!sock_flag(sk, SOCK_DEAD)) {
- sk->sk_state_change(sk);
- /* Do not send POLL_HUP for half duplex close. */
-
- if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
- sk->sk_state == TCP_CLOSE)
- sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
- else
- sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
- }
-out:
- kfree_skb(skb);
-}
-
-static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
-{
- struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
-
- if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
- goto out;
-
- tp = tcp_sk(sk);
-
- tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
-
- switch (sk->sk_state) {
- case TCP_CLOSING:
- chtls_release_resources(sk);
- if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
- chtls_conn_done(sk);
- else
- chtls_timewait(sk);
- break;
- case TCP_LAST_ACK:
- chtls_release_resources(sk);
- chtls_conn_done(sk);
- break;
- case TCP_FIN_WAIT1:
- tcp_set_state(sk, TCP_FIN_WAIT2);
- sk->sk_shutdown |= SEND_SHUTDOWN;
-
- if (!sock_flag(sk, SOCK_DEAD))
- sk->sk_state_change(sk);
- else if (tcp_sk(sk)->linger2 < 0 &&
- !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
- chtls_abort_conn(sk, skb);
- else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT))
- chtls_set_quiesce_ctrl(sk, 0);
- break;
- default:
- pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
- }
-out:
- kfree_skb(skb);
-}
-
-static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
- size_t len, gfp_t gfp)
-{
- if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
- WARN_ONCE(skb->len < len, "skb alloc error");
- __skb_trim(skb, len);
- skb_get(skb);
- } else {
- skb = alloc_skb(len, gfp);
- if (skb)
- __skb_put(skb, len);
- }
- return skb;
-}
-
-static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
- int cmd)
-{
- struct cpl_abort_rpl *rpl = cplhdr(skb);
-
- INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
- rpl->cmd = cmd;
-}
-
-static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_abort_req_rss *req = cplhdr(skb);
- struct sk_buff *reply_skb;
-
- reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
- GFP_KERNEL | __GFP_NOFAIL);
- __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
- set_abort_rpl_wr(reply_skb, GET_TID(req),
- (req->status & CPL_ABORT_NO_RST));
- set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
- cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
- kfree_skb(skb);
-}
-
-/*
- * Add an skb to the deferred skb queue for processing from process context.
- */
-static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
- defer_handler_t handler)
-{
- DEFERRED_SKB_CB(skb)->handler = handler;
- spin_lock_bh(&cdev->deferq.lock);
- __skb_queue_tail(&cdev->deferq, skb);
- if (skb_queue_len(&cdev->deferq) == 1)
- schedule_work(&cdev->deferq_task);
- spin_unlock_bh(&cdev->deferq.lock);
-}
-
-static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
- struct chtls_dev *cdev,
- int status, int queue)
-{
- struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
- struct sk_buff *reply_skb;
- struct chtls_sock *csk;
- unsigned int tid;
-
- csk = rcu_dereference_sk_user_data(sk);
- tid = GET_TID(req);
-
- reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
- if (!reply_skb) {
- req->status = (queue << 1) | status;
- t4_defer_reply(skb, cdev, send_defer_abort_rpl);
- return;
- }
-
- set_abort_rpl_wr(reply_skb, tid, status);
- kfree_skb(skb);
- set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
- if (csk_conn_inline(csk)) {
- struct l2t_entry *e = csk->l2t_entry;
-
- if (e && sk->sk_state != TCP_SYN_RECV) {
- cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
- return;
- }
- }
- cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
-}
-
-/*
- * This is run from a listener's backlog to abort a child connection in
- * SYN_RCV state (i.e., one on the listener's SYN queue).
- */
-static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
-{
- struct chtls_sock *csk;
- struct sock *child;
- int queue;
-
- child = skb->sk;
- csk = rcu_dereference_sk_user_data(child);
- queue = csk->txq_idx;
-
- skb->sk = NULL;
- chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
- CPL_ABORT_NO_RST, queue);
- do_abort_syn_rcv(child, lsk);
-}
-
-static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
-{
- const struct request_sock *oreq;
- struct listen_ctx *listen_ctx;
- struct chtls_sock *csk;
- struct chtls_dev *cdev;
- struct sock *psk;
- void *ctx;
-
- csk = sk->sk_user_data;
- oreq = csk->passive_reap_next;
- cdev = csk->cdev;
-
- if (!oreq)
- return -1;
-
- ctx = lookup_stid(cdev->tids, oreq->ts_recent);
- if (!ctx)
- return -1;
-
- listen_ctx = (struct listen_ctx *)ctx;
- psk = listen_ctx->lsk;
-
- bh_lock_sock(psk);
- if (!sock_owned_by_user(psk)) {
- int queue = csk->txq_idx;
-
- chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
- do_abort_syn_rcv(sk, psk);
- } else {
- skb->sk = sk;
- BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
- __sk_add_backlog(psk, skb);
- }
- bh_unlock_sock(psk);
- return 0;
-}
-
-static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
-{
- const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk = sk->sk_user_data;
- int rst_status = CPL_ABORT_NO_RST;
- int queue = csk->txq_idx;
-
- if (is_neg_adv(req->status)) {
- kfree_skb(skb);
- return;
- }
-
- csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
-
- if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
- !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
- struct tcp_sock *tp = tcp_sk(sk);
-
- if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
- WARN_ONCE(1, "send_tx_flowc error");
- csk_set_flag(csk, CSK_TX_DATA_SENT);
- }
-
- csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
-
- if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
- sk->sk_err = ETIMEDOUT;
-
- if (!sock_flag(sk, SOCK_DEAD))
- sk_error_report(sk);
-
- if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
- return;
-
- }
-
- chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
- rst_status, queue);
- chtls_release_resources(sk);
- chtls_conn_done(sk);
-}
-
-static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
-{
- struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk;
- struct chtls_dev *cdev;
-
- csk = rcu_dereference_sk_user_data(sk);
- cdev = csk->cdev;
-
- if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
- csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
- if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
- if (sk->sk_state == TCP_SYN_SENT) {
- cxgb4_remove_tid(cdev->tids,
- csk->port_id,
- GET_TID(rpl),
- sk->sk_family);
- sock_put(sk);
- }
- chtls_release_resources(sk);
- chtls_conn_done(sk);
- }
- }
- kfree_skb(skb);
-}
-
-static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
- void (*fn)(struct sock *sk, struct sk_buff *skb);
- unsigned int hwtid = GET_TID(req);
- struct chtls_sock *csk;
- struct sock *sk;
- u8 opcode;
-
- opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
-
- sk = lookup_tid(cdev->tids, hwtid);
- if (!sk)
- goto rel_skb;
-
- csk = sk->sk_user_data;
-
- switch (opcode) {
- case CPL_PEER_CLOSE:
- fn = chtls_peer_close;
- break;
- case CPL_CLOSE_CON_RPL:
- fn = chtls_close_con_rpl;
- break;
- case CPL_ABORT_REQ_RSS:
- /*
- * Save the offload device in the skb, we may process this
- * message after the socket has closed.
- */
- BLOG_SKB_CB(skb)->cdev = csk->cdev;
- fn = chtls_abort_req_rss;
- break;
- case CPL_ABORT_RPL_RSS:
- fn = chtls_abort_rpl_rss;
- break;
- default:
- goto rel_skb;
- }
-
- process_cpl_msg(fn, sk, skb);
- return 0;
-
-rel_skb:
- kfree_skb(skb);
- return 0;
-}
-
-static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
-{
- struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
- struct chtls_sock *csk = sk->sk_user_data;
- struct tcp_sock *tp = tcp_sk(sk);
- u32 credits = hdr->credits;
- u32 snd_una;
-
- snd_una = ntohl(hdr->snd_una);
- csk->wr_credits += credits;
-
- if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
- csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
-
- while (credits) {
- struct sk_buff *pskb = csk->wr_skb_head;
- u32 csum;
-
- if (unlikely(!pskb)) {
- if (csk->wr_nondata)
- csk->wr_nondata -= credits;
- break;
- }
- csum = (__force u32)pskb->csum;
- if (unlikely(credits < csum)) {
- pskb->csum = (__force __wsum)(csum - credits);
- break;
- }
- dequeue_wr(sk);
- credits -= csum;
- kfree_skb(pskb);
- }
- if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
- if (unlikely(before(snd_una, tp->snd_una))) {
- kfree_skb(skb);
- return;
- }
-
- if (tp->snd_una != snd_una) {
- tp->snd_una = snd_una;
- tp->rcv_tstamp = tcp_jiffies32;
- if (tp->snd_una == tp->snd_nxt &&
- !csk_flag_nochk(csk, CSK_TX_FAILOVER))
- csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
- }
- }
-
- if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
- unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
-
- csk->wr_credits -= fclen16;
- csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
- csk_reset_flag(csk, CSK_TX_FAILOVER);
- }
- if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
- sk->sk_write_space(sk);
-
- kfree_skb(skb);
-}
-
-static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
- unsigned int hwtid = GET_TID(rpl);
- struct sock *sk;
-
- sk = lookup_tid(cdev->tids, hwtid);
- if (unlikely(!sk)) {
- pr_err("can't find conn. for hwtid %u.\n", hwtid);
- return -EINVAL;
- }
- process_cpl_msg(chtls_rx_ack, sk, skb);
-
- return 0;
-}
-
-static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
-{
- struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR;
- unsigned int hwtid = GET_TID(rpl);
- struct sock *sk;
-
- sk = lookup_tid(cdev->tids, hwtid);
-
- /* return EINVAL if socket doesn't exist */
- if (!sk)
- return -EINVAL;
-
- /* Reusing the skb as size of cpl_set_tcb_field structure
- * is greater than cpl_abort_req
- */
- if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG)
- chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL);
-
- kfree_skb(skb);
- return 0;
-}
-
-chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
- [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl,
- [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
- [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req,
- [CPL_PASS_ESTABLISH] = chtls_pass_establish,
- [CPL_RX_DATA] = chtls_rx_data,
- [CPL_TLS_DATA] = chtls_rx_pdu,
- [CPL_RX_TLS_CMP] = chtls_rx_cmp,
- [CPL_PEER_CLOSE] = chtls_conn_cpl,
- [CPL_CLOSE_CON_RPL] = chtls_conn_cpl,
- [CPL_ABORT_REQ_RSS] = chtls_conn_cpl,
- [CPL_ABORT_RPL_RSS] = chtls_conn_cpl,
- [CPL_FW4_ACK] = chtls_wr_ack,
- [CPL_SET_TCB_RPL] = chtls_set_tcb_rpl,
-};
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
deleted file mode 100644
index 29ceff5a5fcb..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- */
-
-#ifndef __CHTLS_CM_H__
-#define __CHTLS_CM_H__
-
-/*
- * TCB settings
- */
-/* 3:0 */
-#define TCB_ULP_TYPE_W 0
-#define TCB_ULP_TYPE_S 0
-#define TCB_ULP_TYPE_M 0xfULL
-#define TCB_ULP_TYPE_V(x) ((x) << TCB_ULP_TYPE_S)
-
-/* 11:4 */
-#define TCB_ULP_RAW_W 0
-#define TCB_ULP_RAW_S 4
-#define TCB_ULP_RAW_M 0xffULL
-#define TCB_ULP_RAW_V(x) ((x) << TCB_ULP_RAW_S)
-
-#define TF_TLS_KEY_SIZE_S 7
-#define TF_TLS_KEY_SIZE_V(x) ((x) << TF_TLS_KEY_SIZE_S)
-
-#define TF_TLS_CONTROL_S 2
-#define TF_TLS_CONTROL_V(x) ((x) << TF_TLS_CONTROL_S)
-
-#define TF_TLS_ACTIVE_S 1
-#define TF_TLS_ACTIVE_V(x) ((x) << TF_TLS_ACTIVE_S)
-
-#define TF_TLS_ENABLE_S 0
-#define TF_TLS_ENABLE_V(x) ((x) << TF_TLS_ENABLE_S)
-
-#define TF_RX_QUIESCE_S 15
-#define TF_RX_QUIESCE_V(x) ((x) << TF_RX_QUIESCE_S)
-
-/*
- * Max receive window supported by HW in bytes. Only a small part of it can
- * be set through option0, the rest needs to be set through RX_DATA_ACK.
- */
-#define MAX_RCV_WND ((1U << 27) - 1)
-#define MAX_MSS 65536
-
-/*
- * Min receive window. We want it to be large enough to accommodate receive
- * coalescing, handle jumbo frames, and not trigger sender SWS avoidance.
- */
-#define MIN_RCV_WND (24 * 1024U)
-#define LOOPBACK(x) (((x) & htonl(0xff000000)) == htonl(0x7f000000))
-
-/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
-#define TX_HEADER_LEN \
- (sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr))
-#define TX_TLSHDR_LEN \
- (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo) + \
- sizeof(struct sge_opaque_hdr))
-#define TXDATA_SKB_LEN 128
-
-enum {
- CPL_TX_TLS_SFO_TYPE_CCS,
- CPL_TX_TLS_SFO_TYPE_ALERT,
- CPL_TX_TLS_SFO_TYPE_HANDSHAKE,
- CPL_TX_TLS_SFO_TYPE_DATA,
- CPL_TX_TLS_SFO_TYPE_HEARTBEAT,
-};
-
-enum {
- TLS_HDR_TYPE_CCS = 20,
- TLS_HDR_TYPE_ALERT,
- TLS_HDR_TYPE_HANDSHAKE,
- TLS_HDR_TYPE_RECORD,
- TLS_HDR_TYPE_HEARTBEAT,
-};
-
-typedef void (*defer_handler_t)(struct chtls_dev *dev, struct sk_buff *skb);
-extern struct request_sock_ops chtls_rsk_ops;
-extern struct request_sock_ops chtls_rsk_opsv6;
-
-struct deferred_skb_cb {
- defer_handler_t handler;
- struct chtls_dev *dev;
-};
-
-#define DEFERRED_SKB_CB(skb) ((struct deferred_skb_cb *)(skb)->cb)
-#define failover_flowc_wr_len offsetof(struct fw_flowc_wr, mnemval[3])
-#define WR_SKB_CB(skb) ((struct wr_skb_cb *)(skb)->cb)
-#define ACCEPT_QUEUE(sk) (&inet_csk(sk)->icsk_accept_queue.rskq_accept_head)
-
-#define SND_WSCALE(tp) ((tp)->rx_opt.snd_wscale)
-#define RCV_WSCALE(tp) ((tp)->rx_opt.rcv_wscale)
-#define USER_MSS(tp) (READ_ONCE((tp)->rx_opt.user_mss))
-#define TS_RECENT_STAMP(tp) ((tp)->rx_opt.ts_recent_stamp)
-#define WSCALE_OK(tp) ((tp)->rx_opt.wscale_ok)
-#define TSTAMP_OK(tp) ((tp)->rx_opt.tstamp_ok)
-#define SACK_OK(tp) ((tp)->rx_opt.sack_ok)
-
-/* TLS SKB */
-#define skb_ulp_tls_inline(skb) (ULP_SKB_CB(skb)->ulp.tls.ofld)
-#define skb_ulp_tls_iv_imm(skb) (ULP_SKB_CB(skb)->ulp.tls.iv)
-
-void chtls_defer_reply(struct sk_buff *skb, struct chtls_dev *dev,
- defer_handler_t handler);
-
-/*
- * Returns true if the socket is in one of the supplied states.
- */
-static inline unsigned int sk_in_state(const struct sock *sk,
- unsigned int states)
-{
- return states & (1 << sk->sk_state);
-}
-
-static void chtls_rsk_destructor(struct request_sock *req)
-{
- /* do nothing */
-}
-
-static inline void chtls_init_rsk_ops(struct proto *chtls_tcp_prot,
- struct request_sock_ops *chtls_tcp_ops,
- struct proto *tcp_prot, int family)
-{
- memset(chtls_tcp_ops, 0, sizeof(*chtls_tcp_ops));
- chtls_tcp_ops->family = family;
- chtls_tcp_ops->obj_size = sizeof(struct tcp_request_sock);
- chtls_tcp_ops->destructor = chtls_rsk_destructor;
- chtls_tcp_ops->slab = tcp_prot->rsk_prot->slab;
- chtls_tcp_prot->rsk_prot = chtls_tcp_ops;
-}
-
-static inline void chtls_reqsk_free(struct request_sock *req)
-{
- if (req->rsk_listener)
- sock_put(req->rsk_listener);
- kmem_cache_free(req->rsk_ops->slab, req);
-}
-
-#define DECLARE_TASK_FUNC(task, task_param) \
- static void task(struct work_struct *task_param)
-
-static inline void sk_wakeup_sleepers(struct sock *sk, bool interruptable)
-{
- struct socket_wq *wq;
-
- rcu_read_lock();
- wq = rcu_dereference(sk->sk_wq);
- if (skwq_has_sleeper(wq)) {
- if (interruptable)
- wake_up_interruptible(sk_sleep(sk));
- else
- wake_up_all(sk_sleep(sk));
- }
- rcu_read_unlock();
-}
-
-static inline void chtls_set_req_port(struct request_sock *oreq,
- __be16 source, __be16 dest)
-{
- inet_rsk(oreq)->ir_rmt_port = source;
- inet_rsk(oreq)->ir_num = ntohs(dest);
-}
-
-static inline void chtls_set_req_addr(struct request_sock *oreq,
- __be32 local_ip, __be32 peer_ip)
-{
- inet_rsk(oreq)->ir_loc_addr = local_ip;
- inet_rsk(oreq)->ir_rmt_addr = peer_ip;
-}
-
-static inline void chtls_free_skb(struct sock *sk, struct sk_buff *skb)
-{
- skb_dstref_steal(skb);
- __skb_unlink(skb, &sk->sk_receive_queue);
- __kfree_skb(skb);
-}
-
-static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb)
-{
- skb_dstref_steal(skb);
- __skb_unlink(skb, &sk->sk_receive_queue);
- kfree_skb(skb);
-}
-
-static inline void chtls_reset_wr_list(struct chtls_sock *csk)
-{
- csk->wr_skb_head = NULL;
- csk->wr_skb_tail = NULL;
-}
-
-static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb)
-{
- WR_SKB_CB(skb)->next_wr = NULL;
-
- skb_get(skb);
-
- if (!csk->wr_skb_head)
- csk->wr_skb_head = skb;
- else
- WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb;
- csk->wr_skb_tail = skb;
-}
-
-static inline struct sk_buff *dequeue_wr(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct sk_buff *skb = NULL;
-
- skb = csk->wr_skb_head;
-
- if (likely(skb)) {
- /* Don't bother clearing the tail */
- csk->wr_skb_head = WR_SKB_CB(skb)->next_wr;
- WR_SKB_CB(skb)->next_wr = NULL;
- }
- return skb;
-}
-#endif
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c
deleted file mode 100644
index d84473ca844d..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_hw.c
+++ /dev/null
@@ -1,462 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/tls.h>
-#include <net/tls.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-static void __set_tcb_field_direct(struct chtls_sock *csk,
- struct cpl_set_tcb_field *req, u16 word,
- u64 mask, u64 val, u8 cookie, int no_reply)
-{
- struct ulptx_idata *sc;
-
- INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, csk->tid);
- req->wr.wr_mid |= htonl(FW_WR_FLOWID_V(csk->tid));
- req->reply_ctrl = htons(NO_REPLY_V(no_reply) |
- QUEUENO_V(csk->rss_qid));
- req->word_cookie = htons(TCB_WORD_V(word) | TCB_COOKIE_V(cookie));
- req->mask = cpu_to_be64(mask);
- req->val = cpu_to_be64(val);
- sc = (struct ulptx_idata *)(req + 1);
- sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
- sc->len = htonl(0);
-}
-
-static void __set_tcb_field(struct sock *sk, struct sk_buff *skb, u16 word,
- u64 mask, u64 val, u8 cookie, int no_reply)
-{
- struct cpl_set_tcb_field *req;
- struct chtls_sock *csk;
- struct ulptx_idata *sc;
- unsigned int wrlen;
-
- wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
- csk = rcu_dereference_sk_user_data(sk);
-
- req = (struct cpl_set_tcb_field *)__skb_put(skb, wrlen);
- __set_tcb_field_direct(csk, req, word, mask, val, cookie, no_reply);
- set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
-}
-
-/*
- * Send control message to HW, message go as immediate data and packet
- * is freed immediately.
- */
-static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
-{
- struct cpl_set_tcb_field *req;
- unsigned int credits_needed;
- struct chtls_sock *csk;
- struct ulptx_idata *sc;
- struct sk_buff *skb;
- unsigned int wrlen;
- int ret;
-
- wrlen = roundup(sizeof(*req) + sizeof(*sc), 16);
-
- skb = alloc_skb(wrlen, GFP_ATOMIC);
- if (!skb)
- return -ENOMEM;
-
- credits_needed = DIV_ROUND_UP(wrlen, 16);
- csk = rcu_dereference_sk_user_data(sk);
-
- __set_tcb_field(sk, skb, word, mask, val, 0, 1);
- skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
- csk->wr_credits -= credits_needed;
- csk->wr_unacked += credits_needed;
- enqueue_wr(csk, skb);
- ret = cxgb4_ofld_send(csk->egress_dev, skb);
- if (ret < 0)
- kfree_skb(skb);
- return ret < 0 ? ret : 0;
-}
-
-void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
- u64 mask, u64 val, u8 cookie,
- int through_l2t)
-{
- struct sk_buff *skb;
- unsigned int wrlen;
-
- wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
- wrlen = roundup(wrlen, 16);
-
- skb = alloc_skb(wrlen, GFP_KERNEL | __GFP_NOFAIL);
- if (!skb)
- return;
-
- __set_tcb_field(sk, skb, word, mask, val, cookie, 0);
- send_or_defer(sk, tcp_sk(sk), skb, through_l2t);
-}
-
-static int chtls_set_tcb_keyid(struct sock *sk, int keyid)
-{
- return chtls_set_tcb_field(sk, 31, 0xFFFFFFFFULL, keyid);
-}
-
-static int chtls_set_tcb_seqno(struct sock *sk)
-{
- return chtls_set_tcb_field(sk, 28, ~0ULL, 0);
-}
-
-static int chtls_set_tcb_quiesce(struct sock *sk, int val)
-{
- return chtls_set_tcb_field(sk, 1, (1ULL << TF_RX_QUIESCE_S),
- TF_RX_QUIESCE_V(val));
-}
-
-void chtls_set_quiesce_ctrl(struct sock *sk, int val)
-{
- struct chtls_sock *csk;
- struct sk_buff *skb;
- unsigned int wrlen;
- int ret;
-
- wrlen = sizeof(struct cpl_set_tcb_field) + sizeof(struct ulptx_idata);
- wrlen = roundup(wrlen, 16);
-
- skb = alloc_skb(wrlen, GFP_ATOMIC);
- if (!skb)
- return;
-
- csk = rcu_dereference_sk_user_data(sk);
-
- __set_tcb_field(sk, skb, 1, TF_RX_QUIESCE_V(1), 0, 0, 1);
- set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
- ret = cxgb4_ofld_send(csk->egress_dev, skb);
- if (ret < 0)
- kfree_skb(skb);
-}
-
-/* TLS Key bitmap processing */
-int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi)
-{
- unsigned int num_key_ctx, bsize;
- int ksize;
-
- num_key_ctx = (lldi->vr->key.size / TLS_KEY_CONTEXT_SZ);
- bsize = BITS_TO_LONGS(num_key_ctx);
-
- cdev->kmap.size = num_key_ctx;
- cdev->kmap.available = bsize;
- ksize = sizeof(*cdev->kmap.addr) * bsize;
- cdev->kmap.addr = kvzalloc(ksize, GFP_KERNEL);
- if (!cdev->kmap.addr)
- return -ENOMEM;
-
- cdev->kmap.start = lldi->vr->key.start;
- spin_lock_init(&cdev->kmap.lock);
- return 0;
-}
-
-static int get_new_keyid(struct chtls_sock *csk, u32 optname)
-{
- struct net_device *dev = csk->egress_dev;
- struct chtls_dev *cdev = csk->cdev;
- struct chtls_hws *hws;
- struct adapter *adap;
- int keyid;
-
- adap = netdev2adap(dev);
- hws = &csk->tlshws;
-
- spin_lock_bh(&cdev->kmap.lock);
- keyid = find_first_zero_bit(cdev->kmap.addr, cdev->kmap.size);
- if (keyid < cdev->kmap.size) {
- __set_bit(keyid, cdev->kmap.addr);
- if (optname == TLS_RX)
- hws->rxkey = keyid;
- else
- hws->txkey = keyid;
- atomic_inc(&adap->chcr_stats.tls_key);
- } else {
- keyid = -1;
- }
- spin_unlock_bh(&cdev->kmap.lock);
- return keyid;
-}
-
-void free_tls_keyid(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct net_device *dev = csk->egress_dev;
- struct chtls_dev *cdev = csk->cdev;
- struct chtls_hws *hws;
- struct adapter *adap;
-
- if (!cdev->kmap.addr)
- return;
-
- adap = netdev2adap(dev);
- hws = &csk->tlshws;
-
- spin_lock_bh(&cdev->kmap.lock);
- if (hws->rxkey >= 0) {
- __clear_bit(hws->rxkey, cdev->kmap.addr);
- atomic_dec(&adap->chcr_stats.tls_key);
- hws->rxkey = -1;
- }
- if (hws->txkey >= 0) {
- __clear_bit(hws->txkey, cdev->kmap.addr);
- atomic_dec(&adap->chcr_stats.tls_key);
- hws->txkey = -1;
- }
- spin_unlock_bh(&cdev->kmap.lock);
-}
-
-unsigned int keyid_to_addr(int start_addr, int keyid)
-{
- return (start_addr + (keyid * TLS_KEY_CONTEXT_SZ)) >> 5;
-}
-
-static void chtls_rxkey_ivauth(struct _key_ctx *kctx)
-{
- kctx->iv_to_auth = cpu_to_be64(KEYCTX_TX_WR_IV_V(6ULL) |
- KEYCTX_TX_WR_AAD_V(1ULL) |
- KEYCTX_TX_WR_AADST_V(5ULL) |
- KEYCTX_TX_WR_CIPHER_V(14ULL) |
- KEYCTX_TX_WR_CIPHERST_V(0ULL) |
- KEYCTX_TX_WR_AUTH_V(14ULL) |
- KEYCTX_TX_WR_AUTHST_V(16ULL) |
- KEYCTX_TX_WR_AUTHIN_V(16ULL));
-}
-
-static int chtls_key_info(struct chtls_sock *csk,
- struct _key_ctx *kctx,
- u32 keylen, u32 optname,
- int cipher_type)
-{
- unsigned char key[AES_MAX_KEY_SIZE];
- unsigned char *key_p, *salt;
- unsigned char ghash_h[AEAD_H_SIZE];
- int ck_size, key_ctx_size, kctx_mackey_size, salt_size;
- struct aes_enckey aes;
- int ret;
-
- key_ctx_size = sizeof(struct _key_ctx) +
- roundup(keylen, 16) + AEAD_H_SIZE;
-
- /* GCM mode of AES supports 128 and 256 bit encryption, so
- * prepare key context base on GCM cipher type
- */
- switch (cipher_type) {
- case TLS_CIPHER_AES_GCM_128: {
- struct tls12_crypto_info_aes_gcm_128 *gcm_ctx_128 =
- (struct tls12_crypto_info_aes_gcm_128 *)
- &csk->tlshws.crypto_info;
- memcpy(key, gcm_ctx_128->key, keylen);
-
- key_p = gcm_ctx_128->key;
- salt = gcm_ctx_128->salt;
- ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
- salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
- kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
- break;
- }
- case TLS_CIPHER_AES_GCM_256: {
- struct tls12_crypto_info_aes_gcm_256 *gcm_ctx_256 =
- (struct tls12_crypto_info_aes_gcm_256 *)
- &csk->tlshws.crypto_info;
- memcpy(key, gcm_ctx_256->key, keylen);
-
- key_p = gcm_ctx_256->key;
- salt = gcm_ctx_256->salt;
- ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
- salt_size = TLS_CIPHER_AES_GCM_256_SALT_SIZE;
- kctx_mackey_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
- break;
- }
- default:
- pr_err("GCM: Invalid key length %d\n", keylen);
- return -EINVAL;
- }
-
- /* Calculate the H = CIPH(K, 0 repeated 16 times).
- * It will go in key context
- */
- ret = aes_prepareenckey(&aes, key, keylen);
- if (ret)
- return ret;
-
- memset(ghash_h, 0, AEAD_H_SIZE);
- aes_encrypt(&aes, ghash_h, ghash_h);
- memzero_explicit(&aes, sizeof(aes));
- csk->tlshws.keylen = key_ctx_size;
-
- /* Copy the Key context */
- if (optname == TLS_RX) {
- int key_ctx;
-
- key_ctx = ((key_ctx_size >> 4) << 3);
- kctx->ctx_hdr = FILL_KEY_CRX_HDR(ck_size,
- kctx_mackey_size,
- 0, 0, key_ctx);
- chtls_rxkey_ivauth(kctx);
- } else {
- kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
- kctx_mackey_size,
- 0, 0, key_ctx_size >> 4);
- }
-
- memcpy(kctx->salt, salt, salt_size);
- memcpy(kctx->key, key_p, keylen);
- memcpy(kctx->key + keylen, ghash_h, AEAD_H_SIZE);
- /* erase key info from driver */
- memset(key_p, 0, keylen);
-
- return 0;
-}
-
-static void chtls_set_scmd(struct chtls_sock *csk)
-{
- struct chtls_hws *hws = &csk->tlshws;
-
- hws->scmd.seqno_numivs =
- SCMD_SEQ_NO_CTRL_V(3) |
- SCMD_PROTO_VERSION_V(0) |
- SCMD_ENC_DEC_CTRL_V(0) |
- SCMD_CIPH_AUTH_SEQ_CTRL_V(1) |
- SCMD_CIPH_MODE_V(2) |
- SCMD_AUTH_MODE_V(4) |
- SCMD_HMAC_CTRL_V(0) |
- SCMD_IV_SIZE_V(4) |
- SCMD_NUM_IVS_V(1);
-
- hws->scmd.ivgen_hdrlen =
- SCMD_IV_GEN_CTRL_V(1) |
- SCMD_KEY_CTX_INLINE_V(0) |
- SCMD_TLS_FRAG_ENABLE_V(1);
-}
-
-int chtls_setkey(struct chtls_sock *csk, u32 keylen,
- u32 optname, int cipher_type)
-{
- struct tls_key_req *kwr;
- struct chtls_dev *cdev;
- struct _key_ctx *kctx;
- int wrlen, klen, len;
- struct sk_buff *skb;
- struct sock *sk;
- int keyid;
- int kaddr;
- int ret;
-
- cdev = csk->cdev;
- sk = csk->sk;
-
- klen = roundup((keylen + AEAD_H_SIZE) + sizeof(*kctx), 32);
- wrlen = roundup(sizeof(*kwr), 16);
- len = klen + wrlen;
-
- /* Flush out-standing data before new key takes effect */
- if (optname == TLS_TX) {
- lock_sock(sk);
- if (skb_queue_len(&csk->txq))
- chtls_push_frames(csk, 0);
- release_sock(sk);
- }
-
- skb = alloc_skb(len, GFP_KERNEL);
- if (!skb)
- return -ENOMEM;
-
- keyid = get_new_keyid(csk, optname);
- if (keyid < 0) {
- ret = -ENOSPC;
- goto out_nokey;
- }
-
- kaddr = keyid_to_addr(cdev->kmap.start, keyid);
- kwr = (struct tls_key_req *)__skb_put_zero(skb, len);
- kwr->wr.op_to_compl =
- cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) | FW_WR_COMPL_F |
- FW_WR_ATOMIC_V(1U));
- kwr->wr.flowid_len16 =
- cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16) |
- FW_WR_FLOWID_V(csk->tid)));
- kwr->wr.protocol = 0;
- kwr->wr.mfs = htons(TLS_MFS);
- kwr->wr.reneg_to_write_rx = optname;
-
- /* ulptx command */
- kwr->req.cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
- T5_ULP_MEMIO_ORDER_V(1) |
- T5_ULP_MEMIO_IMM_V(1));
- kwr->req.len16 = cpu_to_be32((csk->tid << 8) |
- DIV_ROUND_UP(len - sizeof(kwr->wr), 16));
- kwr->req.dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(klen >> 5));
- kwr->req.lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(kaddr));
-
- /* sub command */
- kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
- kwr->sc_imm.len = cpu_to_be32(klen);
-
- lock_sock(sk);
- /* key info */
- kctx = (struct _key_ctx *)(kwr + 1);
- ret = chtls_key_info(csk, kctx, keylen, optname, cipher_type);
- if (ret)
- goto out_notcb;
-
- if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
- goto out_notcb;
-
- set_wr_txq(skb, CPL_PRIORITY_DATA, csk->tlshws.txqid);
- csk->wr_credits -= DIV_ROUND_UP(len, 16);
- csk->wr_unacked += DIV_ROUND_UP(len, 16);
- enqueue_wr(csk, skb);
- cxgb4_ofld_send(csk->egress_dev, skb);
- skb = NULL;
-
- chtls_set_scmd(csk);
- /* Clear quiesce for Rx key */
- if (optname == TLS_RX) {
- ret = chtls_set_tcb_keyid(sk, keyid);
- if (ret)
- goto out_notcb;
- ret = chtls_set_tcb_field(sk, 0,
- TCB_ULP_RAW_V(TCB_ULP_RAW_M),
- TCB_ULP_RAW_V((TF_TLS_KEY_SIZE_V(1) |
- TF_TLS_CONTROL_V(1) |
- TF_TLS_ACTIVE_V(1) |
- TF_TLS_ENABLE_V(1))));
- if (ret)
- goto out_notcb;
- ret = chtls_set_tcb_seqno(sk);
- if (ret)
- goto out_notcb;
- ret = chtls_set_tcb_quiesce(sk, 0);
- if (ret)
- goto out_notcb;
- csk->tlshws.rxkey = keyid;
- } else {
- csk->tlshws.tx_seq_no = 0;
- csk->tlshws.txkey = keyid;
- }
-
- release_sock(sk);
- return ret;
-out_notcb:
- release_sock(sk);
- free_tls_keyid(sk);
-out_nokey:
- kfree_skb(skb);
- return ret;
-}
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
deleted file mode 100644
index c8e99409a52a..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c
+++ /dev/null
@@ -1,1836 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-
-#include <linux/module.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-#include <linux/skbuff.h>
-#include <linux/timer.h>
-#include <linux/notifier.h>
-#include <linux/inetdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/sched/signal.h>
-#include <net/tcp.h>
-#include <net/busy_poll.h>
-#include <crypto/aes.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-static bool is_tls_tx(struct chtls_sock *csk)
-{
- return csk->tlshws.txkey >= 0;
-}
-
-static bool is_tls_rx(struct chtls_sock *csk)
-{
- return csk->tlshws.rxkey >= 0;
-}
-
-static int data_sgl_len(const struct sk_buff *skb)
-{
- unsigned int cnt;
-
- cnt = skb_shinfo(skb)->nr_frags;
- return sgl_len(cnt) * 8;
-}
-
-static int nos_ivs(struct sock *sk, unsigned int size)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- return DIV_ROUND_UP(size, csk->tlshws.mfs);
-}
-
-static int set_ivs_imm(struct sock *sk, const struct sk_buff *skb)
-{
- int ivs_size = nos_ivs(sk, skb->len) * CIPHER_BLOCK_SIZE;
- int hlen = TLS_WR_CPL_LEN + data_sgl_len(skb);
-
- if ((hlen + KEY_ON_MEM_SZ + ivs_size) <
- MAX_IMM_OFLD_TX_DATA_WR_LEN) {
- ULP_SKB_CB(skb)->ulp.tls.iv = 1;
- return 1;
- }
- ULP_SKB_CB(skb)->ulp.tls.iv = 0;
- return 0;
-}
-
-static int max_ivs_size(struct sock *sk, int size)
-{
- return nos_ivs(sk, size) * CIPHER_BLOCK_SIZE;
-}
-
-static int ivs_size(struct sock *sk, const struct sk_buff *skb)
-{
- return set_ivs_imm(sk, skb) ? (nos_ivs(sk, skb->len) *
- CIPHER_BLOCK_SIZE) : 0;
-}
-
-static int flowc_wr_credits(int nparams, int *flowclenp)
-{
- int flowclen16, flowclen;
-
- flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
- flowclen16 = DIV_ROUND_UP(flowclen, 16);
- flowclen = flowclen16 * 16;
-
- if (flowclenp)
- *flowclenp = flowclen;
-
- return flowclen16;
-}
-
-static struct sk_buff *create_flowc_wr_skb(struct sock *sk,
- struct fw_flowc_wr *flowc,
- int flowclen)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct sk_buff *skb;
-
- skb = alloc_skb(flowclen, GFP_ATOMIC);
- if (!skb)
- return NULL;
-
- __skb_put_data(skb, flowc, flowclen);
- skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
-
- return skb;
-}
-
-static int send_flowc_wr(struct sock *sk, struct fw_flowc_wr *flowc,
- int flowclen)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct tcp_sock *tp = tcp_sk(sk);
- struct sk_buff *skb;
- int flowclen16;
- int ret;
-
- flowclen16 = flowclen / 16;
-
- if (csk_flag(sk, CSK_TX_DATA_SENT)) {
- skb = create_flowc_wr_skb(sk, flowc, flowclen);
- if (!skb)
- return -ENOMEM;
-
- skb_entail(sk, skb,
- ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
- return 0;
- }
-
- ret = cxgb4_immdata_send(csk->egress_dev,
- csk->txq_idx,
- flowc, flowclen);
- if (!ret)
- return flowclen16;
- skb = create_flowc_wr_skb(sk, flowc, flowclen);
- if (!skb)
- return -ENOMEM;
- send_or_defer(sk, tp, skb, 0);
- return flowclen16;
-}
-
-static u8 tcp_state_to_flowc_state(u8 state)
-{
- switch (state) {
- case TCP_ESTABLISHED:
- return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
- case TCP_CLOSE_WAIT:
- return FW_FLOWC_MNEM_TCPSTATE_CLOSEWAIT;
- case TCP_FIN_WAIT1:
- return FW_FLOWC_MNEM_TCPSTATE_FINWAIT1;
- case TCP_CLOSING:
- return FW_FLOWC_MNEM_TCPSTATE_CLOSING;
- case TCP_LAST_ACK:
- return FW_FLOWC_MNEM_TCPSTATE_LASTACK;
- case TCP_FIN_WAIT2:
- return FW_FLOWC_MNEM_TCPSTATE_FINWAIT2;
- }
-
- return FW_FLOWC_MNEM_TCPSTATE_ESTABLISHED;
-}
-
-int send_tx_flowc_wr(struct sock *sk, int compl,
- u32 snd_nxt, u32 rcv_nxt)
-{
- DEFINE_RAW_FLEX(struct fw_flowc_wr, flowc, mnemval, FW_FLOWC_MNEM_MAX);
- int nparams, paramidx, flowclen16, flowclen;
- struct chtls_sock *csk;
- struct tcp_sock *tp;
-
- csk = rcu_dereference_sk_user_data(sk);
- tp = tcp_sk(sk);
-
-#define FLOWC_PARAM(__m, __v) \
- do { \
- flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
- flowc->mnemval[paramidx].val = cpu_to_be32(__v); \
- paramidx++; \
- } while (0)
-
- paramidx = 0;
-
- FLOWC_PARAM(PFNVFN, FW_PFVF_CMD_PFN_V(csk->cdev->lldi->pf));
- FLOWC_PARAM(CH, csk->tx_chan);
- FLOWC_PARAM(PORT, csk->tx_chan);
- FLOWC_PARAM(IQID, csk->rss_qid);
- FLOWC_PARAM(SNDNXT, tp->snd_nxt);
- FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
- FLOWC_PARAM(SNDBUF, csk->sndbuf);
- FLOWC_PARAM(MSS, tp->mss_cache);
- FLOWC_PARAM(TCPSTATE, tcp_state_to_flowc_state(sk->sk_state));
-
- if (SND_WSCALE(tp))
- FLOWC_PARAM(RCV_SCALE, SND_WSCALE(tp));
-
- if (csk->ulp_mode == ULP_MODE_TLS)
- FLOWC_PARAM(ULD_MODE, ULP_MODE_TLS);
-
- if (csk->tlshws.fcplenmax)
- FLOWC_PARAM(TXDATAPLEN_MAX, csk->tlshws.fcplenmax);
-
- nparams = paramidx;
-#undef FLOWC_PARAM
-
- flowclen16 = flowc_wr_credits(nparams, &flowclen);
- flowc->op_to_nparams =
- cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
- FW_WR_COMPL_V(compl) |
- FW_FLOWC_WR_NPARAMS_V(nparams));
- flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |
- FW_WR_FLOWID_V(csk->tid));
-
- return send_flowc_wr(sk, flowc, flowclen);
-}
-
-/* Copy IVs to WR */
-static int tls_copy_ivs(struct sock *sk, struct sk_buff *skb)
-
-{
- struct chtls_sock *csk;
- unsigned char *iv_loc;
- struct chtls_hws *hws;
- unsigned char *ivs;
- u16 number_of_ivs;
- struct page *page;
- int err = 0;
-
- csk = rcu_dereference_sk_user_data(sk);
- hws = &csk->tlshws;
- number_of_ivs = nos_ivs(sk, skb->len);
-
- if (number_of_ivs > MAX_IVS_PAGE) {
- pr_warn("MAX IVs in PAGE exceeded %d\n", number_of_ivs);
- return -ENOMEM;
- }
-
- /* generate the IVs */
- ivs = kmalloc_array(CIPHER_BLOCK_SIZE, number_of_ivs, GFP_ATOMIC);
- if (!ivs)
- return -ENOMEM;
- get_random_bytes(ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
-
- if (skb_ulp_tls_iv_imm(skb)) {
- /* send the IVs as immediate data in the WR */
- iv_loc = (unsigned char *)__skb_push(skb, number_of_ivs *
- CIPHER_BLOCK_SIZE);
- if (iv_loc)
- memcpy(iv_loc, ivs, number_of_ivs * CIPHER_BLOCK_SIZE);
-
- hws->ivsize = number_of_ivs * CIPHER_BLOCK_SIZE;
- } else {
- /* Send the IVs as sgls */
- /* Already accounted IV DSGL for credits */
- skb_shinfo(skb)->nr_frags--;
- page = alloc_pages(sk->sk_allocation | __GFP_COMP, 0);
- if (!page) {
- pr_info("%s : Page allocation for IVs failed\n",
- __func__);
- err = -ENOMEM;
- goto out;
- }
- memcpy(page_address(page), ivs, number_of_ivs *
- CIPHER_BLOCK_SIZE);
- skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, page, 0,
- number_of_ivs * CIPHER_BLOCK_SIZE);
- hws->ivsize = 0;
- }
-out:
- kfree(ivs);
- return err;
-}
-
-/* Copy Key to WR */
-static void tls_copy_tx_key(struct sock *sk, struct sk_buff *skb)
-{
- struct ulptx_sc_memrd *sc_memrd;
- struct chtls_sock *csk;
- struct chtls_dev *cdev;
- struct ulptx_idata *sc;
- struct chtls_hws *hws;
- u32 immdlen;
- int kaddr;
-
- csk = rcu_dereference_sk_user_data(sk);
- hws = &csk->tlshws;
- cdev = csk->cdev;
-
- immdlen = sizeof(*sc) + sizeof(*sc_memrd);
- kaddr = keyid_to_addr(cdev->kmap.start, hws->txkey);
- sc = (struct ulptx_idata *)__skb_push(skb, immdlen);
- if (sc) {
- sc->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
- sc->len = htonl(0);
- sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
- sc_memrd->cmd_to_len =
- htonl(ULPTX_CMD_V(ULP_TX_SC_MEMRD) |
- ULP_TX_SC_MORE_V(1) |
- ULPTX_LEN16_V(hws->keylen >> 4));
- sc_memrd->addr = htonl(kaddr);
- }
-}
-
-static u64 tlstx_incr_seqnum(struct chtls_hws *hws)
-{
- return hws->tx_seq_no++;
-}
-
-static bool is_sg_request(const struct sk_buff *skb)
-{
- return skb->peeked ||
- (skb->len > MAX_IMM_ULPTX_WR_LEN);
-}
-
-/*
- * Returns true if an sk_buff carries urgent data.
- */
-static bool skb_urgent(struct sk_buff *skb)
-{
- return ULP_SKB_CB(skb)->flags & ULPCB_FLAG_URG;
-}
-
-/* TLS content type for CPL SFO */
-static unsigned char tls_content_type(unsigned char content_type)
-{
- switch (content_type) {
- case TLS_HDR_TYPE_CCS:
- return CPL_TX_TLS_SFO_TYPE_CCS;
- case TLS_HDR_TYPE_ALERT:
- return CPL_TX_TLS_SFO_TYPE_ALERT;
- case TLS_HDR_TYPE_HANDSHAKE:
- return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
- case TLS_HDR_TYPE_HEARTBEAT:
- return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
- }
- return CPL_TX_TLS_SFO_TYPE_DATA;
-}
-
-static void tls_tx_data_wr(struct sock *sk, struct sk_buff *skb,
- int dlen, int tls_immd, u32 credits,
- int expn, int pdus)
-{
- struct fw_tlstx_data_wr *req_wr;
- struct cpl_tx_tls_sfo *req_cpl;
- unsigned int wr_ulp_mode_force;
- struct tls_scmd *updated_scmd;
- unsigned char data_type;
- struct chtls_sock *csk;
- struct net_device *dev;
- struct chtls_hws *hws;
- struct tls_scmd *scmd;
- struct adapter *adap;
- unsigned char *req;
- int immd_len;
- int iv_imm;
- int len;
-
- csk = rcu_dereference_sk_user_data(sk);
- iv_imm = skb_ulp_tls_iv_imm(skb);
- dev = csk->egress_dev;
- adap = netdev2adap(dev);
- hws = &csk->tlshws;
- scmd = &hws->scmd;
- len = dlen + expn;
-
- dlen = (dlen < hws->mfs) ? dlen : hws->mfs;
- atomic_inc(&adap->chcr_stats.tls_pdu_tx);
-
- updated_scmd = scmd;
- updated_scmd->seqno_numivs &= 0xffffff80;
- updated_scmd->seqno_numivs |= SCMD_NUM_IVS_V(pdus);
- hws->scmd = *updated_scmd;
-
- req = (unsigned char *)__skb_push(skb, sizeof(struct cpl_tx_tls_sfo));
- req_cpl = (struct cpl_tx_tls_sfo *)req;
- req = (unsigned char *)__skb_push(skb, (sizeof(struct
- fw_tlstx_data_wr)));
-
- req_wr = (struct fw_tlstx_data_wr *)req;
- immd_len = (tls_immd ? dlen : 0);
- req_wr->op_to_immdlen =
- htonl(FW_WR_OP_V(FW_TLSTX_DATA_WR) |
- FW_TLSTX_DATA_WR_COMPL_V(1) |
- FW_TLSTX_DATA_WR_IMMDLEN_V(immd_len));
- req_wr->flowid_len16 = htonl(FW_TLSTX_DATA_WR_FLOWID_V(csk->tid) |
- FW_TLSTX_DATA_WR_LEN16_V(credits));
- wr_ulp_mode_force = TX_ULP_MODE_V(ULP_MODE_TLS);
-
- if (is_sg_request(skb))
- wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
- ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
- FW_OFLD_TX_DATA_WR_SHOVE_F);
-
- req_wr->lsodisable_to_flags =
- htonl(TX_ULP_MODE_V(ULP_MODE_TLS) |
- TX_URG_V(skb_urgent(skb)) |
- T6_TX_FORCE_F | wr_ulp_mode_force |
- TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) &&
- skb_queue_empty(&csk->txq)));
-
- req_wr->ctxloc_to_exp =
- htonl(FW_TLSTX_DATA_WR_NUMIVS_V(pdus) |
- FW_TLSTX_DATA_WR_EXP_V(expn) |
- FW_TLSTX_DATA_WR_CTXLOC_V(CHTLS_KEY_CONTEXT_DDR) |
- FW_TLSTX_DATA_WR_IVDSGL_V(!iv_imm) |
- FW_TLSTX_DATA_WR_KEYSIZE_V(hws->keylen >> 4));
-
- /* Fill in the length */
- req_wr->plen = htonl(len);
- req_wr->mfs = htons(hws->mfs);
- req_wr->adjustedplen_pkd =
- htons(FW_TLSTX_DATA_WR_ADJUSTEDPLEN_V(hws->adjustlen));
- req_wr->expinplenmax_pkd =
- htons(FW_TLSTX_DATA_WR_EXPINPLENMAX_V(hws->expansion));
- req_wr->pdusinplenmax_pkd =
- FW_TLSTX_DATA_WR_PDUSINPLENMAX_V(hws->pdus);
- req_wr->r10 = 0;
-
- data_type = tls_content_type(ULP_SKB_CB(skb)->ulp.tls.type);
- req_cpl->op_to_seg_len = htonl(CPL_TX_TLS_SFO_OPCODE_V(CPL_TX_TLS_SFO) |
- CPL_TX_TLS_SFO_DATA_TYPE_V(data_type) |
- CPL_TX_TLS_SFO_CPL_LEN_V(2) |
- CPL_TX_TLS_SFO_SEG_LEN_V(dlen));
- req_cpl->pld_len = htonl(len - expn);
-
- req_cpl->type_protover = htonl(CPL_TX_TLS_SFO_TYPE_V
- ((data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT) ?
- TLS_HDR_TYPE_HEARTBEAT : 0) |
- CPL_TX_TLS_SFO_PROTOVER_V(0));
-
- /* create the s-command */
- req_cpl->r1_lo = 0;
- req_cpl->seqno_numivs = cpu_to_be32(hws->scmd.seqno_numivs);
- req_cpl->ivgen_hdrlen = cpu_to_be32(hws->scmd.ivgen_hdrlen);
- req_cpl->scmd1 = cpu_to_be64(tlstx_incr_seqnum(hws));
-}
-
-/*
- * Calculate the TLS data expansion size
- */
-static int chtls_expansion_size(struct sock *sk, int data_len,
- int fullpdu,
- unsigned short *pducnt)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_hws *hws = &csk->tlshws;
- struct tls_scmd *scmd = &hws->scmd;
- int fragsize = hws->mfs;
- int expnsize = 0;
- int fragleft;
- int fragcnt;
- int expppdu;
-
- if (SCMD_CIPH_MODE_G(scmd->seqno_numivs) ==
- SCMD_CIPH_MODE_AES_GCM) {
- expppdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
- TLS_HEADER_LENGTH;
-
- if (fullpdu) {
- *pducnt = data_len / (expppdu + fragsize);
- if (*pducnt > 32)
- *pducnt = 32;
- else if (!*pducnt)
- *pducnt = 1;
- expnsize = (*pducnt) * expppdu;
- return expnsize;
- }
- fragcnt = (data_len / fragsize);
- expnsize = fragcnt * expppdu;
- fragleft = data_len % fragsize;
- if (fragleft > 0)
- expnsize += expppdu;
- }
- return expnsize;
-}
-
-/* WR with IV, KEY and CPL SFO added */
-static void make_tlstx_data_wr(struct sock *sk, struct sk_buff *skb,
- int tls_tx_imm, int tls_len, u32 credits)
-{
- unsigned short pdus_per_ulp = 0;
- struct chtls_sock *csk;
- struct chtls_hws *hws;
- int expn_sz;
- int pdus;
-
- csk = rcu_dereference_sk_user_data(sk);
- hws = &csk->tlshws;
- pdus = DIV_ROUND_UP(tls_len, hws->mfs);
- expn_sz = chtls_expansion_size(sk, tls_len, 0, NULL);
- if (!hws->compute) {
- hws->expansion = chtls_expansion_size(sk,
- hws->fcplenmax,
- 1, &pdus_per_ulp);
- hws->pdus = pdus_per_ulp;
- hws->adjustlen = hws->pdus *
- ((hws->expansion / hws->pdus) + hws->mfs);
- hws->compute = 1;
- }
- if (tls_copy_ivs(sk, skb))
- return;
- tls_copy_tx_key(sk, skb);
- tls_tx_data_wr(sk, skb, tls_len, tls_tx_imm, credits, expn_sz, pdus);
- hws->tx_seq_no += (pdus - 1);
-}
-
-static void make_tx_data_wr(struct sock *sk, struct sk_buff *skb,
- unsigned int immdlen, int len,
- u32 credits, u32 compl)
-{
- struct fw_ofld_tx_data_wr *req;
- unsigned int wr_ulp_mode_force;
- struct chtls_sock *csk;
- unsigned int opcode;
-
- csk = rcu_dereference_sk_user_data(sk);
- opcode = FW_OFLD_TX_DATA_WR;
-
- req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
- req->op_to_immdlen = htonl(WR_OP_V(opcode) |
- FW_WR_COMPL_V(compl) |
- FW_WR_IMMDLEN_V(immdlen));
- req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
- FW_WR_LEN16_V(credits));
-
- wr_ulp_mode_force = TX_ULP_MODE_V(csk->ulp_mode);
- if (is_sg_request(skb))
- wr_ulp_mode_force |= FW_OFLD_TX_DATA_WR_ALIGNPLD_F |
- ((tcp_sk(sk)->nonagle & TCP_NAGLE_OFF) ? 0 :
- FW_OFLD_TX_DATA_WR_SHOVE_F);
-
- req->tunnel_to_proxy = htonl(wr_ulp_mode_force |
- TX_URG_V(skb_urgent(skb)) |
- TX_SHOVE_V((!csk_flag(sk, CSK_TX_MORE_DATA)) &&
- skb_queue_empty(&csk->txq)));
- req->plen = htonl(len);
-}
-
-static int chtls_wr_size(struct chtls_sock *csk, const struct sk_buff *skb,
- bool size)
-{
- int wr_size;
-
- wr_size = TLS_WR_CPL_LEN;
- wr_size += KEY_ON_MEM_SZ;
- wr_size += ivs_size(csk->sk, skb);
-
- if (size)
- return wr_size;
-
- /* frags counted for IV dsgl */
- if (!skb_ulp_tls_iv_imm(skb))
- skb_shinfo(skb)->nr_frags++;
-
- return wr_size;
-}
-
-static bool is_ofld_imm(struct chtls_sock *csk, const struct sk_buff *skb)
-{
- int length = skb->len;
-
- if (skb->peeked || skb->len > MAX_IMM_ULPTX_WR_LEN)
- return false;
-
- if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
- /* Check TLS header len for Immediate */
- if (csk->ulp_mode == ULP_MODE_TLS &&
- skb_ulp_tls_inline(skb))
- length += chtls_wr_size(csk, skb, true);
- else
- length += sizeof(struct fw_ofld_tx_data_wr);
-
- return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
- }
- return true;
-}
-
-static unsigned int calc_tx_flits(const struct sk_buff *skb,
- unsigned int immdlen)
-{
- unsigned int flits, cnt;
-
- flits = immdlen / 8; /* headers */
- cnt = skb_shinfo(skb)->nr_frags;
- if (skb_tail_pointer(skb) != skb_transport_header(skb))
- cnt++;
- return flits + sgl_len(cnt);
-}
-
-static void arp_failure_discard(void *handle, struct sk_buff *skb)
-{
- kfree_skb(skb);
-}
-
-int chtls_push_frames(struct chtls_sock *csk, int comp)
-{
- struct chtls_hws *hws = &csk->tlshws;
- struct tcp_sock *tp;
- struct sk_buff *skb;
- int total_size = 0;
- struct sock *sk;
- int wr_size;
-
- wr_size = sizeof(struct fw_ofld_tx_data_wr);
- sk = csk->sk;
- tp = tcp_sk(sk);
-
- if (unlikely(sk_in_state(sk, TCPF_SYN_SENT | TCPF_CLOSE)))
- return 0;
-
- if (unlikely(csk_flag(sk, CSK_ABORT_SHUTDOWN)))
- return 0;
-
- while (csk->wr_credits && (skb = skb_peek(&csk->txq)) &&
- (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_HOLD) ||
- skb_queue_len(&csk->txq) > 1)) {
- unsigned int credit_len = skb->len;
- unsigned int credits_needed;
- unsigned int completion = 0;
- int tls_len = skb->len;/* TLS data len before IV/key */
- unsigned int immdlen;
- int len = skb->len; /* length [ulp bytes] inserted by hw */
- int flowclen16 = 0;
- int tls_tx_imm = 0;
-
- immdlen = skb->len;
- if (!is_ofld_imm(csk, skb)) {
- immdlen = skb_transport_offset(skb);
- if (skb_ulp_tls_inline(skb))
- wr_size = chtls_wr_size(csk, skb, false);
- credit_len = 8 * calc_tx_flits(skb, immdlen);
- } else {
- if (skb_ulp_tls_inline(skb)) {
- wr_size = chtls_wr_size(csk, skb, false);
- tls_tx_imm = 1;
- }
- }
- if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR))
- credit_len += wr_size;
- credits_needed = DIV_ROUND_UP(credit_len, 16);
- if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
- flowclen16 = send_tx_flowc_wr(sk, 1, tp->snd_nxt,
- tp->rcv_nxt);
- if (flowclen16 <= 0)
- break;
- csk->wr_credits -= flowclen16;
- csk->wr_unacked += flowclen16;
- csk->wr_nondata += flowclen16;
- csk_set_flag(csk, CSK_TX_DATA_SENT);
- }
-
- if (csk->wr_credits < credits_needed) {
- if (skb_ulp_tls_inline(skb) &&
- !skb_ulp_tls_iv_imm(skb))
- skb_shinfo(skb)->nr_frags--;
- break;
- }
-
- __skb_unlink(skb, &csk->txq);
- skb_set_queue_mapping(skb, (csk->txq_idx << 1) |
- CPL_PRIORITY_DATA);
- if (hws->ofld)
- hws->txqid = (skb->queue_mapping >> 1);
- skb->csum = (__force __wsum)(credits_needed + csk->wr_nondata);
- csk->wr_credits -= credits_needed;
- csk->wr_unacked += credits_needed;
- csk->wr_nondata = 0;
- enqueue_wr(csk, skb);
-
- if (likely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NEED_HDR)) {
- if ((comp && csk->wr_unacked == credits_needed) ||
- (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) ||
- csk->wr_unacked >= csk->wr_max_credits / 2) {
- completion = 1;
- csk->wr_unacked = 0;
- }
- if (skb_ulp_tls_inline(skb))
- make_tlstx_data_wr(sk, skb, tls_tx_imm,
- tls_len, credits_needed);
- else
- make_tx_data_wr(sk, skb, immdlen, len,
- credits_needed, completion);
- tp->snd_nxt += len;
- tp->lsndtime = tcp_jiffies32;
- if (completion)
- ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR;
- } else {
- struct cpl_close_con_req *req = cplhdr(skb);
- unsigned int cmd = CPL_OPCODE_G(ntohl
- (OPCODE_TID(req)));
-
- if (cmd == CPL_CLOSE_CON_REQ)
- csk_set_flag(csk,
- CSK_CLOSE_CON_REQUESTED);
-
- if ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_COMPL) &&
- (csk->wr_unacked >= csk->wr_max_credits / 2)) {
- req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
- csk->wr_unacked = 0;
- }
- }
- total_size += skb->truesize;
- if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_BARRIER)
- csk_set_flag(csk, CSK_TX_WAIT_IDLE);
- t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
- cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
- }
- sk->sk_wmem_queued -= total_size;
- return total_size;
-}
-
-static void mark_urg(struct tcp_sock *tp, int flags,
- struct sk_buff *skb)
-{
- if (unlikely(flags & MSG_OOB)) {
- tp->snd_up = tp->write_seq;
- ULP_SKB_CB(skb)->flags = ULPCB_FLAG_URG |
- ULPCB_FLAG_BARRIER |
- ULPCB_FLAG_NO_APPEND |
- ULPCB_FLAG_NEED_HDR;
- }
-}
-
-/*
- * Returns true if a connection should send more data to TCP engine
- */
-static bool should_push(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_dev *cdev = csk->cdev;
- struct tcp_sock *tp = tcp_sk(sk);
-
- /*
- * If we've released our offload resources there's nothing to do ...
- */
- if (!cdev)
- return false;
-
- /*
- * If there aren't any work requests in flight, or there isn't enough
- * data in flight, or Nagle is off then send the current TX_DATA
- * otherwise hold it and wait to accumulate more data.
- */
- return csk->wr_credits == csk->wr_max_credits ||
- (tp->nonagle & TCP_NAGLE_OFF);
-}
-
-/*
- * Returns true if a TCP socket is corked.
- */
-static bool corked(const struct tcp_sock *tp, int flags)
-{
- return (flags & MSG_MORE) || (tp->nonagle & TCP_NAGLE_CORK);
-}
-
-/*
- * Returns true if a send should try to push new data.
- */
-static bool send_should_push(struct sock *sk, int flags)
-{
- return should_push(sk) && !corked(tcp_sk(sk), flags);
-}
-
-void chtls_tcp_push(struct sock *sk, int flags)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- int qlen = skb_queue_len(&csk->txq);
-
- if (likely(qlen)) {
- struct sk_buff *skb = skb_peek_tail(&csk->txq);
- struct tcp_sock *tp = tcp_sk(sk);
-
- mark_urg(tp, flags, skb);
-
- if (!(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) &&
- corked(tp, flags)) {
- ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_HOLD;
- return;
- }
-
- ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_HOLD;
- if (qlen == 1 &&
- ((ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
- should_push(sk)))
- chtls_push_frames(csk, 1);
- }
-}
-
-/*
- * Calculate the size for a new send sk_buff. It's maximum size so we can
- * pack lots of data into it, unless we plan to send it immediately, in which
- * case we size it more tightly.
- *
- * Note: we don't bother compensating for MSS < PAGE_SIZE because it doesn't
- * arise in normal cases and when it does we are just wasting memory.
- */
-static int select_size(struct sock *sk, int io_len, int flags, int len)
-{
- const int pgbreak = SKB_MAX_HEAD(len);
-
- /*
- * If the data wouldn't fit in the main body anyway, put only the
- * header in the main body so it can use immediate data and place all
- * the payload in page fragments.
- */
- if (io_len > pgbreak)
- return 0;
-
- /*
- * If we will be accumulating payload get a large main body.
- */
- if (!send_should_push(sk, flags))
- return pgbreak;
-
- return io_len;
-}
-
-void skb_entail(struct sock *sk, struct sk_buff *skb, int flags)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct tcp_sock *tp = tcp_sk(sk);
-
- ULP_SKB_CB(skb)->seq = tp->write_seq;
- ULP_SKB_CB(skb)->flags = flags;
- __skb_queue_tail(&csk->txq, skb);
- sk->sk_wmem_queued += skb->truesize;
-
- if (TCP_PAGE(sk) && TCP_OFF(sk)) {
- put_page(TCP_PAGE(sk));
- TCP_PAGE(sk) = NULL;
- TCP_OFF(sk) = 0;
- }
-}
-
-static struct sk_buff *get_tx_skb(struct sock *sk, int size)
-{
- struct sk_buff *skb;
-
- skb = alloc_skb(size + TX_HEADER_LEN, sk->sk_allocation);
- if (likely(skb)) {
- skb_reserve(skb, TX_HEADER_LEN);
- skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
- skb_reset_transport_header(skb);
- }
- return skb;
-}
-
-static struct sk_buff *get_record_skb(struct sock *sk, int size, bool zcopy)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct sk_buff *skb;
-
- skb = alloc_skb(((zcopy ? 0 : size) + TX_TLSHDR_LEN +
- KEY_ON_MEM_SZ + max_ivs_size(sk, size)),
- sk->sk_allocation);
- if (likely(skb)) {
- skb_reserve(skb, (TX_TLSHDR_LEN +
- KEY_ON_MEM_SZ + max_ivs_size(sk, size)));
- skb_entail(sk, skb, ULPCB_FLAG_NEED_HDR);
- skb_reset_transport_header(skb);
- ULP_SKB_CB(skb)->ulp.tls.ofld = 1;
- ULP_SKB_CB(skb)->ulp.tls.type = csk->tlshws.type;
- }
- return skb;
-}
-
-static void tx_skb_finalize(struct sk_buff *skb)
-{
- struct ulp_skb_cb *cb = ULP_SKB_CB(skb);
-
- if (!(cb->flags & ULPCB_FLAG_NO_HDR))
- cb->flags = ULPCB_FLAG_NEED_HDR;
- cb->flags |= ULPCB_FLAG_NO_APPEND;
-}
-
-static void push_frames_if_head(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
-
- if (skb_queue_len(&csk->txq) == 1)
- chtls_push_frames(csk, 1);
-}
-
-static int chtls_skb_copy_to_page_nocache(struct sock *sk,
- struct iov_iter *from,
- struct sk_buff *skb,
- struct page *page,
- int off, int copy)
-{
- int err;
-
- err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) +
- off, copy, skb->len);
- if (err)
- return err;
-
- skb->len += copy;
- skb->data_len += copy;
- skb->truesize += copy;
- sk->sk_wmem_queued += copy;
- return 0;
-}
-
-static bool csk_mem_free(struct chtls_dev *cdev, struct sock *sk)
-{
- return (cdev->max_host_sndbuf - sk->sk_wmem_queued > 0);
-}
-
-static int csk_wait_memory(struct chtls_dev *cdev,
- struct sock *sk, long *timeo_p)
-{
- DEFINE_WAIT_FUNC(wait, woken_wake_function);
- int ret, err = 0;
- long current_timeo;
- long vm_wait = 0;
- bool noblock;
-
- current_timeo = *timeo_p;
- noblock = (*timeo_p ? false : true);
- if (csk_mem_free(cdev, sk)) {
- current_timeo = get_random_u32_below(HZ / 5) + 2;
- vm_wait = get_random_u32_below(HZ / 5) + 2;
- }
-
- add_wait_queue(sk_sleep(sk), &wait);
- while (1) {
- sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
-
- if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
- goto do_error;
- if (!*timeo_p) {
- if (noblock)
- set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
- goto do_nonblock;
- }
- if (signal_pending(current))
- goto do_interrupted;
- sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
- if (csk_mem_free(cdev, sk) && !vm_wait)
- break;
-
- set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
- sk->sk_write_pending++;
- ret = sk_wait_event(sk, ¤t_timeo, sk->sk_err ||
- (sk->sk_shutdown & SEND_SHUTDOWN) ||
- (csk_mem_free(cdev, sk) && !vm_wait),
- &wait);
- sk->sk_write_pending--;
- if (ret < 0)
- goto do_error;
-
- if (vm_wait) {
- vm_wait -= current_timeo;
- current_timeo = *timeo_p;
- if (current_timeo != MAX_SCHEDULE_TIMEOUT) {
- current_timeo -= vm_wait;
- if (current_timeo < 0)
- current_timeo = 0;
- }
- vm_wait = 0;
- }
- *timeo_p = current_timeo;
- }
-do_rm_wq:
- remove_wait_queue(sk_sleep(sk), &wait);
- return err;
-do_error:
- err = -EPIPE;
- goto do_rm_wq;
-do_nonblock:
- err = -EAGAIN;
- goto do_rm_wq;
-do_interrupted:
- err = sock_intr_errno(*timeo_p);
- goto do_rm_wq;
-}
-
-static int chtls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
- unsigned char *record_type)
-{
- struct cmsghdr *cmsg;
- int rc = -EINVAL;
-
- for_each_cmsghdr(cmsg, msg) {
- if (!CMSG_OK(msg, cmsg))
- return -EINVAL;
- if (cmsg->cmsg_level != SOL_TLS)
- continue;
-
- switch (cmsg->cmsg_type) {
- case TLS_SET_RECORD_TYPE:
- if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
- return -EINVAL;
-
- if (msg->msg_flags & MSG_MORE)
- return -EINVAL;
-
- *record_type = *(unsigned char *)CMSG_DATA(cmsg);
- rc = 0;
- break;
- default:
- return -EINVAL;
- }
- }
-
- return rc;
-}
-
-int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_dev *cdev = csk->cdev;
- struct tcp_sock *tp = tcp_sk(sk);
- struct sk_buff *skb;
- int mss, flags, err;
- int recordsz = 0;
- int copied = 0;
- long timeo;
-
- lock_sock(sk);
- flags = msg->msg_flags;
- timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
-
- if (!sk_in_state(sk, TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
- err = sk_stream_wait_connect(sk, &timeo);
- if (err)
- goto out_err;
- }
-
- sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
- err = -EPIPE;
- if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
- goto out_err;
-
- mss = csk->mss;
- csk_set_flag(csk, CSK_TX_MORE_DATA);
-
- while (msg_data_left(msg)) {
- int copy = 0;
-
- skb = skb_peek_tail(&csk->txq);
- if (skb) {
- copy = mss - skb->len;
- skb->ip_summed = CHECKSUM_UNNECESSARY;
- }
- if (!csk_mem_free(cdev, sk))
- goto wait_for_sndbuf;
-
- if (is_tls_tx(csk) && !csk->tlshws.txleft) {
- unsigned char record_type = TLS_RECORD_TYPE_DATA;
-
- if (unlikely(msg->msg_controllen)) {
- err = chtls_proccess_cmsg(sk, msg,
- &record_type);
- if (err)
- goto out_err;
-
- /* Avoid appending tls handshake, alert to tls data */
- if (skb)
- tx_skb_finalize(skb);
- }
-
- recordsz = size;
- csk->tlshws.txleft = recordsz;
- csk->tlshws.type = record_type;
- }
-
- if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
- copy <= 0) {
-new_buf:
- if (skb) {
- tx_skb_finalize(skb);
- push_frames_if_head(sk);
- }
-
- if (is_tls_tx(csk)) {
- skb = get_record_skb(sk,
- select_size(sk,
- recordsz,
- flags,
- TX_TLSHDR_LEN),
- false);
- } else {
- skb = get_tx_skb(sk,
- select_size(sk, size, flags,
- TX_HEADER_LEN));
- }
- if (unlikely(!skb))
- goto wait_for_memory;
-
- skb->ip_summed = CHECKSUM_UNNECESSARY;
- copy = mss;
- }
- if (copy > size)
- copy = size;
-
- if (msg->msg_flags & MSG_SPLICE_PAGES) {
- err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
- if (err < 0) {
- if (err == -EMSGSIZE)
- goto new_buf;
- goto do_fault;
- }
- copy = err;
- sk_wmem_queued_add(sk, copy);
- } else if (skb_tailroom(skb) > 0) {
- copy = min(copy, skb_tailroom(skb));
- if (is_tls_tx(csk))
- copy = min_t(int, copy, csk->tlshws.txleft);
- err = skb_add_data_nocache(sk, skb,
- &msg->msg_iter, copy);
- if (err)
- goto do_fault;
- } else {
- int i = skb_shinfo(skb)->nr_frags;
- struct page *page = TCP_PAGE(sk);
- int pg_size = PAGE_SIZE;
- int off = TCP_OFF(sk);
- bool merge;
-
- if (page)
- pg_size = page_size(page);
- if (off < pg_size &&
- skb_can_coalesce(skb, i, page, off)) {
- merge = true;
- goto copy;
- }
- merge = false;
- if (i == (is_tls_tx(csk) ? (MAX_SKB_FRAGS - 1) :
- MAX_SKB_FRAGS))
- goto new_buf;
-
- if (page && off == pg_size) {
- put_page(page);
- TCP_PAGE(sk) = page = NULL;
- pg_size = PAGE_SIZE;
- }
-
- if (!page) {
- gfp_t gfp = sk->sk_allocation;
- int order = cdev->send_page_order;
-
- if (order) {
- page = alloc_pages(gfp | __GFP_COMP |
- __GFP_NOWARN |
- __GFP_NORETRY,
- order);
- if (page)
- pg_size <<= order;
- }
- if (!page) {
- page = alloc_page(gfp);
- pg_size = PAGE_SIZE;
- }
- if (!page)
- goto wait_for_memory;
- off = 0;
- }
-copy:
- if (copy > pg_size - off)
- copy = pg_size - off;
- if (is_tls_tx(csk))
- copy = min_t(int, copy, csk->tlshws.txleft);
-
- err = chtls_skb_copy_to_page_nocache(sk, &msg->msg_iter,
- skb, page,
- off, copy);
- if (unlikely(err)) {
- if (!TCP_PAGE(sk)) {
- TCP_PAGE(sk) = page;
- TCP_OFF(sk) = 0;
- }
- goto do_fault;
- }
- /* Update the skb. */
- if (merge) {
- skb_frag_size_add(
- &skb_shinfo(skb)->frags[i - 1],
- copy);
- } else {
- skb_fill_page_desc(skb, i, page, off, copy);
- if (off + copy < pg_size) {
- /* space left keep page */
- get_page(page);
- TCP_PAGE(sk) = page;
- } else {
- TCP_PAGE(sk) = NULL;
- }
- }
- TCP_OFF(sk) = off + copy;
- }
- if (unlikely(skb->len == mss))
- tx_skb_finalize(skb);
- tp->write_seq += copy;
- copied += copy;
- size -= copy;
-
- if (is_tls_tx(csk))
- csk->tlshws.txleft -= copy;
-
- if (corked(tp, flags) &&
- (sk_stream_wspace(sk) < sk_stream_min_wspace(sk)))
- ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_NO_APPEND;
-
- if (size == 0)
- goto out;
-
- if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)
- push_frames_if_head(sk);
- continue;
-wait_for_sndbuf:
- set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
-wait_for_memory:
- err = csk_wait_memory(cdev, sk, &timeo);
- if (err)
- goto do_error;
- }
-out:
- csk_reset_flag(csk, CSK_TX_MORE_DATA);
- if (copied)
- chtls_tcp_push(sk, flags);
-done:
- release_sock(sk);
- return copied;
-do_fault:
- if (!skb->len) {
- __skb_unlink(skb, &csk->txq);
- sk->sk_wmem_queued -= skb->truesize;
- __kfree_skb(skb);
- }
-do_error:
- if (copied)
- goto out;
-out_err:
- if (csk_conn_inline(csk))
- csk_reset_flag(csk, CSK_TX_MORE_DATA);
- copied = sk_stream_error(sk, flags, err);
- goto done;
-}
-
-void chtls_splice_eof(struct socket *sock)
-{
- struct sock *sk = sock->sk;
-
- lock_sock(sk);
- chtls_tcp_push(sk, 0);
- release_sock(sk);
-}
-
-static void chtls_select_window(struct sock *sk)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct tcp_sock *tp = tcp_sk(sk);
- unsigned int wnd = tp->rcv_wnd;
-
- wnd = max_t(unsigned int, wnd, tcp_full_space(sk));
- wnd = max_t(unsigned int, MIN_RCV_WND, wnd);
-
- if (wnd > MAX_RCV_WND)
- wnd = MAX_RCV_WND;
-
-/*
- * Check if we need to grow the receive window in response to an increase in
- * the socket's receive buffer size. Some applications increase the buffer
- * size dynamically and rely on the window to grow accordingly.
- */
-
- if (wnd > tp->rcv_wnd) {
- tp->rcv_wup -= wnd - tp->rcv_wnd;
- tp->rcv_wnd = wnd;
- /* Mark the receive window as updated */
- csk_reset_flag(csk, CSK_UPDATE_RCV_WND);
- }
-}
-
-/*
- * Send RX credits through an RX_DATA_ACK CPL message. We are permitted
- * to return without sending the message in case we cannot allocate
- * an sk_buff. Returns the number of credits sent.
- */
-static u32 send_rx_credits(struct chtls_sock *csk, u32 credits)
-{
- struct cpl_rx_data_ack *req;
- struct sk_buff *skb;
-
- skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
- if (!skb)
- return 0;
- __skb_put(skb, sizeof(*req));
- req = (struct cpl_rx_data_ack *)skb->head;
-
- set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
- INIT_TP_WR(req, csk->tid);
- OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
- csk->tid));
- req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) |
- RX_FORCE_ACK_F);
- cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
- return credits;
-}
-
-#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | \
- TCPF_FIN_WAIT1 | \
- TCPF_FIN_WAIT2)
-
-/*
- * Called after some received data has been read. It returns RX credits
- * to the HW for the amount of data processed.
- */
-static void chtls_cleanup_rbuf(struct sock *sk, int copied)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct tcp_sock *tp;
- int must_send;
- u32 credits;
- u32 thres;
-
- thres = 15 * 1024;
-
- if (!sk_in_state(sk, CREDIT_RETURN_STATE))
- return;
-
- chtls_select_window(sk);
- tp = tcp_sk(sk);
- credits = tp->copied_seq - tp->rcv_wup;
- if (unlikely(!credits))
- return;
-
-/*
- * For coalescing to work effectively ensure the receive window has
- * at least 16KB left.
- */
- must_send = credits + 16384 >= tp->rcv_wnd;
-
- if (must_send || credits >= thres)
- tp->rcv_wup += send_rx_credits(csk, credits);
-}
-
-static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
- int flags)
-{
- struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
- struct chtls_hws *hws = &csk->tlshws;
- struct net_device *dev = csk->egress_dev;
- struct adapter *adap = netdev2adap(dev);
- struct tcp_sock *tp = tcp_sk(sk);
- unsigned long avail;
- int buffers_freed;
- int copied = 0;
- int target;
- long timeo;
- int ret;
-
- buffers_freed = 0;
-
- timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
- target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
-
- if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND)))
- chtls_cleanup_rbuf(sk, copied);
-
- do {
- struct sk_buff *skb;
- u32 offset = 0;
-
- if (unlikely(tp->urg_data &&
- tp->urg_seq == tp->copied_seq)) {
- if (copied)
- break;
- if (signal_pending(current)) {
- copied = timeo ? sock_intr_errno(timeo) :
- -EAGAIN;
- break;
- }
- }
- skb = skb_peek(&sk->sk_receive_queue);
- if (skb)
- goto found_ok_skb;
- if (csk->wr_credits &&
- skb_queue_len(&csk->txq) &&
- chtls_push_frames(csk, csk->wr_credits ==
- csk->wr_max_credits))
- sk->sk_write_space(sk);
-
- if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
- break;
-
- if (copied) {
- if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
- (sk->sk_shutdown & RCV_SHUTDOWN) ||
- signal_pending(current))
- break;
-
- if (!timeo)
- break;
- } else {
- if (sock_flag(sk, SOCK_DONE))
- break;
- if (sk->sk_err) {
- copied = sock_error(sk);
- break;
- }
- if (sk->sk_shutdown & RCV_SHUTDOWN)
- break;
- if (sk->sk_state == TCP_CLOSE) {
- copied = -ENOTCONN;
- break;
- }
- if (!timeo) {
- copied = -EAGAIN;
- break;
- }
- if (signal_pending(current)) {
- copied = sock_intr_errno(timeo);
- break;
- }
- }
- if (READ_ONCE(sk->sk_backlog.tail)) {
- release_sock(sk);
- lock_sock(sk);
- chtls_cleanup_rbuf(sk, copied);
- continue;
- }
-
- if (copied >= target)
- break;
- chtls_cleanup_rbuf(sk, copied);
- ret = sk_wait_data(sk, &timeo, NULL);
- if (ret < 0) {
- copied = copied ? : ret;
- goto unlock;
- }
- continue;
-found_ok_skb:
- if (!skb->len) {
- skb_dstref_steal(skb);
- __skb_unlink(skb, &sk->sk_receive_queue);
- kfree_skb(skb);
-
- if (!copied && !timeo) {
- copied = -EAGAIN;
- break;
- }
-
- if (copied < target) {
- release_sock(sk);
- lock_sock(sk);
- continue;
- }
- break;
- }
- offset = hws->copied_seq;
- avail = skb->len - offset;
- if (len < avail)
- avail = len;
-
- if (unlikely(tp->urg_data)) {
- u32 urg_offset = tp->urg_seq - tp->copied_seq;
-
- if (urg_offset < avail) {
- if (urg_offset) {
- avail = urg_offset;
- } else if (!sock_flag(sk, SOCK_URGINLINE)) {
- /* First byte is urgent, skip */
- tp->copied_seq++;
- offset++;
- avail--;
- if (!avail)
- goto skip_copy;
- }
- }
- }
- /* Set record type if not already done. For a non-data record,
- * do not proceed if record type could not be copied.
- */
- if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
- struct tls_hdr *thdr = (struct tls_hdr *)skb->data;
- int cerr = 0;
-
- cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
- sizeof(thdr->type), &thdr->type);
-
- if (cerr && thdr->type != TLS_RECORD_TYPE_DATA) {
- copied = -EIO;
- break;
- }
- /* don't send tls header, skip copy */
- goto skip_copy;
- }
-
- if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
- if (!copied) {
- copied = -EFAULT;
- break;
- }
- }
-
- copied += avail;
- len -= avail;
- hws->copied_seq += avail;
-skip_copy:
- if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
- tp->urg_data = 0;
-
- if ((avail + offset) >= skb->len) {
- struct sk_buff *next_skb;
- if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
- tp->copied_seq += skb->len;
- hws->rcvpld = skb->hdr_len;
- } else {
- atomic_inc(&adap->chcr_stats.tls_pdu_rx);
- tp->copied_seq += hws->rcvpld;
- }
- chtls_free_skb(sk, skb);
- buffers_freed++;
- hws->copied_seq = 0;
- next_skb = skb_peek(&sk->sk_receive_queue);
- if (copied >= target && !next_skb)
- break;
- if (ULP_SKB_CB(next_skb)->flags & ULPCB_FLAG_TLS_HDR)
- break;
- }
- } while (len > 0);
-
- if (buffers_freed)
- chtls_cleanup_rbuf(sk, copied);
-
-unlock:
- release_sock(sk);
- return copied;
-}
-
-/*
- * Peek at data in a socket's receive buffer.
- */
-static int peekmsg(struct sock *sk, struct msghdr *msg,
- size_t len, int flags)
-{
- struct tcp_sock *tp = tcp_sk(sk);
- u32 peek_seq, offset;
- struct sk_buff *skb;
- int copied = 0;
- size_t avail; /* amount of available data in current skb */
- long timeo;
- int ret;
-
- lock_sock(sk);
- timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
- peek_seq = tp->copied_seq;
-
- do {
- if (unlikely(tp->urg_data && tp->urg_seq == peek_seq)) {
- if (copied)
- break;
- if (signal_pending(current)) {
- copied = timeo ? sock_intr_errno(timeo) :
- -EAGAIN;
- break;
- }
- }
-
- skb_queue_walk(&sk->sk_receive_queue, skb) {
- offset = peek_seq - ULP_SKB_CB(skb)->seq;
- if (offset < skb->len)
- goto found_ok_skb;
- }
-
- /* empty receive queue */
- if (copied)
- break;
- if (sock_flag(sk, SOCK_DONE))
- break;
- if (sk->sk_err) {
- copied = sock_error(sk);
- break;
- }
- if (sk->sk_shutdown & RCV_SHUTDOWN)
- break;
- if (sk->sk_state == TCP_CLOSE) {
- copied = -ENOTCONN;
- break;
- }
- if (!timeo) {
- copied = -EAGAIN;
- break;
- }
- if (signal_pending(current)) {
- copied = sock_intr_errno(timeo);
- break;
- }
-
- if (READ_ONCE(sk->sk_backlog.tail)) {
- /* Do not sleep, just process backlog. */
- release_sock(sk);
- lock_sock(sk);
- } else {
- ret = sk_wait_data(sk, &timeo, NULL);
- if (ret < 0) {
- /* here 'copied' is 0 due to previous checks */
- copied = ret;
- break;
- }
- }
-
- if (unlikely(peek_seq != tp->copied_seq)) {
- if (net_ratelimit())
- pr_info("TCP(%s:%d), race in MSG_PEEK.\n",
- current->comm, current->pid);
- peek_seq = tp->copied_seq;
- }
- continue;
-
-found_ok_skb:
- avail = skb->len - offset;
- if (len < avail)
- avail = len;
- /*
- * Do we have urgent data here? We need to skip over the
- * urgent byte.
- */
- if (unlikely(tp->urg_data)) {
- u32 urg_offset = tp->urg_seq - peek_seq;
-
- if (urg_offset < avail) {
- /*
- * The amount of data we are preparing to copy
- * contains urgent data.
- */
- if (!urg_offset) { /* First byte is urgent */
- if (!sock_flag(sk, SOCK_URGINLINE)) {
- peek_seq++;
- offset++;
- avail--;
- }
- if (!avail)
- continue;
- } else {
- /* stop short of the urgent data */
- avail = urg_offset;
- }
- }
- }
-
- /*
- * If MSG_TRUNC is specified the data is discarded.
- */
- if (likely(!(flags & MSG_TRUNC)))
- if (skb_copy_datagram_msg(skb, offset, msg, len)) {
- if (!copied) {
- copied = -EFAULT;
- break;
- }
- }
- peek_seq += avail;
- copied += avail;
- len -= avail;
- } while (len > 0);
-
- release_sock(sk);
- return copied;
-}
-
-int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
- int flags)
-{
- struct tcp_sock *tp = tcp_sk(sk);
- struct chtls_sock *csk;
- unsigned long avail; /* amount of available data in current skb */
- int buffers_freed;
- int copied = 0;
- long timeo;
- int target; /* Read at least this many bytes */
- int ret;
-
- buffers_freed = 0;
-
- if (unlikely(flags & MSG_OOB))
- return tcp_prot.recvmsg(sk, msg, len, flags);
-
- if (unlikely(flags & MSG_PEEK))
- return peekmsg(sk, msg, len, flags);
-
- if (sk_can_busy_loop(sk) &&
- skb_queue_empty_lockless(&sk->sk_receive_queue) &&
- sk->sk_state == TCP_ESTABLISHED)
- sk_busy_loop(sk, flags & MSG_DONTWAIT);
-
- lock_sock(sk);
- csk = rcu_dereference_sk_user_data(sk);
-
- if (is_tls_rx(csk))
- return chtls_pt_recvmsg(sk, msg, len, flags);
-
- timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
- target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
-
- if (unlikely(csk_flag(sk, CSK_UPDATE_RCV_WND)))
- chtls_cleanup_rbuf(sk, copied);
-
- do {
- struct sk_buff *skb;
- u32 offset;
-
- if (unlikely(tp->urg_data && tp->urg_seq == tp->copied_seq)) {
- if (copied)
- break;
- if (signal_pending(current)) {
- copied = timeo ? sock_intr_errno(timeo) :
- -EAGAIN;
- break;
- }
- }
-
- skb = skb_peek(&sk->sk_receive_queue);
- if (skb)
- goto found_ok_skb;
-
- if (csk->wr_credits &&
- skb_queue_len(&csk->txq) &&
- chtls_push_frames(csk, csk->wr_credits ==
- csk->wr_max_credits))
- sk->sk_write_space(sk);
-
- if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
- break;
-
- if (copied) {
- if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
- (sk->sk_shutdown & RCV_SHUTDOWN) ||
- signal_pending(current))
- break;
- } else {
- if (sock_flag(sk, SOCK_DONE))
- break;
- if (sk->sk_err) {
- copied = sock_error(sk);
- break;
- }
- if (sk->sk_shutdown & RCV_SHUTDOWN)
- break;
- if (sk->sk_state == TCP_CLOSE) {
- copied = -ENOTCONN;
- break;
- }
- if (!timeo) {
- copied = -EAGAIN;
- break;
- }
- if (signal_pending(current)) {
- copied = sock_intr_errno(timeo);
- break;
- }
- }
-
- if (READ_ONCE(sk->sk_backlog.tail)) {
- release_sock(sk);
- lock_sock(sk);
- chtls_cleanup_rbuf(sk, copied);
- continue;
- }
-
- if (copied >= target)
- break;
- chtls_cleanup_rbuf(sk, copied);
- ret = sk_wait_data(sk, &timeo, NULL);
- if (ret < 0) {
- copied = copied ? : ret;
- goto unlock;
- }
- continue;
-
-found_ok_skb:
- if (!skb->len) {
- chtls_kfree_skb(sk, skb);
- if (!copied && !timeo) {
- copied = -EAGAIN;
- break;
- }
-
- if (copied < target)
- continue;
-
- break;
- }
-
- offset = tp->copied_seq - ULP_SKB_CB(skb)->seq;
- avail = skb->len - offset;
- if (len < avail)
- avail = len;
-
- if (unlikely(tp->urg_data)) {
- u32 urg_offset = tp->urg_seq - tp->copied_seq;
-
- if (urg_offset < avail) {
- if (urg_offset) {
- avail = urg_offset;
- } else if (!sock_flag(sk, SOCK_URGINLINE)) {
- tp->copied_seq++;
- offset++;
- avail--;
- if (!avail)
- goto skip_copy;
- }
- }
- }
-
- if (likely(!(flags & MSG_TRUNC))) {
- if (skb_copy_datagram_msg(skb, offset,
- msg, avail)) {
- if (!copied) {
- copied = -EFAULT;
- break;
- }
- }
- }
-
- tp->copied_seq += avail;
- copied += avail;
- len -= avail;
-
-skip_copy:
- if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
- tp->urg_data = 0;
-
- if (avail + offset >= skb->len) {
- chtls_free_skb(sk, skb);
- buffers_freed++;
-
- if (copied >= target &&
- !skb_peek(&sk->sk_receive_queue))
- break;
- }
- } while (len > 0);
-
- if (buffers_freed)
- chtls_cleanup_rbuf(sk, copied);
-
-unlock:
- release_sock(sk);
- return copied;
-}
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
deleted file mode 100644
index 2570575434f9..000000000000
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
+++ /dev/null
@@ -1,642 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2018 Chelsio Communications, Inc.
- *
- * Written by: Atul Gupta (atul.gupta@chelsio.com)
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/skbuff.h>
-#include <linux/socket.h>
-#include <linux/hash.h>
-#include <linux/in.h>
-#include <linux/net.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <net/ipv6.h>
-#include <net/transp_v6.h>
-#include <net/tcp.h>
-#include <net/tls.h>
-
-#include "chtls.h"
-#include "chtls_cm.h"
-
-#define DRV_NAME "chtls"
-
-/*
- * chtls device management
- * maintains a list of the chtls devices
- */
-static LIST_HEAD(cdev_list);
-static DEFINE_MUTEX(cdev_mutex);
-
-static DEFINE_MUTEX(notify_mutex);
-static RAW_NOTIFIER_HEAD(listen_notify_list);
-static struct proto chtls_cpl_prot, chtls_cpl_protv6;
-struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6;
-static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
-
-static void register_listen_notifier(struct notifier_block *nb)
-{
- mutex_lock(¬ify_mutex);
- raw_notifier_chain_register(&listen_notify_list, nb);
- mutex_unlock(¬ify_mutex);
-}
-
-static void unregister_listen_notifier(struct notifier_block *nb)
-{
- mutex_lock(¬ify_mutex);
- raw_notifier_chain_unregister(&listen_notify_list, nb);
- mutex_unlock(¬ify_mutex);
-}
-
-static int listen_notify_handler(struct notifier_block *this,
- unsigned long event, void *data)
-{
- struct chtls_listen *clisten;
- int ret = NOTIFY_DONE;
-
- clisten = (struct chtls_listen *)data;
-
- switch (event) {
- case CHTLS_LISTEN_START:
- ret = chtls_listen_start(clisten->cdev, clisten->sk);
- kfree(clisten);
- break;
- case CHTLS_LISTEN_STOP:
- chtls_listen_stop(clisten->cdev, clisten->sk);
- kfree(clisten);
- break;
- }
- return ret;
-}
-
-static struct notifier_block listen_notifier = {
- .notifier_call = listen_notify_handler
-};
-
-static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
-{
- if (likely(skb_transport_header(skb) != skb_network_header(skb)))
- return tcp_v4_do_rcv(sk, skb);
- BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
- return 0;
-}
-
-static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk)
-{
- struct chtls_listen *clisten;
-
- if (sk->sk_protocol != IPPROTO_TCP)
- return -EPROTONOSUPPORT;
-
- if (sk->sk_family == PF_INET &&
- LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
- return -EADDRNOTAVAIL;
-
- sk->sk_backlog_rcv = listen_backlog_rcv;
- clisten = kmalloc_obj(*clisten);
- if (!clisten)
- return -ENOMEM;
- clisten->cdev = cdev;
- clisten->sk = sk;
- mutex_lock(¬ify_mutex);
- raw_notifier_call_chain(&listen_notify_list,
- CHTLS_LISTEN_START, clisten);
- mutex_unlock(¬ify_mutex);
- return 0;
-}
-
-static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk)
-{
- struct chtls_listen *clisten;
-
- if (sk->sk_protocol != IPPROTO_TCP)
- return;
-
- clisten = kmalloc_obj(*clisten);
- if (!clisten)
- return;
- clisten->cdev = cdev;
- clisten->sk = sk;
- mutex_lock(¬ify_mutex);
- raw_notifier_call_chain(&listen_notify_list,
- CHTLS_LISTEN_STOP, clisten);
- mutex_unlock(¬ify_mutex);
-}
-
-static int chtls_inline_feature(struct tls_toe_device *dev)
-{
- struct net_device *netdev;
- struct chtls_dev *cdev;
- int i;
-
- cdev = to_chtls_dev(dev);
-
- for (i = 0; i < cdev->lldi->nports; i++) {
- netdev = cdev->ports[i];
- if (netdev->features & NETIF_F_HW_TLS_RECORD)
- return 1;
- }
- return 0;
-}
-
-static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk)
-{
- struct chtls_dev *cdev = to_chtls_dev(dev);
-
- if (sk->sk_state == TCP_LISTEN)
- return chtls_start_listen(cdev, sk);
- return 0;
-}
-
-static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk)
-{
- struct chtls_dev *cdev = to_chtls_dev(dev);
-
- if (sk->sk_state == TCP_LISTEN)
- chtls_stop_listen(cdev, sk);
-}
-
-static void chtls_free_uld(struct chtls_dev *cdev)
-{
- int i;
-
- tls_toe_unregister_device(&cdev->tlsdev);
- kvfree(cdev->kmap.addr);
- idr_destroy(&cdev->hwtid_idr);
- for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
- kfree_skb(cdev->rspq_skb_cache[i]);
- kfree(cdev->lldi);
- kfree_skb(cdev->askb);
- kfree(cdev);
-}
-
-static inline void chtls_dev_release(struct kref *kref)
-{
- struct tls_toe_device *dev;
- struct chtls_dev *cdev;
- struct adapter *adap;
-
- dev = container_of(kref, struct tls_toe_device, kref);
- cdev = to_chtls_dev(dev);
-
- /* Reset tls rx/tx stats */
- adap = pci_get_drvdata(cdev->pdev);
- atomic_set(&adap->chcr_stats.tls_pdu_tx, 0);
- atomic_set(&adap->chcr_stats.tls_pdu_rx, 0);
-
- chtls_free_uld(cdev);
-}
-
-static void chtls_register_dev(struct chtls_dev *cdev)
-{
- struct tls_toe_device *tlsdev = &cdev->tlsdev;
-
- strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
- strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
- TLS_TOE_DEVICE_NAME_MAX);
- tlsdev->feature = chtls_inline_feature;
- tlsdev->hash = chtls_create_hash;
- tlsdev->unhash = chtls_destroy_hash;
- tlsdev->release = chtls_dev_release;
- kref_init(&tlsdev->kref);
- tls_toe_register_device(tlsdev);
- cdev->cdev_state = CHTLS_CDEV_STATE_UP;
-}
-
-static void process_deferq(struct work_struct *task_param)
-{
- struct chtls_dev *cdev = container_of(task_param,
- struct chtls_dev, deferq_task);
- struct sk_buff *skb;
-
- spin_lock_bh(&cdev->deferq.lock);
- while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
- spin_unlock_bh(&cdev->deferq.lock);
- DEFERRED_SKB_CB(skb)->handler(cdev, skb);
- spin_lock_bh(&cdev->deferq.lock);
- }
- spin_unlock_bh(&cdev->deferq.lock);
-}
-
-static int chtls_get_skb(struct chtls_dev *cdev)
-{
- cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
- if (!cdev->askb)
- return -ENOMEM;
-
- skb_put(cdev->askb, sizeof(struct tcphdr));
- skb_reset_transport_header(cdev->askb);
- memset(cdev->askb->data, 0, cdev->askb->len);
- return 0;
-}
-
-static void *chtls_uld_add(const struct cxgb4_lld_info *info)
-{
- struct cxgb4_lld_info *lldi;
- struct chtls_dev *cdev;
- int i, j;
-
- cdev = kzalloc_obj(*cdev);
- if (!cdev)
- goto out;
-
- lldi = kzalloc_obj(*lldi);
- if (!lldi)
- goto out_lldi;
-
- if (chtls_get_skb(cdev))
- goto out_skb;
-
- *lldi = *info;
- cdev->lldi = lldi;
- cdev->pdev = lldi->pdev;
- cdev->tids = lldi->tids;
- cdev->ports = lldi->ports;
- cdev->mtus = lldi->mtus;
- cdev->tids = lldi->tids;
- cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
- << FW_VIID_PFN_S;
-
- for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
- unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
-
- cdev->rspq_skb_cache[i] = __alloc_skb(size,
- gfp_any(), 0,
- lldi->nodeid);
- if (unlikely(!cdev->rspq_skb_cache[i]))
- goto out_rspq_skb;
- }
-
- idr_init(&cdev->hwtid_idr);
- INIT_WORK(&cdev->deferq_task, process_deferq);
- spin_lock_init(&cdev->listen_lock);
- spin_lock_init(&cdev->idr_lock);
- cdev->send_page_order = min_t(uint, get_order(32768),
- send_page_order);
- cdev->max_host_sndbuf = 48 * 1024;
-
- if (lldi->vr->key.size)
- if (chtls_init_kmap(cdev, lldi))
- goto out_rspq_skb;
-
- mutex_lock(&cdev_mutex);
- list_add_tail(&cdev->list, &cdev_list);
- mutex_unlock(&cdev_mutex);
-
- return cdev;
-out_rspq_skb:
- for (j = 0; j < i; j++)
- kfree_skb(cdev->rspq_skb_cache[j]);
- kfree_skb(cdev->askb);
-out_skb:
- kfree(lldi);
-out_lldi:
- kfree(cdev);
-out:
- return NULL;
-}
-
-static void chtls_free_all_uld(void)
-{
- struct chtls_dev *cdev, *tmp;
-
- mutex_lock(&cdev_mutex);
- list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
- if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) {
- list_del(&cdev->list);
- kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
- }
- }
- mutex_unlock(&cdev_mutex);
-}
-
-static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
-{
- struct chtls_dev *cdev = handle;
-
- switch (new_state) {
- case CXGB4_STATE_UP:
- chtls_register_dev(cdev);
- break;
- case CXGB4_STATE_DOWN:
- break;
- case CXGB4_STATE_START_RECOVERY:
- break;
- case CXGB4_STATE_DETACH:
- mutex_lock(&cdev_mutex);
- list_del(&cdev->list);
- mutex_unlock(&cdev_mutex);
- kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
- break;
- default:
- break;
- }
- return 0;
-}
-
-static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
- const __be64 *rsp,
- u32 pktshift)
-{
- struct sk_buff *skb;
-
- /* Allocate space for cpl_pass_accept_req which will be synthesized by
- * driver. Once driver synthesizes cpl_pass_accept_req the skb will go
- * through the regular cpl_pass_accept_req processing in TOM.
- */
- skb = alloc_skb(size_add(gl->tot_len,
- sizeof(struct cpl_pass_accept_req)) -
- pktshift, GFP_ATOMIC);
- if (unlikely(!skb))
- return NULL;
- __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
- - pktshift);
- /* For now we will copy cpl_rx_pkt in the skb */
- skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
- skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
- , gl->va + pktshift,
- gl->tot_len - pktshift);
-
- return skb;
-}
-
-static int chtls_recv_packet(struct chtls_dev *cdev,
- const struct pkt_gl *gl, const __be64 *rsp)
-{
- unsigned int opcode = *(u8 *)rsp;
- struct sk_buff *skb;
- int ret;
-
- skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
- if (!skb)
- return -ENOMEM;
-
- ret = chtls_handlers[opcode](cdev, skb);
- if (ret & CPL_RET_BUF_DONE)
- kfree_skb(skb);
-
- return 0;
-}
-
-static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
-{
- unsigned long rspq_bin;
- unsigned int opcode;
- struct sk_buff *skb;
- unsigned int len;
- int ret;
-
- len = 64 - sizeof(struct rsp_ctrl) - 8;
- opcode = *(u8 *)rsp;
-
- rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
- skb = cdev->rspq_skb_cache[rspq_bin];
- if (skb && !skb_is_nonlinear(skb) &&
- !skb_shared(skb) && !skb_cloned(skb)) {
- refcount_inc(&skb->users);
- if (refcount_read(&skb->users) == 2) {
- __skb_trim(skb, 0);
- if (skb_tailroom(skb) >= len)
- goto copy_out;
- }
- refcount_dec(&skb->users);
- }
- skb = alloc_skb(len, GFP_ATOMIC);
- if (unlikely(!skb))
- return -ENOMEM;
-
-copy_out:
- __skb_put(skb, len);
- skb_copy_to_linear_data(skb, rsp, len);
- skb_reset_network_header(skb);
- skb_reset_transport_header(skb);
- ret = chtls_handlers[opcode](cdev, skb);
-
- if (ret & CPL_RET_BUF_DONE)
- kfree_skb(skb);
- return 0;
-}
-
-static void chtls_recv(struct chtls_dev *cdev,
- struct sk_buff **skbs, const __be64 *rsp)
-{
- struct sk_buff *skb = *skbs;
- unsigned int opcode;
- int ret;
-
- opcode = *(u8 *)rsp;
-
- __skb_push(skb, sizeof(struct rss_header));
- skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
-
- ret = chtls_handlers[opcode](cdev, skb);
- if (ret & CPL_RET_BUF_DONE)
- kfree_skb(skb);
-}
-
-static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
- const struct pkt_gl *gl)
-{
- struct chtls_dev *cdev = handle;
- unsigned int opcode;
- struct sk_buff *skb;
-
- opcode = *(u8 *)rsp;
-
- if (unlikely(opcode == CPL_RX_PKT)) {
- if (chtls_recv_packet(cdev, gl, rsp) < 0)
- goto nomem;
- return 0;
- }
-
- if (!gl)
- return chtls_recv_rsp(cdev, rsp);
-
-#define RX_PULL_LEN 128
- skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
- if (unlikely(!skb))
- goto nomem;
- chtls_recv(cdev, &skb, rsp);
- return 0;
-
-nomem:
- return -ENOMEM;
-}
-
-static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
- int __user *optlen)
-{
- struct tls_crypto_info crypto_info = { 0 };
-
- crypto_info.version = TLS_1_2_VERSION;
- if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
- return -EFAULT;
- return 0;
-}
-
-static int chtls_getsockopt(struct sock *sk, int level, int optname,
- char __user *optval, int __user *optlen)
-{
- struct tls_context *ctx = tls_get_ctx(sk);
-
- if (level != SOL_TLS)
- return ctx->sk_proto->getsockopt(sk, level,
- optname, optval, optlen);
-
- return do_chtls_getsockopt(sk, optval, optlen);
-}
-
-static int do_chtls_setsockopt(struct sock *sk, int optname,
- sockptr_t optval, unsigned int optlen)
-{
- struct tls_crypto_info *crypto_info, tmp_crypto_info;
- struct chtls_sock *csk;
- int keylen;
- int cipher_type;
- int rc = 0;
-
- csk = rcu_dereference_sk_user_data(sk);
-
- if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) {
- rc = -EINVAL;
- goto out;
- }
-
- rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info));
- if (rc) {
- rc = -EFAULT;
- goto out;
- }
-
- /* check version */
- if (tmp_crypto_info.version != TLS_1_2_VERSION) {
- rc = -ENOTSUPP;
- goto out;
- }
-
- crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
-
- /* GCM mode of AES supports 128 and 256 bit encryption, so
- * copy keys from user based on GCM cipher type.
- */
- switch (tmp_crypto_info.cipher_type) {
- case TLS_CIPHER_AES_GCM_128: {
- /* Obtain version and type from previous copy */
- crypto_info[0] = tmp_crypto_info;
- /* Now copy the following data */
- rc = copy_from_sockptr_offset((char *)crypto_info +
- sizeof(*crypto_info),
- optval, sizeof(*crypto_info),
- sizeof(struct tls12_crypto_info_aes_gcm_128)
- - sizeof(*crypto_info));
-
- if (rc) {
- rc = -EFAULT;
- goto out;
- }
-
- keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
- cipher_type = TLS_CIPHER_AES_GCM_128;
- break;
- }
- case TLS_CIPHER_AES_GCM_256: {
- crypto_info[0] = tmp_crypto_info;
- rc = copy_from_sockptr_offset((char *)crypto_info +
- sizeof(*crypto_info),
- optval, sizeof(*crypto_info),
- sizeof(struct tls12_crypto_info_aes_gcm_256)
- - sizeof(*crypto_info));
-
- if (rc) {
- rc = -EFAULT;
- goto out;
- }
-
- keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
- cipher_type = TLS_CIPHER_AES_GCM_256;
- break;
- }
- default:
- rc = -EINVAL;
- goto out;
- }
- rc = chtls_setkey(csk, keylen, optname, cipher_type);
-out:
- return rc;
-}
-
-static int chtls_setsockopt(struct sock *sk, int level, int optname,
- sockptr_t optval, unsigned int optlen)
-{
- struct tls_context *ctx = tls_get_ctx(sk);
-
- if (level != SOL_TLS)
- return ctx->sk_proto->setsockopt(sk, level,
- optname, optval, optlen);
-
- return do_chtls_setsockopt(sk, optname, optval, optlen);
-}
-
-static struct cxgb4_uld_info chtls_uld_info = {
- .name = DRV_NAME,
- .nrxq = MAX_ULD_QSETS,
- .ntxq = MAX_ULD_QSETS,
- .rxq_size = 1024,
- .add = chtls_uld_add,
- .state_change = chtls_uld_state_change,
- .rx_handler = chtls_uld_rx_handler,
-};
-
-void chtls_install_cpl_ops(struct sock *sk)
-{
- if (sk->sk_family == AF_INET)
- sk->sk_prot = &chtls_cpl_prot;
- else
- sk->sk_prot = &chtls_cpl_protv6;
-}
-
-static void __init chtls_init_ulp_ops(void)
-{
- chtls_cpl_prot = tcp_prot;
- chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
- &tcp_prot, PF_INET);
- chtls_cpl_prot.close = chtls_close;
- chtls_cpl_prot.disconnect = chtls_disconnect;
- chtls_cpl_prot.destroy = chtls_destroy_sock;
- chtls_cpl_prot.shutdown = chtls_shutdown;
- chtls_cpl_prot.sendmsg = chtls_sendmsg;
- chtls_cpl_prot.splice_eof = chtls_splice_eof;
- chtls_cpl_prot.recvmsg = chtls_recvmsg;
- chtls_cpl_prot.setsockopt = chtls_setsockopt;
- chtls_cpl_prot.getsockopt = chtls_getsockopt;
-#if IS_ENABLED(CONFIG_IPV6)
- chtls_cpl_protv6 = chtls_cpl_prot;
- chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
- &tcpv6_prot, PF_INET6);
-#endif
-}
-
-static int __init chtls_register(void)
-{
- chtls_init_ulp_ops();
- register_listen_notifier(&listen_notifier);
- cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
- return 0;
-}
-
-static void __exit chtls_unregister(void)
-{
- unregister_listen_notifier(&listen_notifier);
- chtls_free_all_uld();
- cxgb4_unregister_uld(CXGB4_ULD_TLS);
-}
-
-module_init(chtls_register);
-module_exit(chtls_unregister);
-
-MODULE_DESCRIPTION("Chelsio TLS Inline driver");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Chelsio Communications");
-MODULE_VERSION(CHTLS_DRV_VERSION);
diff --git a/include/net/tls.h b/include/net/tls.h
index 3811943288b3..e57bef58851e 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -85,7 +85,6 @@ enum {
TLS_BASE,
TLS_SW,
TLS_HW,
- TLS_HW_RECORD,
TLS_NUM_CONFIG,
};
diff --git a/include/net/tls_toe.h b/include/net/tls_toe.h
deleted file mode 100644
index b3aa7593ce2c..000000000000
--- a/include/net/tls_toe.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
- * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/kref.h>
-#include <linux/list.h>
-
-struct sock;
-
-#define TLS_TOE_DEVICE_NAME_MAX 32
-
-/*
- * This structure defines the routines for Inline TLS driver.
- * The following routines are optional and filled with a
- * null pointer if not defined.
- *
- * @name: Its the name of registered Inline tls device
- * @dev_list: Inline tls device list
- * int (*feature)(struct tls_toe_device *device);
- * Called to return Inline TLS driver capability
- *
- * int (*hash)(struct tls_toe_device *device, struct sock *sk);
- * This function sets Inline driver for listen and program
- * device specific functioanlity as required
- *
- * void (*unhash)(struct tls_toe_device *device, struct sock *sk);
- * This function cleans listen state set by Inline TLS driver
- *
- * void (*release)(struct kref *kref);
- * Release the registered device and allocated resources
- * @kref: Number of reference to tls_toe_device
- */
-struct tls_toe_device {
- char name[TLS_TOE_DEVICE_NAME_MAX];
- struct list_head dev_list;
- int (*feature)(struct tls_toe_device *device);
- int (*hash)(struct tls_toe_device *device, struct sock *sk);
- void (*unhash)(struct tls_toe_device *device, struct sock *sk);
- void (*release)(struct kref *kref);
- struct kref kref;
-};
-
-int tls_toe_bypass(struct sock *sk);
-int tls_toe_hash(struct sock *sk);
-void tls_toe_unhash(struct sock *sk);
-
-void tls_toe_register_device(struct tls_toe_device *device);
-void tls_toe_unregister_device(struct tls_toe_device *device);
diff --git a/include/uapi/linux/tls.h b/include/uapi/linux/tls.h
index b8b9c42f848c..1245ab38afc1 100644
--- a/include/uapi/linux/tls.h
+++ b/include/uapi/linux/tls.h
@@ -203,6 +203,6 @@ enum {
#define TLS_CONF_BASE 1
#define TLS_CONF_SW 2
#define TLS_CONF_HW 3
-#define TLS_CONF_HW_RECORD 4
+#define TLS_CONF_HW_RECORD 4 /* unused */
#endif /* _UAPI_LINUX_TLS_H */
diff --git a/net/tls/Kconfig b/net/tls/Kconfig
index a25bf57f2673..4f4d5973a28f 100644
--- a/net/tls/Kconfig
+++ b/net/tls/Kconfig
@@ -27,13 +27,3 @@ config TLS_DEVICE
Enable kernel support for HW offload of the TLS protocol.
If unsure, say N.
-
-config TLS_TOE
- bool "Transport Layer Security TCP stack bypass"
- depends on TLS
- default n
- help
- Enable kernel support for legacy HW offload of the TLS protocol,
- which is incompatible with the Linux networking stack semantics.
-
- If unsure, say N.
diff --git a/net/tls/Makefile b/net/tls/Makefile
index e41c800489ac..4c7d296081ee 100644
--- a/net/tls/Makefile
+++ b/net/tls/Makefile
@@ -9,5 +9,4 @@ obj-$(CONFIG_TLS) += tls.o
tls-y := tls_main.o tls_sw.o tls_proc.o trace.o tls_strp.o
-tls-$(CONFIG_TLS_TOE) += tls_toe.o
tls-$(CONFIG_TLS_DEVICE) += tls_device.o tls_device_fallback.o
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index c10a3fd7fc17..13c88a7b8787 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -43,8 +43,6 @@
#include <net/snmp.h>
#include <net/tls.h>
-#include <net/tls_toe.h>
-
#include "tls.h"
MODULE_AUTHOR("Mellanox Technologies");
@@ -963,9 +961,6 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG]
ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ];
#endif
-#ifdef CONFIG_TLS_TOE
- ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
-#endif
}
static void tls_build_proto(struct sock *sk)
@@ -1037,11 +1032,6 @@ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
#endif
-#ifdef CONFIG_TLS_TOE
- prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
- prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash;
- prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash;
-#endif
}
static int tls_init(struct sock *sk)
@@ -1051,11 +1041,6 @@ static int tls_init(struct sock *sk)
tls_build_proto(sk);
-#ifdef CONFIG_TLS_TOE
- if (tls_toe_bypass(sk))
- return 0;
-#endif
-
/* The TLS ulp is currently supported only for TCP sockets
* in ESTABLISHED state.
* Supporting sockets in LISTEN state will require us
@@ -1111,8 +1096,6 @@ static u16 tls_user_config(struct tls_context *ctx, bool tx)
return TLS_CONF_SW;
case TLS_HW:
return TLS_CONF_HW;
- case TLS_HW_RECORD:
- return TLS_CONF_HW_RECORD;
}
return 0;
}
diff --git a/net/tls/tls_toe.c b/net/tls/tls_toe.c
deleted file mode 100644
index 825669e1ab47..000000000000
--- a/net/tls/tls_toe.c
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
- * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/list.h>
-#include <linux/rcupdate.h>
-#include <linux/spinlock.h>
-#include <net/inet_connection_sock.h>
-#include <net/tls.h>
-#include <net/tls_toe.h>
-
-#include "tls.h"
-
-static LIST_HEAD(device_list);
-static DEFINE_SPINLOCK(device_spinlock);
-
-static void tls_toe_sk_destruct(struct sock *sk)
-{
- struct inet_connection_sock *icsk = inet_csk(sk);
- struct tls_context *ctx = tls_get_ctx(sk);
-
- ctx->sk_destruct(sk);
- /* Free ctx */
- rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
- tls_ctx_free(sk, ctx);
-}
-
-int tls_toe_bypass(struct sock *sk)
-{
- struct tls_toe_device *dev;
- struct tls_context *ctx;
- int rc = 0;
-
- spin_lock_bh(&device_spinlock);
- list_for_each_entry(dev, &device_list, dev_list) {
- if (dev->feature && dev->feature(dev)) {
- ctx = tls_ctx_create(sk);
- if (!ctx)
- goto out;
-
- ctx->sk_destruct = sk->sk_destruct;
- sk->sk_destruct = tls_toe_sk_destruct;
- ctx->rx_conf = TLS_HW_RECORD;
- ctx->tx_conf = TLS_HW_RECORD;
- update_sk_prot(sk, ctx);
- rc = 1;
- break;
- }
- }
-out:
- spin_unlock_bh(&device_spinlock);
- return rc;
-}
-
-void tls_toe_unhash(struct sock *sk)
-{
- struct tls_context *ctx = tls_get_ctx(sk);
- struct tls_toe_device *dev;
-
- spin_lock_bh(&device_spinlock);
- list_for_each_entry(dev, &device_list, dev_list) {
- if (dev->unhash) {
- kref_get(&dev->kref);
- spin_unlock_bh(&device_spinlock);
- dev->unhash(dev, sk);
- kref_put(&dev->kref, dev->release);
- spin_lock_bh(&device_spinlock);
- }
- }
- spin_unlock_bh(&device_spinlock);
- ctx->sk_proto->unhash(sk);
-}
-
-int tls_toe_hash(struct sock *sk)
-{
- struct tls_context *ctx = tls_get_ctx(sk);
- struct tls_toe_device *dev;
- int err;
-
- err = ctx->sk_proto->hash(sk);
- spin_lock_bh(&device_spinlock);
- list_for_each_entry(dev, &device_list, dev_list) {
- if (dev->hash) {
- kref_get(&dev->kref);
- spin_unlock_bh(&device_spinlock);
- err |= dev->hash(dev, sk);
- kref_put(&dev->kref, dev->release);
- spin_lock_bh(&device_spinlock);
- }
- }
- spin_unlock_bh(&device_spinlock);
-
- if (err)
- tls_toe_unhash(sk);
- return err;
-}
-
-void tls_toe_register_device(struct tls_toe_device *device)
-{
- spin_lock_bh(&device_spinlock);
- list_add_tail(&device->dev_list, &device_list);
- spin_unlock_bh(&device_spinlock);
-}
-EXPORT_SYMBOL(tls_toe_register_device);
-
-void tls_toe_unregister_device(struct tls_toe_device *device)
-{
- spin_lock_bh(&device_spinlock);
- list_del(&device->dev_list);
- spin_unlock_bh(&device_spinlock);
-}
-EXPORT_SYMBOL(tls_toe_unregister_device);
--
2.54.0
^ permalink raw reply related
* Re: [RFC v1 0/9] kho: granular compatibility and header decoupling
From: Pratyush Yadav @ 2026-06-09 14:28 UTC (permalink / raw)
To: Pasha Tatashin
Cc: Mike Rapoport, linux-kselftest, shuah, akpm, linux-mm, skhan,
linux-doc, jasonmiu, linux-kernel, corbet, ran.xiaokai, kexec,
pratyush, graf, Logan Odell
In-Reply-To: <aibYJvzQQnpoN6YW@plex>
On Mon, Jun 08 2026, Pasha Tatashin wrote:
> On 06-08 13:26, Mike Rapoport wrote:
>> On 2026-06-07 13:43:09+00:00, Pasha Tatashin wrote:
>> > On 06-07 14:58, Mike Rapoport wrote:
>> >
>> > > On Fri, 05 Jun 2026 03:32:26 +0000, Pasha Tatashin <pasha.tatashin@soleen.com> wrote:
[...]
>> > External users only need to include the headers they actually use. For
>> > example, LUO shouldn't have to pull vmalloc or radix tree KHO
>> > declarations, and memfd does not need block.
>> >
>> > From a maintenance point of view, it is much easier to catch ABI
>> > changes when the file with the appropriate version has been changed,
>> > and most likely the version of that file should be updated. If a single
>> > header contains compatibility versions for several different data
>> > structures, it is easier to miss the correct version update.
>>
>> No matter in what files the definition lives, someone can forget to
>> update version and we may miss it during review.
Perhaps we should have some tests (maybe with kunit?) that can catch
this? If you change the format, the test fails. So you'd have to go and
update the test, and at that point it should be more obvious that ABI
version needs bumping.
[...]
>>
>> Sorry I wasn't clear. I agree that kho_vmalloc, block and radix tree
>> should have their own versioning rather than rely on global KHO version.
>>
>> What I don't like in your proposal is mixing versioning of a component
>> with its dependencies.
>>
>> I think that versioning should be completely local to each component.
>> LUO should not care about kho_block "on wire" layout. This should be
>> encapsulated in kho_block.
>
> That is a fair point.
>
> As I mentioned in my previous reply, we can definitely look into making
> the version checking more modular. For example, each component could
> implement a standard compatibility-checking interface.
>
> These checks could run early in boot to determine whether each component
> is capable of accepting the incoming preserved data format.
>
> Whenever the component is later used by LUO, memfd, etc., we can query
> that cached status. This achieves four key benefits:
>
> 1. It avoids delaying the compatibility check to the actual time of data
> retrieval, which is too late to safely abort.
>
> 2. It prevents a local incompatibility from triggering a global kernel
> panic, allowing us to handle failures gracefully for just that specific
> component or session.
I think the right time to do the compatibility check is _before_ kexec.
That is the only point where you can safely abort. Once you boot into
the new kernel and discover you can't understand the passed data, you
are in a bad spot already and should reboot. I don't think think you
really can gracefully handle these failures.
For example, say you fail to understand the incoming PCI data. So you
have no idea which devices are participating in live update and cannot
correctly probe any of them. Which effectively means you cannot resume
any of your guests since you have no idea how to restore their device
state. The only path you are left with is to reboot. I haven't read the
IOMMU series, but I imagine the same story applies there.
For a more benign example, let's assume one of your memfds that back VM
memory fail to restore.
In this case, you can safely leak that memory and run the other guests,
but at that point the host is in impaired state. You don't want to keep
running it in this state. You likely either do a reboot, or if you feel
more adventurous, you do another live update.
In either case, there is no "safely abort" after the kexec happens.
So I think our energy is better spent solving the versioning story
_before_ kexec. After kexec I think it is perfectly fine to error out
and panic or expect a reboot. You can't salvage much at that point
anyway.
And I think how the versioning format looks also should be based on the
design of this pre-kexec check, not the other way round.
>
> 3. It keeps the local version local, as you suggested, so it is checked
> only by the consumers of that specific component.
>
> 4. It provides a clean path for backward compatibility, as components
> can individually decide whether they understand the incoming data
> format.
>
[...]
>>
>> Actually FDT "compatible" handles versioning nicer than composite strings
>> You can have
>>
>> compatible="kho-v4", "vmalloc-v1", "radix-v1", "block-v2";
>>
>> and check fdt_node_check_compatible("vmalloc-v1") for vmalloc and
>> fdt_node_check_compatible("block-v2") for block.
I agree. Even if we don't use FDT, something more structured than
composite strings would be nice to have.
>
> That is actually very similar to what I am proposing—individual version
> tokens (which in my current series are concatenated into a composite
> compatibility string separated by ';').
>
> But let's not get too fixated on the composite string formatting. I
> actually really like what you are proposing: using integers for versions
> and having each registered component carry its own "NAME" and version
> number in the KHO FDT.
There is another nice thing about numbers that Logan (+cc) recently
pointed out. You can tell which one is bigger.
At some point I think we will support multiple versions of a data
structure to allow for upgrades. At that point, it will help to know
which one is "newer". So if both kernel versions support version 3 and
4, you can use 4 to serialize.
This of course is harder to do with strings.
>
>> And we wouldn't need to reimplement string parsing ;-)
>>
>> But yeah, I do see value of making components versioning and KHO global
>> versioning independent. I just don't like composite strings and I don't
>> like mixing versioning with dependencies.
>>
>> Since we are moving from FDT for the most things, version should become
>> a number rather than a string and version compatibility should be
[...]
--
Regards,
Pratyush Yadav
^ permalink raw reply
* Re: [PATCH] Docs/damon: add TLB flush policy document
From: SeongJae Park @ 2026-06-09 14:28 UTC (permalink / raw)
To: KunWu Chan
Cc: SeongJae Park, akpm, david, ljs, liam, vbabka, rppt, surenb,
mhocko, corbet, skhan, damon, linux-mm, linux-doc, linux-kernel,
Wang Lian
In-Reply-To: <CAN_Opa8sqs-V-jFrGZa6jwJKdHqT=7JPeLKcU8E-cCM74ON4DA@mail.gmail.com>
On Tue, 9 Jun 2026 16:11:30 +0800 KunWu Chan <kunwu.chan@gmail.com> wrote:
> On Sat, Jun 6, 2026 at 8:54 AM SeongJae Park <sj@kernel.org> wrote:
> >
> > Hi Kunwu and Lian,
> >
> > On Fri, 5 Jun 2026 11:10:08 +0800 Kunwu Chan <kunwu.chan@gmail.com> wrote:
[...]
> > Mainly due to the verbosity, as I above mentionedd, I'm not sure if the current
> > shape of this patch is the best to be merged as is. I also find the background
> > part of the document is a kind of duplicate of some information in design.rst.
> > What about putting only essential information in a condensed way on the
> > design.rst?
> >
> Thanks, SJ. I agree — this document reads more like an article than short docs.
>
> I'll follow your suggestion and submit it as a blog post to the DAMON
> project blog instead.
>
> Will send a PR to [2] soon. Thanks for the clear guidance.
Sounds good, looking forward to the PR!
>
> > [1] https://damonitor.github.io/site_about
> > [2] https://github.com/damonitor/damonitor.github.io/tree/master/blog_src
Thanks,
SJ
[...]
^ permalink raw reply
* [PATCH v4 00/19] perf cs-etm: Queue context packets for frontend
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
Fix thread tracking when decoding Coresight trace and add a new test for
it.
The new test is added as a Perf test workload instead of a custom binary
with its own build system, but this requires a new feature in Perf test
to pass in control pipes which can enable and disable events. This
scopes the recording to just the workload and helps to reduce the amount
of data recorded in tracing tests.
With this new feature we can re-write all of the Coresight tests to make
use of it and remove the remaining binaries which fixes the following
issues:
* They didn't work in out of source builds
* A lot of the tests unnecessarily required root and didn't skip
without it
* They were mainly qualitative tests which didn't look for specific
behavior
Most importantly, the long build and runtime has been reduced. On a
Radxa Orion O6, unroll_loop_thread.c took 37s to compile which is longer
than the entire Perf build. Now the build time is negligible and the
before and after test runtimes for all the Coresight tests are:
| N1SDP | Orion O6
-----------------------------------
Before | 4m 0s | 14m 49s
After | 26s | 56s
-----------------------------------
Signed-off-by: James Clark <james.clark@linaro.org>
---
Changes in v4:
- Rename workload-ctl to record-ctl and improve docs (Leo)
- Use new packet argument everywhere in
cs_etm__synth_instruction_sample() (Sashiko)
- Test for actual length of expected raw dump (Leo)
- Use -fno-inline instead of keyword (Leo)
- Don't test any brace or call lines in deterministic test
- Make sure context switch loop test does cleanup on failure (Sashiko)
- Remove undef int overflows in workloads (Sashiko)
- Link to v3: https://lore.kernel.org/r/20260603-james-cs-context-tracking-fix-v3-0-c392945d9ed5@linaro.org
Changes in v3:
- Minor sashiko comments
- Close some more pipes
- Fix warning messages
- Error handling improvements
- Pass packet into cs_etm__synth_instruction_sample()
- Fixup stale comment (Leo)
- Link to v2: https://lore.kernel.org/r/20260602-james-cs-context-tracking-fix-v2-0-85b5ce6f55c6@linaro.org
Changes in v2:
- Add --workload-ctl option to Perf test
- Re-write all the Coresight tests and speed them up
- Pass packet to memory access function so frontend can use either the
previous or current packet's EL
- Link to v1: https://lore.kernel.org/r/20260526-james-cs-context-tracking-fix-v1-0-ebd602e18287@linaro.org
---
James Clark (19):
perf cs-etm: Queue context packets for frontend
perf test: Add workload-ctl option
perf test: Add a workload that forces context switches
perf test cs-etm: Test process attribution
perf test: Add deterministic workload
perf test cs-etm: Replace unroll loop thread with deterministic decode test
perf test cs-etm: Remove asm_pure_loop test
perf test cs-etm: Replace memcpy test with raw dump stress test
perf test: Add named_threads workload
perf test cs-etm: Test decoding for concurrent threads test
perf test cs-etm: Remove duplicate branch tests
perf test cs-etm: Skip if not root
perf test cs-etm: Reduce snapshot size
perf test cs-etm: Speed up basic test
perf test cs-etm: Remove unused Coresight workloads
perf test cs-etm: Make disassembly test use kcore
perf test cs-etm: Add all branch instructions to test
perf test cs-etm: Speed up disassembly test
perf test cs-etm: Move existing tests to coresight folder
Documentation/trace/coresight/coresight-perf.rst | 78 +------
MAINTAINERS | 2 -
tools/perf/Documentation/perf-test.txt | 24 ++-
tools/perf/Makefile.perf | 14 +-
tools/perf/scripts/python/arm-cs-trace-disasm.py | 20 +-
tools/perf/tests/builtin-test.c | 187 +++++++++++++++-
tools/perf/tests/shell/coresight/Makefile | 29 ---
.../perf/tests/shell/coresight/Makefile.miniconfig | 14 --
tools/perf/tests/shell/coresight/asm_pure_loop.sh | 22 --
.../tests/shell/coresight/asm_pure_loop/.gitignore | 1 -
.../tests/shell/coresight/asm_pure_loop/Makefile | 34 ---
.../shell/coresight/asm_pure_loop/asm_pure_loop.S | 30 ---
.../tests/shell/coresight/concurrent_threads.sh | 45 ++++
.../tests/shell/coresight/context_switch_thread.sh | 69 ++++++
tools/perf/tests/shell/coresight/deterministic.sh | 72 +++++++
.../tests/shell/coresight/memcpy_thread/.gitignore | 1 -
.../tests/shell/coresight/memcpy_thread/Makefile | 33 ---
.../shell/coresight/memcpy_thread/memcpy_thread.c | 80 -------
.../tests/shell/coresight/memcpy_thread_16k_10.sh | 22 --
.../perf/tests/shell/coresight/raw_dump_stress.sh | 47 ++++
.../shell/{ => coresight}/test_arm_coresight.sh | 43 ++--
.../{ => coresight}/test_arm_coresight_disasm.sh | 23 +-
.../tests/shell/coresight/thread_loop/.gitignore | 1 -
.../tests/shell/coresight/thread_loop/Makefile | 33 ---
.../shell/coresight/thread_loop/thread_loop.c | 85 --------
.../shell/coresight/thread_loop_check_tid_10.sh | 23 --
.../shell/coresight/thread_loop_check_tid_2.sh | 23 --
.../shell/coresight/unroll_loop_thread/.gitignore | 1 -
.../shell/coresight/unroll_loop_thread/Makefile | 33 ---
.../unroll_loop_thread/unroll_loop_thread.c | 75 -------
.../tests/shell/coresight/unroll_loop_thread_10.sh | 22 --
tools/perf/tests/shell/lib/coresight.sh | 134 ------------
tools/perf/tests/tests.h | 3 +
tools/perf/tests/workloads/Build | 4 +
tools/perf/tests/workloads/context_switch_loop.c | 110 ++++++++++
tools/perf/tests/workloads/deterministic.c | 39 ++++
tools/perf/tests/workloads/named_threads.c | 109 ++++++++++
tools/perf/util/cs-etm-decoder/cs-etm-decoder.c | 21 +-
tools/perf/util/cs-etm.c | 236 ++++++++++++---------
tools/perf/util/cs-etm.h | 8 +-
40 files changed, 908 insertions(+), 942 deletions(-)
---
base-commit: 351a37f2fda4db668cff8ba12f2992d73dccdaea
change-id: 20260515-james-cs-context-tracking-fix-754998bae7ed
Best regards,
--
James Clark <james.clark@linaro.org>
^ permalink raw reply
* [PATCH v4 01/19] perf cs-etm: Queue context packets for frontend
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
In-Reply-To: <20260609-james-cs-context-tracking-fix-v4-0-44f9fb9e5c42@linaro.org>
PE_CONTEXT elements update the context ID and exception level, but the
decoder may still have prior packets cached for frontend processing.
Updating the context immediately in the decoder backend can make those
cached packets get consumed with the wrong thread or EL state.
Add a CS_ETM_CONTEXT packet carrying the TID and EL to the frontend,
this keeps context changes ordered with the rest of the packet stream
and avoids mismatches when synthesizing samples from cached packets.
Separate the memory access function into one for the frontend and one
for decoding. The frontend also needs memory access to attach the
instruction to samples. Because the frontend does memory access for
both previous and current packets, change all the frontend memory access
function signatures to take both a tidq and packet. But backend always
uses the current backend EL and thread from the tidq.
Treat context packets as a boundary for branch sample generation and
remove tidq->prev_packet_thread because it's not possible to branch to a
different thread, so only tracking the current thread is required for
sample generation.
Fixes: e573e978fb12 ("perf cs-etm: Inject capabilitity for CoreSight traces")
Reported-by: Amir Ayupov <aaupov@meta.com>
Closes: https://lore.kernel.org/linux-perf-users/20260515021135.1729028-1-aaupov@meta.com/
Co-authored-by: James Clark <james.clark@linaro.org>
Signed-off-by: Leo Yan <leo.yan@arm.com>
Signed-off-by: James Clark <james.clark@linaro.org>
---
tools/perf/util/cs-etm-decoder/cs-etm-decoder.c | 21 ++-
tools/perf/util/cs-etm.c | 236 ++++++++++++++----------
tools/perf/util/cs-etm.h | 8 +-
3 files changed, 163 insertions(+), 102 deletions(-)
diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c
index dee3020ceaa9..26940f1f1b0b 100644
--- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c
+++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c
@@ -402,6 +402,8 @@ cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq,
packet_queue->packet_buffer[et].flags = 0;
packet_queue->packet_buffer[et].exception_number = UINT32_MAX;
packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id;
+ packet_queue->packet_buffer[et].el = ocsd_EL_unknown;
+ packet_queue->packet_buffer[et].tid = -1;
if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1)
return OCSD_RESP_WAIT;
@@ -449,6 +451,7 @@ cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq,
packet->last_instr_type = elem->last_i_type;
packet->last_instr_subtype = elem->last_i_subtype;
packet->last_instr_cond = elem->last_instr_cond;
+ packet->el = elem->context.exception_level;
if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT)
packet->last_instr_taken_branch = elem->last_instr_exec;
@@ -525,7 +528,9 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq,
const ocsd_generic_trace_elem *elem,
const uint8_t trace_chan_id)
{
+ struct cs_etm_packet *packet;
pid_t tid = -1;
+ int ret;
/*
* Process the PE_CONTEXT packets if we have a valid contextID or VMID.
@@ -546,12 +551,18 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq,
break;
}
- if (cs_etm__etmq_set_tid_el(etmq, tid, trace_chan_id,
- elem->context.exception_level))
+ if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id,
+ elem->context.exception_level, tid))
return OCSD_RESP_FATAL_SYS_ERR;
- if (tid == -1)
- return OCSD_RESP_CONT;
+ ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id,
+ CS_ETM_CONTEXT);
+ if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT)
+ return ret;
+
+ packet = &packet_queue->packet_buffer[packet_queue->tail];
+ packet->tid = tid;
+ packet->el = elem->context.exception_level;
/*
* A timestamp is generated after a PE_CONTEXT element so make sure
@@ -559,7 +570,7 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq,
*/
cs_etm_decoder__reset_timestamp(packet_queue);
- return OCSD_RESP_CONT;
+ return ret;
}
static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer(
diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c
index 40c6ddfa8c8d..5e92359f51a7 100644
--- a/tools/perf/util/cs-etm.c
+++ b/tools/perf/util/cs-etm.c
@@ -85,15 +85,22 @@ struct cs_etm_traceid_queue {
u64 period_instructions;
size_t last_branch_pos;
union perf_event *event_buf;
- struct thread *thread;
- struct thread *prev_packet_thread;
- ocsd_ex_level prev_packet_el;
- ocsd_ex_level el;
struct branch_stack *last_branch;
struct branch_stack *last_branch_rb;
struct cs_etm_packet *prev_packet;
struct cs_etm_packet *packet;
struct cs_etm_packet_queue packet_queue;
+
+ struct thread *decode_thread;
+ ocsd_ex_level decode_el;
+
+ /*
+ * The frontend accesses the EL from '[prev_]packet' because it needs
+ * previous EL for branch and current EL for instruction samples. It's
+ * not possible to change thread in a single branch sample so no need to
+ * store or access the thread through the packet.
+ */
+ struct thread *frontend_thread;
};
enum cs_etm_format {
@@ -614,10 +621,11 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
tidq->trace_chan_id = trace_chan_id;
- tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;
- tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,
+ tidq->decode_el = ocsd_EL_unknown;
+ tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1,
+ queue->tid);
+ tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1,
queue->tid);
- tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);
tidq->packet = zalloc(sizeof(struct cs_etm_packet));
if (!tidq->packet)
@@ -750,21 +758,10 @@ static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
/*
* Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
* the next incoming packet.
- *
- * Threads and exception levels are also tracked for both the
- * previous and current packets. This is because the previous
- * packet is used for the 'from' IP for branch samples, so the
- * thread at that time must also be assigned to that sample.
- * Across discontinuity packets the thread can change, so by
- * tracking the thread for the previous packet the branch sample
- * will have the correct info.
*/
tmp = tidq->packet;
tidq->packet = tidq->prev_packet;
tidq->prev_packet = tmp;
- tidq->prev_packet_el = tidq->el;
- thread__put(tidq->prev_packet_thread);
- tidq->prev_packet_thread = thread__get(tidq->thread);
}
}
@@ -937,8 +934,8 @@ static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
/* Free this traceid_queue from the array */
tidq = etmq->traceid_queues[idx];
- thread__zput(tidq->thread);
- thread__zput(tidq->prev_packet_thread);
+ thread__zput(tidq->frontend_thread);
+ thread__zput(tidq->decode_thread);
zfree(&tidq->event_buf);
zfree(&tidq->last_branch);
zfree(&tidq->last_branch_rb);
@@ -1083,47 +1080,43 @@ static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address,
}
}
-static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
- u64 address, size_t size, u8 *buffer,
- const ocsd_mem_space_acc_t mem_space)
+static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq,
+ u64 address, size_t size, u8 *buffer,
+ const ocsd_mem_space_acc_t mem_space,
+ ocsd_ex_level el, struct thread *thread)
{
u8 cpumode;
u64 offset;
int len;
struct addr_location al;
struct dso *dso;
- struct cs_etm_traceid_queue *tidq;
int ret = 0;
if (!etmq)
return 0;
addr_location__init(&al);
- tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
- if (!tidq)
- goto out;
/*
- * We've already tracked EL along side the PID in cs_etm__set_thread()
- * so double check that it matches what OpenCSD thinks as well. It
- * doesn't distinguish between EL0 and EL1 for this mem access callback
- * so we had to do the extra tracking. Skip validation if it's any of
- * the 'any' values.
+ * We track EL for the frontend and the backend when receiving context
+ * and range packets. OpenCSD doesn't distinguish between EL0 and EL1
+ * for this mem access callback so we had to do the extra tracking. Skip
+ * validation if it's any of the 'any' values.
*/
if (!(mem_space == OCSD_MEM_SPACE_ANY ||
mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) {
if (mem_space & OCSD_MEM_SPACE_EL1N) {
/* Includes both non secure EL1 and EL0 */
- assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);
+ assert(el == ocsd_EL1 || el == ocsd_EL0);
} else if (mem_space & OCSD_MEM_SPACE_EL2)
- assert(tidq->el == ocsd_EL2);
+ assert(el == ocsd_EL2);
else if (mem_space & OCSD_MEM_SPACE_EL3)
- assert(tidq->el == ocsd_EL3);
+ assert(el == ocsd_EL3);
}
- cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);
+ cpumode = cs_etm__cpu_mode(etmq, address, el);
- if (!thread__find_map(tidq->thread, cpumode, address, &al))
+ if (!thread__find_map(thread, cpumode, address, &al))
goto out;
dso = map__dso(al.map);
@@ -1138,7 +1131,7 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
map__load(al.map);
- len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),
+ len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)),
offset, buffer, size);
if (len <= 0) {
@@ -1158,6 +1151,30 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
return ret;
}
+static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq,
+ struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet,
+ u64 address, size_t size, u8 *buffer)
+{
+ return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el,
+ tidq->frontend_thread);
+}
+
+static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
+ u64 address, size_t size, u8 *buffer,
+ const ocsd_mem_space_acc_t mem_space)
+{
+ struct cs_etm_traceid_queue *tidq;
+
+ tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
+ if (!tidq)
+ return 0;
+
+ return __cs_etm__mem_access(etmq, address, size, buffer,
+ mem_space, tidq->decode_el,
+ tidq->decode_thread);
+}
+
static struct cs_etm_queue *cs_etm__alloc_queue(void)
{
struct cs_etm_queue *etmq = zalloc(sizeof(*etmq));
@@ -1333,12 +1350,13 @@ void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
}
static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
- u8 trace_chan_id, u64 addr)
+ struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet, u64 addr)
{
u8 instrBytes[2];
- cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes),
- instrBytes, 0);
+ cs_etm__frontend_mem_access(etmq, tidq, packet, addr,
+ ARRAY_SIZE(instrBytes), instrBytes);
/*
* T32 instruction size is indicated by bits[15:11] of the first
* 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
@@ -1371,16 +1389,16 @@ u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
}
static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
- u64 trace_chan_id,
- const struct cs_etm_packet *packet,
+ struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet,
u64 offset)
{
if (packet->isa == CS_ETM_ISA_T32) {
u64 addr = packet->start_addr;
while (offset) {
- addr += cs_etm__t32_instr_size(etmq,
- trace_chan_id, addr);
+ addr += cs_etm__t32_instr_size(etmq, tidq, packet,
+ addr);
offset--;
}
return addr;
@@ -1490,34 +1508,51 @@ cs_etm__get_trace(struct cs_etm_queue *etmq)
return etmq->buf_len;
}
-static void cs_etm__set_thread(struct cs_etm_queue *etmq,
- struct cs_etm_traceid_queue *tidq, pid_t tid,
- ocsd_ex_level el)
+/*
+ * Convert a raw thread number to a thread struct and assign it to **thread.
+ */
+static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq,
+ ocsd_ex_level el, pid_t tid,
+ struct thread **thread)
{
struct machine *machine = cs_etm__get_machine(etmq, el);
+ if (!machine || !*thread)
+ return -EINVAL;
+
if (tid != -1) {
- thread__zput(tidq->thread);
- tidq->thread = machine__find_thread(machine, -1, tid);
+ thread__zput(*thread);
+ *thread = machine__find_thread(machine, -1, tid);
}
/* Couldn't find a known thread */
- if (!tidq->thread)
- tidq->thread = machine__idle_thread(machine);
+ if (!*thread)
+ *thread = machine__idle_thread(machine);
- tidq->el = el;
+ return 0;
}
-int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
- u8 trace_chan_id, ocsd_ex_level el)
+/*
+ * Set the thread and EL of the decode context which is ahead in time of the
+ * frontend context.
+ */
+int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq,
+ u8 trace_chan_id,
+ ocsd_ex_level el, pid_t tid)
{
struct cs_etm_traceid_queue *tidq;
+ int ret;
tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
if (!tidq)
return -EINVAL;
- cs_etm__set_thread(etmq, tidq, tid, el);
+ ret = cs_etm__etmq_update_thread(etmq, el, tid,
+ &tidq->decode_thread);
+ if (ret)
+ return ret;
+
+ tidq->decode_el = el;
return 0;
}
@@ -1527,8 +1562,8 @@ bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
}
static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
- u64 trace_chan_id,
- const struct cs_etm_packet *packet,
+ struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet,
struct perf_sample *sample)
{
/*
@@ -1545,14 +1580,14 @@ static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
* cs_etm__t32_instr_size().
*/
if (packet->isa == CS_ETM_ISA_T32)
- sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
+ sample->insn_len = cs_etm__t32_instr_size(etmq, tidq, packet,
sample->ip);
/* Otherwise, A64 and A32 instruction size are always 32-bit. */
else
sample->insn_len = 4;
- cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len,
- (void *)sample->insn, 0);
+ cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip,
+ sample->insn_len, (void *)sample->insn);
}
u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
@@ -1579,6 +1614,7 @@ static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet,
u64 addr, u64 period)
{
int ret = 0;
@@ -1588,23 +1624,23 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
perf_sample__init(&sample, /*all=*/true);
event->sample.header.type = PERF_RECORD_SAMPLE;
- event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);
+ event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el);
event->sample.header.size = sizeof(struct perf_event_header);
/* Set time field based on etm auxtrace config. */
sample.time = cs_etm__resolve_sample_time(etmq, tidq);
sample.ip = addr;
- sample.pid = thread__pid(tidq->thread);
- sample.tid = thread__tid(tidq->thread);
+ sample.pid = thread__pid(tidq->frontend_thread);
+ sample.tid = thread__tid(tidq->frontend_thread);
sample.id = etmq->etm->instructions_id;
sample.stream_id = etmq->etm->instructions_id;
sample.period = period;
- sample.cpu = tidq->packet->cpu;
+ sample.cpu = packet->cpu;
sample.flags = tidq->prev_packet->flags;
sample.cpumode = event->sample.header.misc;
- cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
+ cs_etm__copy_insn(etmq, tidq, packet, &sample);
if (etm->synth_opts.last_branch)
sample.branch_stack = tidq->last_branch;
@@ -1649,15 +1685,15 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
event->sample.header.type = PERF_RECORD_SAMPLE;
event->sample.header.misc = cs_etm__cpu_mode(etmq, ip,
- tidq->prev_packet_el);
+ tidq->prev_packet->el);
event->sample.header.size = sizeof(struct perf_event_header);
/* Set time field based on etm auxtrace config. */
sample.time = cs_etm__resolve_sample_time(etmq, tidq);
sample.ip = ip;
- sample.pid = thread__pid(tidq->prev_packet_thread);
- sample.tid = thread__tid(tidq->prev_packet_thread);
+ sample.pid = thread__pid(tidq->frontend_thread);
+ sample.tid = thread__tid(tidq->frontend_thread);
sample.addr = cs_etm__first_executed_instr(tidq->packet);
sample.id = etmq->etm->branches_id;
sample.stream_id = etmq->etm->branches_id;
@@ -1666,8 +1702,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
sample.flags = tidq->prev_packet->flags;
sample.cpumode = event->sample.header.misc;
- cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
- &sample);
+ cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample);
/*
* perf report cannot handle events without a branch stack
@@ -1788,7 +1823,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
{
struct cs_etm_auxtrace *etm = etmq->etm;
int ret;
- u8 trace_chan_id = tidq->trace_chan_id;
u64 instrs_prev;
/* Get instructions remainder from previous packet */
@@ -1874,10 +1908,10 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
* been executed, but PC has not advanced to next
* instruction)
*/
- addr = cs_etm__instr_addr(etmq, trace_chan_id,
- tidq->packet, offset - 1);
+ addr = cs_etm__instr_addr(etmq, tidq, tidq->packet,
+ offset - 1);
ret = cs_etm__synth_instruction_sample(
- etmq, tidq, addr,
+ etmq, tidq, tidq->packet, addr,
etm->instructions_sample_period);
if (ret)
return ret;
@@ -1959,7 +1993,7 @@ static int cs_etm__flush(struct cs_etm_queue *etmq,
addr = cs_etm__last_executed_instr(tidq->prev_packet);
err = cs_etm__synth_instruction_sample(
- etmq, tidq, addr,
+ etmq, tidq, tidq->prev_packet, addr,
tidq->period_instructions);
if (err)
return err;
@@ -2014,7 +2048,7 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq,
addr = cs_etm__last_executed_instr(tidq->prev_packet);
err = cs_etm__synth_instruction_sample(
- etmq, tidq, addr,
+ etmq, tidq, tidq->prev_packet, addr,
tidq->period_instructions);
if (err)
return err;
@@ -2051,9 +2085,9 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
return etmq->buf_len;
}
-static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
- struct cs_etm_packet *packet,
- u64 end_addr)
+static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq,
+ struct cs_etm_traceid_queue *tidq,
+ struct cs_etm_packet *packet, u64 end_addr)
{
/* Initialise to keep compiler happy */
u16 instr16 = 0;
@@ -2075,8 +2109,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
* so below only read 2 bytes as instruction size for T32.
*/
addr = end_addr - 2;
- cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16),
- (u8 *)&instr16, 0);
+ cs_etm__frontend_mem_access(etmq, tidq, packet, addr,
+ sizeof(instr16), (u8 *)&instr16);
if ((instr16 & 0xFF00) == 0xDF00)
return true;
@@ -2091,8 +2125,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
* +---------+---------+-------------------------+
*/
addr = end_addr - 4;
- cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
- (u8 *)&instr32, 0);
+ cs_etm__frontend_mem_access(etmq, tidq, packet, addr,
+ sizeof(instr32), (u8 *)&instr32);
if ((instr32 & 0x0F000000) == 0x0F000000 &&
(instr32 & 0xF0000000) != 0xF0000000)
return true;
@@ -2108,8 +2142,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
* +-----------------------+---------+-----------+
*/
addr = end_addr - 4;
- cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
- (u8 *)&instr32, 0);
+ cs_etm__frontend_mem_access(etmq, tidq, packet, addr,
+ sizeof(instr32), (u8 *)&instr32);
if ((instr32 & 0xFFE0001F) == 0xd4000001)
return true;
@@ -2125,7 +2159,6 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
struct cs_etm_traceid_queue *tidq, u64 magic)
{
- u8 trace_chan_id = tidq->trace_chan_id;
struct cs_etm_packet *packet = tidq->packet;
struct cs_etm_packet *prev_packet = tidq->prev_packet;
@@ -2140,7 +2173,7 @@ static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
*/
if (magic == __perf_cs_etmv4_magic) {
if (packet->exception_number == CS_ETMV4_EXC_CALL &&
- cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
+ cs_etm__is_svc_instr(etmq, tidq, prev_packet,
prev_packet->end_addr))
return true;
}
@@ -2178,7 +2211,6 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
struct cs_etm_traceid_queue *tidq,
u64 magic)
{
- u8 trace_chan_id = tidq->trace_chan_id;
struct cs_etm_packet *packet = tidq->packet;
struct cs_etm_packet *prev_packet = tidq->prev_packet;
@@ -2204,7 +2236,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
* (SMC, HVC) are taken as sync exceptions.
*/
if (packet->exception_number == CS_ETMV4_EXC_CALL &&
- !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
+ !cs_etm__is_svc_instr(etmq, tidq, prev_packet,
prev_packet->end_addr))
return true;
@@ -2228,7 +2260,6 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
{
struct cs_etm_packet *packet = tidq->packet;
struct cs_etm_packet *prev_packet = tidq->prev_packet;
- u8 trace_chan_id = tidq->trace_chan_id;
u64 magic;
int ret;
@@ -2309,11 +2340,11 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
PERF_IP_FLAG_RETURN |
PERF_IP_FLAG_INTERRUPT) &&
- cs_etm__is_svc_instr(etmq, trace_chan_id,
- packet, packet->start_addr))
+ cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) {
prev_packet->flags = PERF_IP_FLAG_BRANCH |
PERF_IP_FLAG_RETURN |
PERF_IP_FLAG_SYSCALLRET;
+ }
break;
case CS_ETM_DISCONTINUITY:
/*
@@ -2394,6 +2425,7 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
PERF_IP_FLAG_RETURN |
PERF_IP_FLAG_INTERRUPT;
break;
+ case CS_ETM_CONTEXT:
case CS_ETM_EMPTY:
default:
break;
@@ -2469,6 +2501,19 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
*/
cs_etm__sample(etmq, tidq);
break;
+ case CS_ETM_CONTEXT:
+ /*
+ * Update context but don't swap packet. Keep the
+ * previous one for branch source address info, if
+ * tracing the kernel the context packet will be emitted
+ * between two ranges.
+ */
+ ret = cs_etm__etmq_update_thread(etmq, tidq->packet->el,
+ tidq->packet->tid,
+ &tidq->frontend_thread);
+ if (ret)
+ goto out;
+ break;
case CS_ETM_EXCEPTION:
case CS_ETM_EXCEPTION_RET:
/*
@@ -2497,6 +2542,7 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
}
}
+out:
return ret;
}
@@ -2620,7 +2666,7 @@ static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
if (!tidq)
continue;
- if (tid == -1 || thread__tid(tidq->thread) == tid)
+ if (tid == -1 || thread__tid(tidq->frontend_thread) == tid)
cs_etm__run_per_thread_timeless_decoder(etmq);
} else
cs_etm__run_per_cpu_timeless_decoder(etmq);
@@ -3328,7 +3374,7 @@ static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq)
*/
if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
0x0L, ((u64) -1L),
- cs_etm__mem_access))
+ cs_etm__decoder_mem_access))
goto out_free_decoder;
zfree(&t_params);
diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h
index aa9bb4a32eca..b81099c2b301 100644
--- a/tools/perf/util/cs-etm.h
+++ b/tools/perf/util/cs-etm.h
@@ -158,6 +158,7 @@ enum cs_etm_sample_type {
CS_ETM_DISCONTINUITY,
CS_ETM_EXCEPTION,
CS_ETM_EXCEPTION_RET,
+ CS_ETM_CONTEXT,
};
enum cs_etm_isa {
@@ -184,6 +185,8 @@ struct cs_etm_packet {
u8 last_instr_size;
u8 trace_chan_id;
int cpu;
+ int el;
+ pid_t tid;
};
#define CS_ETM_PACKET_MAX_BUFFER 1024
@@ -259,8 +262,9 @@ enum cs_etm_pid_fmt {
#include <opencsd/ocsd_if_types.h>
int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu);
enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq);
-int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
- u8 trace_chan_id, ocsd_ex_level el);
+int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq,
+ u8 trace_chan_id, ocsd_ex_level el,
+ pid_t tid);
bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq);
void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
u8 trace_chan_id);
--
2.34.1
^ permalink raw reply related
* [PATCH v4 02/19] perf test: Add workload-ctl option
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
In-Reply-To: <20260609-james-cs-context-tracking-fix-v4-0-44f9fb9e5c42@linaro.org>
Add a --workload-ctl=fifo:ctl-fifo[,ack-fifo] option for 'perf test
-w'. When set, run_workload() opens the named FIFO, writes enable before
invoking the builtin workload, writes disable before returning, and
waits for ack responses when an ack FIFO is provided to ensure that the
workload doesn't run until the events are enabled.
This can be used to limit the scope of the recording to only the
workload execution and avoid recording Perf setup and teardown code if
Perf record is started with events disabled (-D 1).
Assisted-by: Codex:GPT-5.5
Signed-off-by: James Clark <james.clark@linaro.org>
---
tools/perf/Documentation/perf-test.txt | 12 +++
tools/perf/tests/builtin-test.c | 184 ++++++++++++++++++++++++++++++++-
2 files changed, 194 insertions(+), 2 deletions(-)
diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt
index 32da0d1fa86a..2f4a91f5b9dc 100644
--- a/tools/perf/Documentation/perf-test.txt
+++ b/tools/perf/Documentation/perf-test.txt
@@ -69,3 +69,15 @@ OPTIONS
--list-workloads::
List the available workloads to use with -w/--workload.
+
+--record-ctl=fifo:ctl-fifo[,ack-fifo]::
+ This option is used to communicate with a perf record session in order
+ to control the recording scope to only the workload and avoid recording
+ setup and teardown code. When specifying this option, the same FIFO path
+ must be specified in the record session via:
+
+ perf record -D -1 --control=fifo:ctl-fifo[,ack-fifo] ...
+
+ Perf test sends 'enable' and 'disable' commands through ctl-fifo to
+ control event recording. If 'ack-fifo' is provided, the workload runner
+ waits for an 'ack' response after each command.
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index b64fc2204f22..86ea427eb0aa 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -56,6 +56,7 @@ static unsigned int runs_per_test = 1;
static unsigned int failure_snippet_lines = 10;
const char *dso_to_test;
const char *test_objdump_path = "objdump";
+static const char *workload_control;
/*
* List of architecture specific tests. Not a weak symbol as the array length is
@@ -168,6 +169,11 @@ static struct test_workload *workloads[] = {
#endif
};
+struct workload_control {
+ int ctl_fd;
+ int ack_fd;
+};
+
#define workloads__for_each(workload) \
for (unsigned i = 0; i < ARRAY_SIZE(workloads) && ({ workload = workloads[i]; 1; }); i++)
@@ -1387,13 +1393,185 @@ static int workloads__fprintf_list(FILE *fp)
return printed;
}
+static int perf_control_open_fifo(struct workload_control *ctl, const char *str)
+{
+ char *s, *p;
+ int ret;
+
+ if (strncmp(str, "fifo:", 5))
+ return -EINVAL;
+
+ str += 5;
+ if (!*str || *str == ',')
+ return -EINVAL;
+
+ s = strdup(str);
+ if (!s)
+ return -ENOMEM;
+
+ p = strchr(s, ',');
+ if (p)
+ *p = '\0';
+
+ ctl->ctl_fd = open(s, O_WRONLY | O_CLOEXEC);
+ if (ctl->ctl_fd < 0) {
+ ret = -errno;
+ pr_err("Failed to open workload control FIFO '%s': %m\n", s);
+ free(s);
+ return ret;
+ }
+
+ if (p && *++p) {
+ ctl->ack_fd = open(p, O_RDONLY | O_CLOEXEC);
+ if (ctl->ack_fd < 0) {
+ ret = -errno;
+ pr_err("Failed to open workload control ack FIFO '%s': %m\n", p);
+ close(ctl->ctl_fd);
+ ctl->ctl_fd = -1;
+ free(s);
+ return ret;
+ }
+ }
+
+ free(s);
+ return 0;
+}
+
+static int perf_control_open(struct workload_control *ctl)
+{
+ int ret;
+
+ if (!workload_control)
+ return 0;
+
+ ret = perf_control_open_fifo(ctl, workload_control);
+
+ if (ret == -EINVAL) {
+ pr_err("Unsupported workload control spec '%s', expected fifo:ctl-fifo[,ack-fifo]\n",
+ workload_control);
+ }
+
+ return ret;
+}
+
+static void perf_control_close(struct workload_control *ctl)
+{
+ if (ctl->ctl_fd >= 0) {
+ close(ctl->ctl_fd);
+ ctl->ctl_fd = -1;
+ }
+ if (ctl->ack_fd >= 0) {
+ close(ctl->ack_fd);
+ ctl->ack_fd = -1;
+ }
+}
+
+static int perf_control_write_cmd(int fd, const char *cmd)
+{
+ size_t len = strlen(cmd);
+ ssize_t ret;
+
+ while (len) {
+ ret = write(fd, cmd, len);
+ if (ret < 0) {
+ if (errno == EINTR)
+ continue;
+ pr_err("Failed to write perf control command: %m\n");
+ return -1;
+ }
+
+ if (!ret) {
+ pr_err("Failed to write perf control command: short write\n");
+ return -1;
+ }
+
+ cmd += ret;
+ len -= ret;
+ }
+
+ return 0;
+}
+
+static int perf_control_read_ack(int fd)
+{
+ char buf[16];
+ ssize_t ret;
+
+ do {
+ ret = read(fd, buf, sizeof(buf) - 1);
+ } while (ret < 0 && errno == EINTR);
+
+ if (ret < 0) {
+ pr_err("Failed to read perf control ack: %m\n");
+ return -1;
+ }
+
+ if (!ret) {
+ pr_err("Unexpected EOF while reading perf control ack\n");
+ return -1;
+ }
+
+ buf[ret] = '\0';
+ for (ssize_t i = 0; i < ret; i++) {
+ if (buf[i] == '\n' || buf[i] == '\0') {
+ buf[i] = '\0';
+ break;
+ }
+ }
+
+ if (strcmp(buf, "ack")) {
+ pr_err("Unexpected perf control ack: %s\n", buf);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int perf_control_send(struct workload_control *ctl, const char *cmd)
+{
+ if (ctl->ctl_fd < 0)
+ return 0;
+
+ if (perf_control_write_cmd(ctl->ctl_fd, cmd))
+ return -1;
+
+ if (ctl->ack_fd >= 0 && perf_control_read_ack(ctl->ack_fd))
+ return -1;
+
+ return 0;
+}
+
static int run_workload(const char *work, int argc, const char **argv)
{
struct test_workload *twl;
workloads__for_each(twl) {
- if (!strcmp(twl->name, work))
- return twl->func(argc, argv);
+ struct workload_control ctl = {
+ .ctl_fd = -1,
+ .ack_fd = -1,
+ };
+ int control_ret, ret;
+
+ if (strcmp(twl->name, work))
+ continue;
+
+ ret = perf_control_open(&ctl);
+ if (ret)
+ return ret;
+
+ if (perf_control_send(&ctl, "enable\n")) {
+ perf_control_close(&ctl);
+ return -1;
+ }
+
+ ret = twl->func(argc, argv);
+
+ control_ret = perf_control_send(&ctl, "disable\n");
+ perf_control_close(&ctl);
+ if (control_ret)
+ return -1;
+
+ return ret;
}
pr_info("No workload found: %s\n", work);
@@ -1486,6 +1664,8 @@ int cmd_test(int argc, const char **argv)
OPT_UINTEGER('r', "runs-per-test", &runs_per_test,
"Run each test the given number of times, default 1"),
OPT_STRING('w', "workload", &workload, "work", "workload to run for testing, use '--list-workloads' to list the available ones."),
+ OPT_STRING(0, "record-ctl", &workload_control, "fifo:ctl-fifo[,ack-fifo]",
+ "Write enable to the fifo just before running the workload and disable after, with optional ack from ack-fifo"),
OPT_BOOLEAN(0, "list-workloads", &list_workloads, "List the available builtin workloads to use with -w/--workload"),
OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"),
OPT_STRING(0, "objdump", &test_objdump_path, "path",
--
2.34.1
^ permalink raw reply related
* [PATCH v4 03/19] perf test: Add a workload that forces context switches
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
In-Reply-To: <20260609-james-cs-context-tracking-fix-v4-0-44f9fb9e5c42@linaro.org>
This workload launches two processes that block when reading and writing
to each other forcing the other process to be scheduled for each
read/write pair.
Signed-off-by: James Clark <james.clark@linaro.org>
---
tools/perf/Documentation/perf-test.txt | 7 +-
tools/perf/tests/builtin-test.c | 1 +
tools/perf/tests/tests.h | 1 +
tools/perf/tests/workloads/Build | 1 +
tools/perf/tests/workloads/context_switch_loop.c | 110 +++++++++++++++++++++++
5 files changed, 117 insertions(+), 3 deletions(-)
diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt
index 2f4a91f5b9dc..213eb62603eb 100644
--- a/tools/perf/Documentation/perf-test.txt
+++ b/tools/perf/Documentation/perf-test.txt
@@ -55,15 +55,16 @@ OPTIONS
-w::
--workload=::
- Run a built-in workload, to list them use '--list-workloads', current ones include:
- noploop, thloop, leafloop, sqrtloop, brstack, datasym and landlock.
+ Run a built-in workload, to list them use '--list-workloads', current
+ ones include: noploop, thloop, leafloop, sqrtloop, brstack, datasym,
+ context_switch_loop and landlock.
Used with the shell script regression tests.
Some accept an extra parameter:
seconds: leafloop, noploop, sqrtloop, thloop
- nrloops: brstack
+ nrloops: brstack, context_switch_loop
The datasym and landlock workloads don't accept any.
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 86ea427eb0aa..9284f897de3c 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -163,6 +163,7 @@ static struct test_workload *workloads[] = {
&workload__traploop,
&workload__inlineloop,
&workload__jitdump,
+ &workload__context_switch_loop,
#ifdef HAVE_RUST_SUPPORT
&workload__code_with_type,
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index bf8ff7d54727..7cd4da4e96d3 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -245,6 +245,7 @@ DECLARE_WORKLOAD(landlock);
DECLARE_WORKLOAD(traploop);
DECLARE_WORKLOAD(inlineloop);
DECLARE_WORKLOAD(jitdump);
+DECLARE_WORKLOAD(context_switch_loop);
#ifdef HAVE_RUST_SUPPORT
DECLARE_WORKLOAD(code_with_type);
diff --git a/tools/perf/tests/workloads/Build b/tools/perf/tests/workloads/Build
index 0eb6d99528eb..7134a031cb7c 100644
--- a/tools/perf/tests/workloads/Build
+++ b/tools/perf/tests/workloads/Build
@@ -10,6 +10,7 @@ perf-test-y += landlock.o
perf-test-y += traploop.o
perf-test-y += inlineloop.o
perf-test-y += jitdump.o
+perf-test-y += context_switch_loop.o
ifeq ($(CONFIG_RUST_SUPPORT),y)
perf-test-y += code_with_type.o
diff --git a/tools/perf/tests/workloads/context_switch_loop.c b/tools/perf/tests/workloads/context_switch_loop.c
new file mode 100644
index 000000000000..5431af6147e6
--- /dev/null
+++ b/tools/perf/tests/workloads/context_switch_loop.c
@@ -0,0 +1,110 @@
+
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/compiler.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/prctl.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "../tests.h"
+
+static int loops = 100;
+static char buf;
+int context_switch_loop_work = 1234;
+
+#define write_block(fd) \
+ do { \
+ if (write(fd, &buf, 1) <= 0) \
+ return 1; \
+ } while (0)
+
+#define read_block(fd) \
+ do { \
+ if (read(fd, &buf, 1) <= 0) \
+ return 1; \
+ } while (0)
+
+/* Not static to avoid LTO clobbering the function name */
+int context_switch_loop_proc1(int in_fd, int out_fd);
+int context_switch_loop_proc1(int in_fd, int out_fd)
+{
+ for (int i = 0; i < loops; i++) {
+ read_block(in_fd);
+ context_switch_loop_work += i * 3;
+ write_block(out_fd);
+ }
+ return 0;
+}
+
+int context_switch_loop_proc2(int in_fd, int out_fd);
+int context_switch_loop_proc2(int in_fd, int out_fd)
+{
+ for (int i = 0; i < loops; i++) {
+ write_block(out_fd);
+ context_switch_loop_work += i * 7;
+ read_block(in_fd);
+ }
+ return 0;
+}
+
+/*
+ * Launches two processes that take turns to execute a multiplication N times
+ */
+static int context_switch_loop(int argc, const char **argv)
+{
+ int a_to_b[2], b_to_a[2];
+ pid_t proc1_pid;
+ int status;
+ int ret;
+
+ if (argc > 0) {
+ loops = atoi(argv[0]);
+ if (loops < 0) {
+ fprintf(stderr, "Invalid number of loops: %s\n", argv[0]);
+ return 1;
+ }
+ }
+
+ if (pipe(a_to_b) || pipe(b_to_a)) {
+ perror("Pipe error");
+ return 1;
+ }
+
+ proc1_pid = fork();
+ if (proc1_pid < 0) {
+ perror("Fork error");
+ return 1;
+ }
+
+ if (!proc1_pid) {
+ close(a_to_b[0]);
+ close(b_to_a[1]);
+ prctl(PR_SET_NAME, "proc1", 0, 0, 0);
+ ret = context_switch_loop_proc1(b_to_a[0], a_to_b[1]);
+ close(a_to_b[1]);
+ close(b_to_a[0]);
+ exit(ret);
+ }
+
+ close(a_to_b[1]);
+ close(b_to_a[0]);
+ prctl(PR_SET_NAME, "proc2", 0, 0, 0);
+ ret = context_switch_loop_proc2(a_to_b[0], b_to_a[1]);
+ close(a_to_b[0]);
+ close(b_to_a[1]);
+
+ if (ret) {
+ kill(proc1_pid, SIGKILL);
+ return ret;
+ }
+
+ if (waitpid(proc1_pid, &status, 0) != proc1_pid || !WIFEXITED(status) ||
+ WEXITSTATUS(status))
+ return 1;
+
+ return 0;
+}
+
+DEFINE_WORKLOAD(context_switch_loop);
--
2.34.1
^ permalink raw reply related
* [PATCH v4 04/19] perf test cs-etm: Test process attribution
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
In-Reply-To: <20260609-james-cs-context-tracking-fix-v4-0-44f9fb9e5c42@linaro.org>
Run the context switch workload on one CPU and trace it to test that
symbols are attributed to the correct process and that the attribution
changes at the exact point that the context switch happened.
Reviewed-by: Leo Yan <leo.yan@arm.com>
Signed-off-by: James Clark <james.clark@linaro.org>
---
.../tests/shell/coresight/context_switch_thread.sh | 69 ++++++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/tools/perf/tests/shell/coresight/context_switch_thread.sh b/tools/perf/tests/shell/coresight/context_switch_thread.sh
new file mode 100755
index 000000000000..2b9c44b86c59
--- /dev/null
+++ b/tools/perf/tests/shell/coresight/context_switch_thread.sh
@@ -0,0 +1,69 @@
+#!/bin/bash -e
+# CoreSight context switch thread attribution (exclusive)
+
+# SPDX-License-Identifier: GPL-2.0
+
+# If CoreSight is not available, skip the test
+perf list pmu | grep -q cs_etm || exit 2
+
+if [ "$(id -u)" != 0 ]; then
+ # Requires root for "-C 0" in record command
+ echo "[Skip] No root permission"
+ exit 2
+fi
+
+tmpdir=$(mktemp -d /tmp/__perf_test.coresight_context_switch.XXXXX)
+
+cleanup() {
+ rm -rf "${tmpdir}"
+ trap - EXIT TERM INT
+}
+
+trap_cleanup() {
+ cleanup
+ exit 1
+}
+trap trap_cleanup EXIT TERM INT
+
+check_samples() {
+ owner_samples=$(grep -c "proc1.*context_switch_loop_proc1" "$tmpdir/script" || true)
+ next_samples=$(grep -c "proc2.*context_switch_loop_proc2" "$tmpdir/script" || true)
+
+ if [ "$owner_samples" -eq 0 ] || [ "$next_samples" -eq 0 ]; then
+ echo "No samples found"
+ cleanup
+ exit 1
+ fi
+
+ if grep "proc2.*context_switch_loop_proc1" "$tmpdir/script"; then
+ echo "Thread1 symbol was attributed to proc2"
+ cleanup
+ exit 1
+ fi
+
+ if grep "proc1.*context_switch_loop_proc2" "$tmpdir/script"; then
+ echo "Thread2 symbol was attributed to proc1"
+ cleanup
+ exit 1
+ fi
+}
+
+cf="$tmpdir/ctl"
+af="$tmpdir/ack"
+mkfifo "$cf" "$af"
+
+# Pin to one CPU so the two threads alternate running but record into the same
+# trace buffer. Start disabled and use the control FIFO to only record the
+# workload and not startup.
+perf record -o "$tmpdir/data" -e cs_etm/timestamp=0/u -C 0 -D -1 --control fifo:"$cf","$af" -- \
+ taskset --cpu-list 0 perf test --record-ctl fifo:"$cf","$af" \
+ -w context_switch_loop > /dev/null 2>&1
+
+# Test both instruction and branch sample generation modes.
+perf script -i "$tmpdir/data" --itrace=i4 -F comm,pid,tid,ip,sym > "$tmpdir/script" 2>/dev/null
+check_samples
+perf script -i "$tmpdir/data" --itrace=b -F comm,pid,tid,ip,sym > "$tmpdir/script" 2>/dev/null
+check_samples
+
+cleanup
+exit 0
--
2.34.1
^ permalink raw reply related
* [PATCH v4 05/19] perf test: Add deterministic workload
From: James Clark @ 2026-06-09 14:30 UTC (permalink / raw)
To: Suzuki K Poulose, Mike Leach, Leo Yan, Arnaldo Carvalho de Melo,
Namhyung Kim, Jiri Olsa, Ian Rogers, Amir Ayupov, Jonathan Corbet,
Shuah Khan, Paschalis Mpeis
Cc: coresight, linux-perf-users, linux-kernel,
Arnaldo Carvalho de Melo, linux-doc, James Clark
In-Reply-To: <20260609-james-cs-context-tracking-fix-v4-0-44f9fb9e5c42@linaro.org>
Add a workload that does the same thing every time for testing CPU trace
decoding.
Reviewed-by: Leo Yan <leo.yan@arm.com>
Signed-off-by: James Clark <james.clark@linaro.org>
---
tools/perf/Documentation/perf-test.txt | 4 +--
tools/perf/tests/builtin-test.c | 1 +
tools/perf/tests/tests.h | 1 +
tools/perf/tests/workloads/Build | 2 ++
tools/perf/tests/workloads/deterministic.c | 39 ++++++++++++++++++++++++++++++
5 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt
index 213eb62603eb..c50a4b2d2d29 100644
--- a/tools/perf/Documentation/perf-test.txt
+++ b/tools/perf/Documentation/perf-test.txt
@@ -57,7 +57,7 @@ OPTIONS
--workload=::
Run a built-in workload, to list them use '--list-workloads', current
ones include: noploop, thloop, leafloop, sqrtloop, brstack, datasym,
- context_switch_loop and landlock.
+ context_switch_loop, deterministic and landlock.
Used with the shell script regression tests.
@@ -66,7 +66,7 @@ OPTIONS
seconds: leafloop, noploop, sqrtloop, thloop
nrloops: brstack, context_switch_loop
- The datasym and landlock workloads don't accept any.
+ The datasym, landlock and deterministic workloads don't accept any.
--list-workloads::
List the available workloads to use with -w/--workload.
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 9284f897de3c..ef7e3f52a383 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -164,6 +164,7 @@ static struct test_workload *workloads[] = {
&workload__inlineloop,
&workload__jitdump,
&workload__context_switch_loop,
+ &workload__deterministic,
#ifdef HAVE_RUST_SUPPORT
&workload__code_with_type,
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index 7cd4da4e96d3..bcfe9c33fc66 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -246,6 +246,7 @@ DECLARE_WORKLOAD(traploop);
DECLARE_WORKLOAD(inlineloop);
DECLARE_WORKLOAD(jitdump);
DECLARE_WORKLOAD(context_switch_loop);
+DECLARE_WORKLOAD(deterministic);
#ifdef HAVE_RUST_SUPPORT
DECLARE_WORKLOAD(code_with_type);
diff --git a/tools/perf/tests/workloads/Build b/tools/perf/tests/workloads/Build
index 7134a031cb7c..90f2d8aa4941 100644
--- a/tools/perf/tests/workloads/Build
+++ b/tools/perf/tests/workloads/Build
@@ -11,6 +11,7 @@ perf-test-y += traploop.o
perf-test-y += inlineloop.o
perf-test-y += jitdump.o
perf-test-y += context_switch_loop.o
+perf-test-y += deterministic.o
ifeq ($(CONFIG_RUST_SUPPORT),y)
perf-test-y += code_with_type.o
@@ -23,3 +24,4 @@ CFLAGS_brstack.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE
CFLAGS_datasym.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE
CFLAGS_traploop.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE
CFLAGS_inlineloop.o = -g -O2
+CFLAGS_deterministic.o = -g -O0 -fno-inline -U_FORTIFY_SOURCE
diff --git a/tools/perf/tests/workloads/deterministic.c b/tools/perf/tests/workloads/deterministic.c
new file mode 100644
index 000000000000..8a78519fd075
--- /dev/null
+++ b/tools/perf/tests/workloads/deterministic.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/compiler.h>
+#include "../tests.h"
+
+int dt_work = 1234;
+
+static void function1(void)
+{
+ dt_work += 7;
+ dt_work += 7;
+ dt_work += 7;
+}
+
+static void function2(void)
+{
+ dt_work += 7;
+ dt_work += 7;
+ dt_work += 7;
+}
+
+static int deterministic(int argc __maybe_unused,
+ const char **argv __maybe_unused)
+{
+ dt_work += 7;
+ dt_work += 7;
+ dt_work += 7;
+
+ function1();
+
+ dt_work += 7;
+ dt_work += 7;
+ dt_work += 7;
+
+ function2();
+
+ return 0;
+}
+
+DEFINE_WORKLOAD(deterministic);
--
2.34.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox